package ADSCRAWLER; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.net.URL; import java.nio.charset.Charset; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; //import TESTTHIS.SiteObject; public class APPSCANNER { //APP SCANNER // BUILT BY YAWN public static void main(String[] args) { // TODO Auto-generated method stub System.out.println("APP CRUNCHER......."); List mysites = new ArrayList(); List Blacklisted_sites = new ArrayList(); String MUV = ""; //String filename = "C:///JAVA/SIMILARWEB/similarwebreport.txt"; String filelist = "C:///JAVA/APPSCANNER/APPlist2.txt"; String fileblacklist = "C:///JAVA/APPSCANNER/GlobalAPPBlackList.txt"; String fileblacklistreport = "C:///JAVA/APPSCANNER/BlackListReport.txt"; String website = "bbc.com"; String fileheaders = "Ticket \t Developer \t Bundle id \t Title \t category \t Category Keys \t Rating \t User Rating \t Downloads \t Blacklisted \t Ads.txt \t Market Source \t Market Status \t Contains Ads \t Available In \t Not Available In \t Screen shots"; String fileheadersios = "Ticket \t Seller Name\t Developer\t Bundle ID\t Title\t Category\t IAB Category\t Rating\t User Rating\t Rank\t Release date\t Blacklisted\t Ads.txt\t Market Status\t Languages\t Available in\t Screen shot "; String ticketid,bundleid,adstxt,category,rating,screenshot; try { System.out.println("Loading Global Black list..."); File file2 = new File(fileblacklist); FileReader fr2 = new FileReader(file2); BufferedReader br2 = new BufferedReader(fr2); String line2; while((line2 = br2.readLine()) != null){ Blacklisted_sites.add(line2); } br2.close(); System.out.println("Blacklist is loaded.."); //LOAD APP LIST TO CHECK //READ SITE LIST TO BE CHECKED String sitedata = "torontosun.com"; File file = new File(filelist); FileReader fr = new FileReader(file); BufferedReader br = new BufferedReader(fr); String line; while((line = br.readLine()) != null){ //process the line String tickettemp[] = line.split("\\t"); SiteObject so = new SiteObject(tickettemp[0],tickettemp[1]); mysites.add(so); } br.close(); System.out.println("Site list loaded... Processing..."); writerecordtofile(fileheaders); writerecordtofileIOS(fileheadersios); for(int z =0; z < mysites.size(); z++) { SiteObject so = mysites.get(z); website = so.getDomain(); sitedata = website; boolean isblacklisted = false; if(Blacklisted_sites.contains(website)) { String finalstring = ""; isblacklisted = true; finalstring = so.getTicket() +","+so.getDomain(); finalstring+= "N/A\t N/A\t N/A\t N/A\t N/A\t N/A\t N/A\t N/A\t N/A\t N/A\t Blacklisted"; System.out.println(mysites.get(z) +" Found on GBL"); System.out.println(finalstring); try { //WRITE TO BLACKLIST REPORT BufferedWriter writerfinalreport = new BufferedWriter(new FileWriter(fileblacklistreport, true)); writerfinalreport.newLine(); writerfinalreport.write(finalstring); writerfinalreport.flush(); writerfinalreport.newLine(); } catch(Exception dfdfd) { } } else { //HERE WE NEED TO START CALLING OUR API AND GETTING OUR DATA boolean isandroid = false; try { int y = Integer.parseInt(so.getDomain()); isandroid = false; } catch(Exception fdfdfd) { isandroid = true; //was not numeric so call android } if(isandroid) { //ANDROID mysites.get(z).setOS("android"); adstxt = get_Android_APIS(mysites.get(z)); } else { //IOS mysites.get(z).setOS("IOS"); //adstxt = get_IOS_APIS(mysites.get(z)); get_IOS_Data(mysites.get(z)); } } } } catch(Exception dfgdf) { System.out.println("I crasher because " + dfgdf); } } //CALLED TO START GATHERING IOS DATA FOR REPORT public static void get_IOS_Data(SiteObject so) { System.out.println("IN Callis IOS"); try { //get json objects search for match update SO and write to file. String APIString = "https://data.42matters.com/api/v3.0/ios/apps/app_ads_txt.json?id="+so.getDomain()+"&access_token=4fca25efb66a81a0362aa4ac2ff2887054b4bbe2"; System.out.println("Calling " + APIString); JSONObject json = readJsonFromUrl(APIString); for(int i = 0; i