001/* 002 * 003 * $Revision: 569 $ 004 * $Date: 2012-11-05 16:44:26 -0800 (Mon, 05 Nov 2012) $ 005 * $Locker: $ 006 * 007 */ 008package ca.bc.webarts.tools; 009 010import java.util.HashMap; 011import java.util.Properties; 012import javax.json.JsonObject; 013import javax.json.JsonArray; 014import javax.json.JsonNumber; 015import javax.json.JsonString; 016 017import ca.bc.webarts.widgets.ResultSetConverter; 018 019import java.text.DecimalFormat; 020 021 022 /** 023 * The Credential Direct webpage scraper. It is a class that extends the UrlScraper class to wrap/abstract the login and 024 * credentials away and just focusses on the scraping of data from the restricted pages.<br><br> 025 * <a href="https://trading.credentialdirect.com">https://trading.credentialdirect.com</a> is a site requiring login/authentication, so this class provides autoLogin and scraping of sub-pages.<br> 026 * The site login credentials are NOT kept inside the code for this class; they are in a properties file called '<i>.credential</i>' in the '<i>user.home</i>' directory. 027 * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too.<br> 028 * <b>copyright (c) 2017-2018 Tom B. Gutwin</b> 029 **/ 030public class YahooFinanceScraper extends UrlScraper 031{ 032 /** A holder for this clients System File Separator. */ 033 public final static String SYSTEM_FILE_SEPERATOR = java.io.File.separator; 034 035 /** A holder for this clients System line termination separator. */ 036 public final static String SYSTEM_LINE_SEPERATOR = 037 System.getProperty("line.separator"); 038 039 public final static String COL_DELIM = SqlQuery.DEFAULT_COLUMN_DELIMITOR; 040 041 /** The users home ditrectory. */ 042 public static String USERHOME = System.getProperty("user.home"); 043 044 private String yahooFinanceFilename_ = USERHOME+SYSTEM_FILE_SEPERATOR+".yahooFinance"; 045 046 HashMap <String, String> reqProps = new HashMap<String, String>(); 047 048 //https://login.yahoo.com/?.src=fpctx&.intl=ca&.lang=en-CA&authMechanism=primary&yid=&done=https%3A%2F%2Fca.yahoo.com%2F&eid=100&as=1&login=tgutwin&crumb=9D8I.8WDkTW 049 String yLoginUrl_plong = "https://login.yahoo.com/account/challenge/password?.src=fpctx&.intl=ca&.lang=en-CA&authMechanism=primary&yid=tgutwin&done=https%3A%2F%2Ffinance.yahoo.com%2F&as=1&login=tgutwin&crumb=1EAn939fa%2Ft&display=login&s=QQ--&sessionIndex=QQ--&acrumb=FBRp8xFN"; 050 String yLoginUrl_pshort = "https://login.yahoo.com/?.intl=ca&.lang=en-CA&login=tgutwin&.src=fpctx&done=https%3A%2F%2Ffinance.yahoo.com%2F&prefill=0"; //"https://login.yahoo.com/account/challenge/password?.src=fpctx&.intl=ca&.lang=en-CA&authMechanism=primary&yid=tgutwin&done=https%3A%2F%2Ffinance.yahoo.com%2F&as=1&login=tgutwin"; 051 String yLoginUrl = "https://login.yahoo.com/?display=login&login=tgutwin&done=https%3A%2F%2Ffinance.yahoo.com%2F&prefill=1"; //https://login.yahoo.com/?authMechanism=primary&yid=&done=https%3A%2F%2Ffinance.yahoo.com%2F&eid=100&as=1&login=tgutwin&crumb=9D8I.8WDkTW 052 String yScrapePageUrl = "https://finance.yahoo.com/portfolio/pf_2/view?bypass=true"; 053 String quoteSymbolToken = "^%SYMBOL%^"; 054 String yScrapeQuoteTokenizedUrl = "https://finance.yahoo.com/quote/"+quoteSymbolToken+"?p="+quoteSymbolToken; 055 056 String yLoginFormElement = "pure-form pure-form-stacked"; /* id of the form element */ 057 String yUserLoginElement = "displayName"; 058 String yPasswordElement = "login-passwd"; 059 String yUsername = ""; // load from outside somewhere 060 String yPassword = ""; // load from outside somewhere 061 062 String yScrapeQuoteStart= "<body>"; 063 String yScrapeQuoteEnd= "</body>"; 064 065 private JsonArray holdings = null; 066 private JsonObject dataHome = null; 067 068 069 /** 070 * Default constructor for the Yahoo! Finance webpage scraper. 071 * <a href="https://finance.yahoo.com">https://finance.yahoo.com</a> is a site requiring login; this class wraps the login and 072 * credentials away and just focusses on the scraping of data from the restricted pages.<br><br> 073 * 074 * The constructor ONLY gets the class fields and properties setup to go, BUT does no web page access; 075 * that is left up to the doLogin and doScrape methods. 076 **/ 077 public YahooFinanceScraper( ) 078 { 079 super(); 080 081 loadProperties(); 082 083 reqProps.put("Accept","text/html,application/xhtml+xml,application/xml"); 084 reqProps.put("Accept-Encoding ","gzip, deflate, br"); 085 reqProps.put("Accept-Language ","en-US,en;q=0.5"); 086 reqProps.put("Connection","keep-alive"); 087 reqProps.put("Cookie","B=2cisolhbt1ins&b=4&d=RakojeJpYFTgvjVCRIqbwpzSdhmdL.DhjO2tAos4&s=52&i=dIBnpl_3wMdbPr20edhQ; F=d=PF04Ce89vHlp7F7OhDLJaA8kmWCeFNA9hoSr; PH=fn=0sR98pXp9KSrFvTQ&i=ca; AO=u=1; ucs=fs=1&pnid=20160606&pnct=1510186662; PRF=t%3DMA%252BAUTO%252BHIVE.V; T=z=e65AaBeOhFaBdmZrdlPcsf8NDU2MwYwNjEzTjA2MjE-&a=YAE&sk=DAAGdnihCnwYZz&ks=EAA0YpCwZdxHFGE5CoNr79qvQ--~G&kt=EAAphCs8vup1NWv3yXrr9G.Ag--~I&ku=FAAaJDjSxrACUwSZxjMbOzCKCDdnZDhjfWSaCSpGw9o0g4FN5OmPbCCe1EgPbVtqVT3201R.4kooU6E0ys5ckDZ.Awdrp_GvsSboEVMxk4op0vRemPEKkLz0VuT776kPzTiL3Oax7fdNybj7f9_XlMIdfjFPLVfYkeVHwFzjQbmMFw-~A&d=bnMBeWFob28BZwFZTUFZNEUzWVFDV1pURDZZNUlRRDJMR1pWWQFzbAFNekl4TkFFM01UWTBPVGN4TlRZLQFhAVlBRQFhYwFBSTFUZ1VWLgFvawFaVzAtAXNjAWRlc2t0b3Bfd2ViAWZzAU5MQkVNSlpaem1pUwF6egFlNjVBYUJBN0U-&af=JnRzPTE1MTAxODY2NTQmcHM9dktfMmwySGJPWmV2Lk5zQjZETG5ydy0t; Y=v=1&n=6vum1udm49nmm&l=j6kjm8d/o&p=m23vvca00000000&jb=25|37|18&ig=0a91i&iz=V7L+2S4&r=4s&intl=ca; SSL=v=1&s=RbCrkRc8alNIG2QdIqj8AG7wtmf2bAz0NNq7EF6Jm4GlxfwBKlScw9ZcIi7VNIORunod8MguYRQ2Mbhev4MJww--&kv=0&ku=kBQsrjE98YO.ldgA2MgKqTrLJkjMRiWbd._3Em4fef7UhBYnc.EYhRzG7sb3OY2RVxTy6y.gMHkVjMZwZyXTMK9dqASDf0Dm8RzMvcmi6s5zkrVO5V5enEkpJBunxuF4t936zvXp8Gk0DpnJCY_ATGJTI6.ww75pX9q0G3iejMo-~A"); 088 reqProps.put("Content-Type","application/x-www-form-urlencoded"); 089 reqProps.put("Upgrade-Insecure-Requests","1"); 090 reqProps.put("Host","finance.yahoo.com.com"); 091 092 setLoginUrl(yLoginUrl_pshort); 093 setLoginFormID(yLoginFormElement); 094 setUsernameFormElementName(yUserLoginElement); 095 setPasswordFormElementName(yPasswordElement); 096 setUsername(yUsername); 097 setPassword(yPassword); 098 setScrapePageUrl(yScrapePageUrl); 099 setScrapeStart("<body>"); 100 setScrapeEnd("</body>"); 101 setRequestProps(reqProps); 102 } 103 104 105 /** 106 * Load the private properties from an external/private file. 107 * Site login credentials are NOT kept in this class; they are in a properties file called .credential in the 'user.home' directory. 108 * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too. 109 * 110 * @return truee if successful, false if could not load from file 111 **/ 112 private boolean loadProperties() 113 { 114 boolean retVal = false; 115 try 116 { 117 //get user/pass from ~/.credential 118 Properties yProps = new Properties(); 119 yProps.load(new java.io.FileReader(yahooFinanceFilename_)); 120 yUsername = yProps.getProperty("username"); 121 yPassword = yProps.getProperty("password"); 122 retVal = true; 123 } 124 catch (Exception ex) 125 { 126 // use defaults or set before use 127 System.out.println("ERROR : did not read props from :"+yahooFinanceFilename_); 128 } 129 130 return retVal; 131 } 132 133 134 /** 135 * Sends the POST to the login url parameters from the classVars. 136 * It also caches the page response text into thew classVar loginPageResponse_.<br> 137 * Yahoo Login Form:<pre> 138 * <form method="post" class="pure-form pure-form-stacked"> 139 * <input type="hidden" name="browser-fp-data" id="browser-fp-data" value="" /> 140 * <input type="hidden" name="crumb" value="9D8I.8WDkTW" /> 141 * <input type="hidden" name="acrumb" value="FBRp8xFN" /> 142 * <input type="hidden" name="sessionIndex" value="QQ--" /> 143 * <input type="hidden" name="displayName" value="tgutwin" /> 144 * <button type="submit" name="rejectTarget" value="yak" 145 * class="pure-button puree-button-primary puree-spinner-button" 146 * data-ylk="elm:btn;elmt:selectYak;slk:selectYak"> 147 * Sign in using your Yahoo app 148 * </button> 149 * <p class="seperator-or">or</p> 150 * <p class="m-t-8px secondary-sign-in-link"> 151 * <button type="submit" name="skipInterstitial" value="eyJwYXNzd29yZENvbnRleHQiOiJ5YWtFbGlnaWJsZSJ9" 152 * class="pure-button puree-button-link" 153 * data-ylk="elm:btn;elmt:selectSecondary;slk:selectSecondary"> 154 * Sign in with your password 155 * </button> 156 * </p> 157 * </form> 158 159 <!-- Password direct Form page --> 160 <form action="" method="post" class="pure-form pure-form-stacked"> 161 <input type="hidden" name="browser-fp-data" id="browser-fp-data" value="" /> 162 <input type="hidden" name="crumb" value="9D8I.8WDkTW" /> 163 <input type="hidden" name="acrumb" value="FBRp8xFN" /> 164 <input type="hidden" name="sessionIndex" value="QQ--" /> 165 <input type="hidden" name="displayName" value="tgutwin" /> 166 <div class="hidden-username"> 167 <input type="text" tabindex="-1" aria-hidden="true" role="presentation" 168 autocorrect="off" spellcheck="false" 169 name="username" value="tgutwin" /> 170 </div> 171 <input type="hidden" name="passwordContext" value="yakEligible" /> 172 <input type="password" id="login-passwd" name="password" placeholder="Password" autofocus/> 173 <p class="signin-cont"> 174 <button type="submit" id="login-signin" class="pure-button puree-button-primary puree-spinner-button" name="verifyPassword" value="Sign in" data-ylk="elm:btn;elmt:next;slk:next"> 175 Sign in 176 </button> 177 </p> 178 <p class="forgot-cont"> 179 <input type="submit" class="pure-button puree-button-link" 180 data-ylk="elm:btn;elmt:skip;slk:skip" id="mbr-forgot-link" 181 name="skip" value="I forgot my password" /> 182 </p> 183 <p class="yak-signin-or">or</p> 184 <button type="submit" class="yak-signin-card" 185 data-ylk="elm:btn;elmt:selectYak;slk:selectYak" 186 name="yakPreferred" value="yakPreferred"> 187 <div class="yak-signin-desc"> 188 <p class="desc-first">Try the new way to sign in</p> 189 <p class="desc-second">Use your phone instead</p> 190 </div> 191 </button> 192 </form> 193 * </pre> 194 **/ 195 @Override 196 public boolean doLogin() 197 { 198 boolean retVal = alreadyLoggedIn_; 199 if(!alreadyLoggedIn_) 200 if(!"".equals(getUsername()) && !"".equals(getPassword())) 201 retVal = super.doLogin(); 202 else 203 System.out.println("ERROR reading username and password"); 204 205 return retVal; 206 } 207 208 209 /** Create the stock QUOTE data string for the given stock symbol for today. 210 * this data gets used as the data load into the InvestmentTrackerQuery database.<br> 211 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 212 * open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare, dividendPayDate<br> 213 * 90.01, 89.13, 90.01, 90.34, 386147, 4.18, "10/27/2017" 214 * 215 * @param stockSymbol is the Symbol of the stock to lookup 216 * @param country is the stock market country (US, CA etc) 217 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare, dividendPayDate OR '' if not found 218 **/ 219 private String getQuoteString(String stockSymbol, String country) 220 { 221 222 String retVal = ""; 223 boolean success = false; 224 boolean useDataTool = false; 225 226 String currPrefix = (country.equalsIgnoreCase("us")?"":""); 227 String currSuffix = (country.equalsIgnoreCase("us")?"":(!stockSymbol.equalsIgnoreCase("HIVE")?".TO":".V")); 228 String tokenReplacedScrapeUrl = yScrapeQuoteTokenizedUrl.replace(quoteSymbolToken,currPrefix+stockSymbol+currSuffix); 229 setScrapePageUrl(tokenReplacedScrapeUrl); 230 231 HashMap <String, String> reqProps = null; 232 233 if(holdings==null) 234 { 235 if(!"".equals(getUsername()) && !"".equals(getPassword())) 236 { 237 success = doLogin(); 238 System.out.println("\nLogin Response:\n"+postPageResponse_); 239 } 240 else 241 System.out.println("ERROR reading username and password"); 242 243 if(success) 244 { 245 String result = doScrape(getScrapePageUrl(), reqProps); // without the trimming 246 retVal = result; 247 } 248 } 249 return retVal; 250 } 251 252 253 254 /** Create the STOCK_DAILY data string for the given stock symbol for today. 255 * this data gets used as the data load into the InvestmentTrackerQuery database.<br> 256 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 257 * open, daysLow, daysHigh, close, previousClose, daysVolume, date, dividendPerShare, dividendPayDate<br> 258 * 90.01, 89.13, 90.01, 90.00, 90.34, 386147, "11/08/2017", 4.18, "10/27/2017" 259 * 260 * @param stockSymbol is the Symbol of the stock to lookup * 261 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare, dividendPayDate OR '' if not found 262 **/ 263 public String getStockDailyString(String stockSymbol) 264 { 265 String retVal = ""; 266 boolean success = false; 267 JsonObject dataHome = null; 268 if(holdings==null) 269 { 270 if(!"".equals(getUsername()) && !"".equals(getPassword())) 271 success = doLogin(); 272 else 273 System.out.println("ERROR reading username and password"); 274 275 if(success) 276 { 277 // scape the summary first to get a list of holdings 278 String result = doScrape(); 279 JsonObject jsO = toJsonObject(result); 280 JsonArray jsA = jsO.getJsonArray("Data"); 281 dataHome = jsA.getJsonObject(0); 282 holdings = dataHome.getJsonArray("Holdings"); 283 } 284 } 285 else 286 System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]"); 287 288 if(dataHome!=null && holdings!=null) 289 { 290 retVal+="\"Open\""; 291 retVal+=COL_DELIM; 292 retVal+="\"daysLow\""; 293 retVal+=COL_DELIM; 294 retVal+="\"daysHigh\""; 295 retVal+=COL_DELIM; 296 retVal+="\"close\""; 297 retVal+=COL_DELIM; 298 retVal+="\"previousClose\""; 299 retVal+=COL_DELIM; 300 retVal+="\"date\""; 301 retVal+=COL_DELIM; 302 retVal+="\"dividendPerShare\""; 303 retVal+=COL_DELIM; 304 retVal+="\"dividendPayDate\""; 305 retVal+="\n"; 306 307 setScrapeStart(yScrapeQuoteStart); 308 setScrapeEnd(yScrapeQuoteEnd); 309 String currSymbol = ""; 310 String currPrefix = ""; 311 JsonObject stock =null; 312 313 for(int i=0; i<holdings.size();i++) 314 { 315 stock = holdings.getJsonObject(i); 316 if(stock!=null) 317 { 318 currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1); 319 currPrefix = stock.getJsonString("CountryPrefix").toString().substring(1,stock.getJsonString("CountryPrefix").toString().length()-1); 320 321 if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol.toString())) 322 { 323 324 //get the daily quote for each individual holding 325 setScrapePageUrl(yScrapeQuoteTokenizedUrl.replace(quoteSymbolToken,currPrefix+currSymbol)); 326 String result = doScrape(); 327 System.out.println(" DEBUG Stock Quote Result \n----------------------------\n"); 328 System.out.println(result); 329 330 /* 331 JsonObject jsO = toJsonObject(result); 332 JsonString currSymbol = stock.getJsonString("Symbol"); 333 if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol.toString())) 334 { 335 JsonNumber open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare; 336 String dividendPayDate; 337 //open=stock.getJsonString("Price"); 338 retVal+="\""+dataHome.getString("FriendlyName")+"\""; 339 retVal+=COL_DELIM; 340 retVal+=""+dataHome.getJsonString("Currency"); 341 retVal+=COL_DELIM; 342 343 } 344 */ 345 i=holdings.size(); 346 } 347 } 348 } 349 } 350 351 return retVal; 352 } 353 354 355 /** Create the Portfolio Summary data string for the CR account for today. 356 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 357 * FriendlyName, Currency, BookValue, TradeCash, MarketValue, EquityValue, UnrealizedGainLoss, UnrealizedGainLossPercent, numHoldings, date<br> 358 * "#2V9669V6 - TFSA", 50.5, 10.1, 66.4, 15.9, 8, "10/27/2017" 359 * 360 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 361 **/ 362 public String getPortfolioSummaryString() 363 { 364 String retVal = ""; 365 boolean success = false; 366 JsonObject dataHome = null; 367 if(holdings==null) 368 { 369 if(!"".equals(getUsername()) && !"".equals(getPassword())) 370 success = doLogin(); 371 else 372 System.out.println("ERROR reading username and password"); 373 374 if(success) 375 { 376 String result = doScrape(); 377 JsonObject jsO = toJsonObject(result); 378 JsonArray jsA = jsO.getJsonArray("Data"); 379 dataHome = jsA.getJsonObject(0); 380 //holdings = dataHome.getJsonArray("Holdings"); 381 } 382 } 383 else 384 System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]"); 385 386 if(dataHome!=null) // && holdings!=null) 387 { 388 retVal+="\"friendlyName\""; 389 retVal+=COL_DELIM; 390 retVal+="\"Currency\""; 391 retVal+=COL_DELIM; 392 retVal+="\"BookValue\""; 393 retVal+=COL_DELIM; 394 retVal+="\"TradeCash\""; 395 retVal+=COL_DELIM; 396 retVal+="\"MarketValue\""; 397 retVal+=COL_DELIM; 398 retVal+="\"EquityValue\""; 399 retVal+=COL_DELIM; 400 retVal+="\"UnrealizedGainLoss\""; 401 retVal+=COL_DELIM; 402 retVal+="\"UnrealizedGainLossPercent\""; 403 retVal+=COL_DELIM; 404 retVal+="\"NumberOfHoldings\""; 405 retVal+=COL_DELIM; 406 retVal+="\"date\""; 407 retVal+="\n"; 408 409 retVal+="\""+dataHome.getString("FriendlyName")+"\""; 410 retVal+=COL_DELIM; 411 retVal+=""+dataHome.getJsonString("Currency"); 412 retVal+=COL_DELIM; 413 retVal+=""+dataHome.getJsonNumber("BookValue"); 414 retVal+=COL_DELIM; 415 retVal+=""+dataHome.getJsonNumber("TradeCash"); 416 retVal+=COL_DELIM; 417 retVal+=""+dataHome.getJsonNumber("MarketValue"); 418 retVal+=COL_DELIM; 419 retVal+=""+dataHome.getJsonNumber("EquityValue"); 420 retVal+=COL_DELIM; 421 retVal+=""+dataHome.getJsonNumber("UnrealizedGainLoss"); 422 retVal+=COL_DELIM; 423 retVal+=""+dataHome.getJsonNumber("UnrealizedGainLossPercent"); 424 retVal+=COL_DELIM; 425 retVal+=""+(holdings!=null?holdings.size():"0"); 426 retVal+=COL_DELIM; 427 retVal+="\""+dateStr_+"\""; 428 429 /* 430 for(int i=0; i<holdings.size();i++) 431 { 432 JsonObject stock = holdings.getJsonObject(i); 433 if(stock!=null) 434 { 435 String currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1); 436 System.out.println(currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+ 437 stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+ 438 " = "+stock.getJsonNumber("MarketValue").toString()); 439 } 440 } 441 */ 442 } 443 444 return retVal; 445 } 446 447 448 449 /** Create the Portfolio data string for the CR account for today. 450 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 451 * stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br> 452 * Hive Tech, HIVE, HIVE, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, "10/27/2017" 453 * 454 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 455 **/ 456 public String getPortfolioString(){return getPortfolioString(true);} 457 458 459 460 /** Create the Portfolio data string for the CR account for today. 461 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 462 * stockName, symbol, CRExchangeSymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br> 463 * Hive Tech, HIVE, V, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, 10-27-2017 464 * 465 * @param resultSetOnly true to return only the default delimited resultSet OR false to send a longer more readable string 466 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 467 **/ 468 public String getPortfolioString(boolean resultSetOnly) 469 { 470 String retVal = "stockName"+COL_DELIM+"symbol"+COL_DELIM+"CRQuerySymbol"+COL_DELIM+"numShares"+COL_DELIM+"currentPrice"+COL_DELIM+ 471 "currency"+COL_DELIM+"bookValue"+COL_DELIM+"marketValue"+COL_DELIM+"gain"+COL_DELIM+"gainPercent"+COL_DELIM+"currentDate\n"; 472 if(!resultSetOnly) retVal = "\n\n---------------------------------\n Portfolio Summary\n---------------------------------\n"; 473 474 DecimalFormat dfp = new DecimalFormat( "##0" ); 475 DecimalFormat dfe = new DecimalFormat( "##0.000" ); 476 boolean success = false; 477 478 if(holdings==null) 479 { 480 if(!"".equals(getUsername()) && !"".equals(getPassword())) 481 success = doLogin(); 482 else 483 System.out.println("ERROR reading username and password"); 484 485 if(success) 486 { 487 String result = doScrape(); 488 JsonObject jsO = toJsonObject(result); 489 JsonArray jsA = jsO.getJsonArray("Data"); 490 dataHome = jsA.getJsonObject(0); 491 holdings = dataHome.getJsonArray("Holdings"); 492 } 493 } 494 else 495 System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]"); 496 497 if(dataHome!=null && holdings!=null) 498 { 499 if(!resultSetOnly) 500 { 501 String friendlyName = dataHome.getString("FriendlyName"); 502 503 retVal+=" FriendlyName : "+friendlyName; 504 retVal+="\n"; 505 retVal+=" MarketValue : "+dataHome.getJsonNumber("MarketValue"); 506 retVal+="\n"; 507 retVal+=" TradeCash : "+dataHome.getJsonNumber("TradeCash"); 508 retVal+="\n"; 509 retVal+=" BookValue : "+dataHome.getJsonNumber("BookValue"); 510 retVal+="\n"; 511 retVal+=" ------------------ ------------------------"; 512 retVal+="\n"; 513 retVal+=" UnrealizedGainLoss : "+dataHome.getJsonNumber("UnrealizedGainLoss"); 514 retVal+="\n"; 515 retVal+=" UnrealizedGain% : "+ 516 dfe.format(dataHome.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0); 517 retVal+="\n"; 518 retVal+="\n Holdings Summary\n - - - - - - - - - - - - -\n"; 519 } 520 521 String currSymbol = ""; 522 String priceStr = ""; 523 String currencySymbol = ""; 524 JsonObject stock = null; 525 for(int i=0; i<holdings.size();i++) 526 { 527 stock = holdings.getJsonObject(i); 528 if(stock!=null) 529 { 530 currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1); 531 if(!resultSetOnly) 532 { 533 retVal+=" "+currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+ 534 stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+ 535 " = "+stock.getJsonNumber("MarketValue").toString(); 536 retVal+="\n"; 537 } 538 else 539 { 540 if(right(stock.getJsonString("Price").toString() ,2).equals("U\"")) 541 { 542 priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-2); 543 currencySymbol = "US$"; 544 } 545 else 546 { 547 priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1); 548 currencySymbol = "C$"; 549 } 550 //stockName, symbol, CRExchangeSymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 551 retVal+=stock.getString("SymbolDescription").toString()+COL_DELIM+ 552 currSymbol+COL_DELIM+ 553 stock.getString("Exchange").toString()+COL_DELIM+ 554 stock.getJsonNumber("Quantity").toString()+COL_DELIM+ 555 priceStr+COL_DELIM+ 556 currencySymbol+COL_DELIM+ 557 stock.getJsonNumber("BookValue").toString()+COL_DELIM+ 558 stock.getJsonNumber("MarketValue").toString()+COL_DELIM+ 559 stock.getJsonNumber("UnrealizedGainLoss").toString()+COL_DELIM+ 560 dfe.format(stock.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0)+COL_DELIM+ 561 dateStr_; 562 retVal+="\n"; 563 } 564 } 565 } 566 if(!resultSetOnly) retVal+="\n---------------------------\nHoldings Details:\n---------------------------\n"+ 567 prettyJson(holdings.toString()); 568 } 569 570 return retVal; 571 } 572 573 574 public String right(String value, int length) 575 { 576 // To get right characters from a string, change the begin index. 577 return value.substring(value.length() - length); 578 } 579 580 581 /** Test method to do whatever tests I want. **/ 582 @Override 583 protected void test(String[] args) 584 { 585 super.debugOut_=true; 586 debugOut_=true; 587 588 if(true) 589 { 590 String symbol = "HIVE"; 591 String result = getQuoteString(symbol,"CA"); 592 System.out.println("Test Quote ResultSet For HIVE\n"+result); 593 writeStringToFile(result,"yQuote-"+symbol+".txt"); 594 } 595 else if(false) 596 { 597 System.out.println("..."); 598 try 599 { 600 601 } 602 catch (Exception ex) 603 { 604 System.out.println("NO GO..."); 605 ex.printStackTrace(); 606 } 607 } 608 } 609 610 611 public static void main(String[] args) throws Exception 612 { 613 YahooFinanceScraper instance = new YahooFinanceScraper(); 614 615 if(args.length>0 && args[0].toLowerCase().equals("-t")) 616 instance.test(args); 617 else 618 { 619 boolean success = false; 620 if(!"".equals(instance.getUsername()) && !"".equals(instance.getPassword())) 621 success = instance.doLogin(); 622 else 623 System.out.println("ERROR reading username and password"); 624 625 if(success) 626 { 627 String result = instance.doScrape(); 628 System.out.println("Saving Scraped page results to file: "+"crPageContent-"+instance.dateStr_+".json"); 629 writeStringToFile(prettyJson(result),"crPageContent-"+instance.dateStr_+".json"); 630 631 System.out.println(instance.getPortfolioString()); 632 } 633 } 634 } 635}