001/* 002 * 003 * $Revision: 569 $ 004 * $Date: 2012-11-05 16:44:26 -0800 (Mon, 05 Nov 2012) $ 005 * $Locker: $ 006 * 007 */ 008package ca.bc.webarts.tools; 009 010import java.util.HashMap; 011import java.util.Properties; 012import javax.json.JsonObject; 013import javax.json.JsonArray; 014//import javax.json.JsonNumber; 015//import javax.json.JsonString; 016 017//import ca.bc.webarts.widgets.ResultSetConverter; 018 019import java.text.DecimalFormat; 020 021 022 /** 023 * The Credential Direct webpage scraper. It is a class that extends the UrlScraper class to wrap/abstract the login and 024 * credentials away and just focusses on the scraping of data from the restricted pages.<br><br> 025 * <a href="https://trading.credentialdirect.com">https://trading.credentialdirect.com</a> is a site requiring login/authentication, so this class provides autoLogin and scraping of sub-pages.<br> 026 * The site login credentials are NOT kept inside the code for this class; they are in a properties file called '<i>.credential</i>' in the '<i>user.home</i>' directory. 027 * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too.<br> 028 * <b>copyright (c) 2017-2018 Tom B. Gutwin</b> 029 **/ 030public class MyPortfolioScraper extends UrlScraper 031{ 032 /** A holder for this clients System File Separator. */ 033 public final static String SYSTEM_FILE_SEPERATOR = java.io.File.separator; 034 035 /** A holder for this clients System line termination separator. */ 036 public final static String SYSTEM_LINE_SEPERATOR = 037 System.getProperty("line.separator"); 038 039 public final static String COL_DELIM = SqlQuery.DEFAULT_COLUMN_DELIMITOR; 040 041 /** The users home ditrectory. */ 042 public static String USERHOME = System.getProperty("user.home"); 043 044 private String myPortfolioFilename_ = USERHOME+SYSTEM_FILE_SEPERATOR+".myPortfolio"; 045 046 HashMap <String, String> reqProps = new HashMap<String, String>(); 047 048 String pLoginUrl = "https://www.my-portfolio.ca/pcis/login.jsp?external=&fullsite="; 049 String pScrapePageUrl = "https://www.my-portfolio.ca/portfolio/pf_2/view?bypass=true"; 050 String quoteSymbolToken = "^%SYMBOL%^"; 051 String pScrapeQuoteTokenizedUrl = "https://idms.my-portfolio.ca/secured/popup/quote.idms?SECURITY_TYPE=STO&COUNTRY=CAN&SYMBOL="+quoteSymbolToken; 052 053 054 String pLoginFormElement = "loginForm"; /* id of the form element */ 055 String pUserLoginElement = "username"; 056 String pPasswordElement = "password"; 057 String pUsername = ""; // load from outside somewhere 058 String pPassword = ""; // load from outside somewhere 059 060 String pScrapeQuoteStart= "<body>"; 061 String pScrapeQuoteEnd= "</body>"; 062 063 private JsonArray holdings = null; 064 private JsonObject dataHome = null; 065 066 067 /** 068 * Default constructor for the My-portfolio.ca webpage scraper. 069 * <a href="https://www.my-portfolio.ca">https://www.my-portfolio.ca</a> is a site requiring login; this class wraps the login and 070 * credentials away and just focusses on the scraping of data from the restricted pages.<br><br> 071 * 072 * The constructor ONLY gets the class fields and properties setup to go, BUT does no web page access; 073 * that is left up to the doLogin and doScrape methods. 074 **/ 075 public MyPortfolioScraper( ) 076 { 077 super(); 078 079 loadProperties(); 080 /* 081 GET /pcis/login.jsp?external=&fullsite= HTTP/1.1 082 Host: www.my-portfolio.ca 083 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:58.0) Gecko/20100101 Firefox/58.0 084 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,* /*;q=0.8 085 Accept-Language: en-US,en;q=0.5 086 Accept-Encoding: gzip, deflate, br 087 Referer: https://www.my-portfolio.ca/pcis/login.jsp?url=/.do 088 Cookie: JSESSIONID=A555AEE5099100D0821F2D16F29593DC.t23p12; username=76836836; check=1; lang=en; hostname=www.my-portfolio.ca; sea=1510429423370; protocol=https; language=en; BIGipServer~PR_MSESZ~WWW.MY-PORTFOLIO.CA_HTTPS_443.app~WWW.MY-PORTFOLIO.CA_HTTPS_443_pool=rd202o00000000000000000000ffffac10c229o443; auth=478e6d44b73b2daaea3f8ee9ff51b4c3a6b00b8381f533df6f647048dc893c2180bc9443b6ad7f588f6960fcfa1d3543c61a909f44d56fac; fontSize=1 089 Connection: keep-alive 090 Upgrade-Insecure-Requests: 1 091 */ 092 reqProps.put("Host","www.my-portfolio.ca"); 093 reqProps.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); 094 reqProps.put("Accept-Encoding ","gzip, deflate, br"); 095 reqProps.put("Accept-Language ","en-US,en;q=0.5"); 096 reqProps.put("Connection","keep-alive"); 097 reqProps.put("Cookie",//"JSESSIONID=955E5FC778B8D711CC4E1DBC0623963F.t23p12; "+ 098 "username=76836836; check=1; lang=en; hostname=www.my-portfolio.ca; "+ 099 "sea=1510429408165; protocol=https; language=en; "+ 100 "BIGipServer~PR_MSESZ~WWW.MY-PORTFOLIO.CA_HTTPS_443.app~WWW.MY-PORTFOLIO.CA_HTTPS_443_pool=rd202o00000000000000000000ffffac10c229o443; "+ 101 "auth=478e6d44b73b2daaea3f8ee9ff51b4c3a6b00b8381f533df6f647048dc893c2180bc9443b6ad7f588f6960fcfa1d3543c61a909f44d56fac; "+ 102 "fontSize=1"); 103 reqProps.put("Content-Type","application/x-www-form-urlencoded"); 104 reqProps.put("Upgrade-Insecure-Requests","1"); 105 reqProps.put("Referer","https://www.my-portfolio.ca/pcis/login.jsp?url=/.do"); 106 107 setLoginUrl(pLoginUrl); 108 setLoginFormID(pLoginFormElement); 109 setUsernameFormElementName(pUserLoginElement); 110 setPasswordFormElementName(pPasswordElement); 111 setUsername(pUsername); 112 setPassword(pPassword); 113 setScrapePageUrl(pScrapePageUrl); 114 setScrapeStart("<body>"); 115 setScrapeEnd("</body>"); 116 setRequestProps(reqProps); 117 } 118 119 120 /** 121 * Load the private properties from an external/private file. 122 * Site login credentials are NOT kept in this class; they are in a properties file called .credential in the 'user.home' directory. 123 * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too. 124 * 125 * @return truee if successful, false if could not load from file 126 **/ 127 private boolean loadProperties() 128 { 129 boolean retVal = false; 130 try 131 { 132 //get user/pass from ~/.credential 133 Properties pProps = new Properties(); 134 pProps.load(new java.io.FileReader(myPortfolioFilename_)); 135 pUsername = pProps.getProperty("username"); 136 pPassword = pProps.getProperty("password"); 137 retVal = true; 138 } 139 catch (Exception ex) 140 { 141 // use defaults or set before use 142 System.out.println("ERROR : did not read props from :"+myPortfolioFilename_); 143 } 144 145 return retVal; 146 } 147 148 149 /** 150 * Sends the POST to the login url parameters from the classVars. 151 * It also caches the page response text into thew classVar retVal.<br> 152 153 **/ 154 @Override 155 public boolean doLogin() 156 { 157 boolean retVal = alreadyLoggedIn_; 158 if(!alreadyLoggedIn_) 159 if(!"".equals(getUsername()) && !"".equals(getPassword())) 160 retVal = super.doLogin(); 161 else 162 System.out.println("ERROR reading username and password"); 163 164 return retVal; 165 } 166 167 168 /** Create the stock QUOTE data string for the given stock symbol for today. 169 * this data gets used as the data load into the InvestmentTrackerQuery database.<br> 170 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 171 * open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare, dividendPayDate<br> 172 * 90.01, 89.13, 90.01, 90.34, 386147, 4.18, "10/27/2017" 173 * 174 * @param stockSymbol is the Symbol of the stock to lookup 175 * @param country is the stock market country (US, CA etc) 176 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare, dividendPayDate OR '' if not found 177 **/ 178 private String getQuoteString(String stockSymbol, String country) 179 { 180 181 String retVal = ""; 182 boolean success = false; 183 184 String currPrefix = (country.equalsIgnoreCase("us")?"":""); 185 String currSuffix = (country.equalsIgnoreCase("us")?"":(!stockSymbol.equalsIgnoreCase("HIVE")?".TO":".V")); 186 String tokenReplacedScrapeUrl = pScrapeQuoteTokenizedUrl.replace(quoteSymbolToken,currPrefix+stockSymbol+currSuffix); 187 setScrapePageUrl(tokenReplacedScrapeUrl); 188 189 HashMap <String, String> reqProps = null; 190 191 if(holdings==null) 192 { 193 if(!"".equals(getUsername()) && !"".equals(getPassword())) 194 { 195 success = doLogin(); 196 //System.out.println("\nLogin Response:\n"+postPageResponse_); 197 } 198 else 199 System.out.println("ERROR reading username and password"); 200 201 if(success) 202 { 203 String result = doScrape(getScrapePageUrl(), reqProps); // without the trimming 204 retVal = result; 205 } 206 } 207 return retVal; 208 } 209 210 211 212 /** Create the STOCK_DAILY data string for the given stock symbol for today. 213 * this data gets used as the data load into the InvestmentTrackerQuery database.<br> 214 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 215 * open, daysLow, daysHigh, close, previousClose, daysVolume, date, dividendPerShare, dividendPayDate<br> 216 * 90.01, 89.13, 90.01, 90.00, 90.34, 386147, "11/08/2017", 4.18, "10/27/2017" 217 * 218 * @param stockSymbol is the Symbol of the stock to lookup * 219 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare, dividendPayDate OR '' if not found 220 **/ 221 public String getStockDailyString(String stockSymbol) 222 { 223 String retVal = ""; 224 boolean success = false; 225 JsonObject dataHome = null; 226 if(holdings==null) 227 { 228 if(!"".equals(getUsername()) && !"".equals(getPassword())) 229 success = doLogin(); 230 else 231 System.out.println("ERROR reading username and password"); 232 233 if(success) 234 { 235 // scape the summary first to get a list of holdings 236 String result = doScrape(); 237 JsonObject jsO = toJsonObject(result); 238 JsonArray jsA = jsO.getJsonArray("Data"); 239 dataHome = jsA.getJsonObject(0); 240 holdings = dataHome.getJsonArray("Holdings"); 241 } 242 } 243 else 244 System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]"); 245 246 if(dataHome!=null && holdings!=null) 247 { 248 retVal+="\"Open\""; 249 retVal+=COL_DELIM; 250 retVal+="\"daysLow\""; 251 retVal+=COL_DELIM; 252 retVal+="\"daysHigh\""; 253 retVal+=COL_DELIM; 254 retVal+="\"close\""; 255 retVal+=COL_DELIM; 256 retVal+="\"previousClose\""; 257 retVal+=COL_DELIM; 258 retVal+="\"date\""; 259 retVal+=COL_DELIM; 260 retVal+="\"dividendPerShare\""; 261 retVal+=COL_DELIM; 262 retVal+="\"dividendPayDate\""; 263 retVal+="\n"; 264 265 setScrapeStart(pScrapeQuoteStart); 266 setScrapeEnd(pScrapeQuoteEnd); 267 String currSymbol = ""; 268 String currPrefix = ""; 269 JsonObject stock =null; 270 271 for(int i=0; i<holdings.size();i++) 272 { 273 stock = holdings.getJsonObject(i); 274 if(stock!=null) 275 { 276 currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1); 277 currPrefix = stock.getJsonString("CountryPrefix").toString().substring(1,stock.getJsonString("CountryPrefix").toString().length()-1); 278 279 if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol)) 280 { 281 282 //get the daily quote for each individual holding 283 setScrapePageUrl(pScrapeQuoteTokenizedUrl.replace(quoteSymbolToken,currPrefix+currSymbol)); 284 String result = doScrape(); 285 System.out.println(" DEBUG Stock Quote Result \n----------------------------\n"); 286 System.out.println(result); 287 288 /* 289 JsonObject jsO = toJsonObject(result); 290 JsonString currSymbol = stock.getJsonString("Symbol"); 291 if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol.toString())) 292 { 293 JsonNumber open, daysLow, daysHigh, previousClose, daysVolume, dividendPerShare; 294 String dividendPayDate; 295 //open=stock.getJsonString("Price"); 296 retVal+="\""+dataHome.getString("FriendlyName")+"\""; 297 retVal+=COL_DELIM; 298 retVal+=""+dataHome.getJsonString("Currency"); 299 retVal+=COL_DELIM; 300 301 } 302 */ 303 i=holdings.size(); 304 } 305 } 306 } 307 } 308 309 return retVal; 310 } 311 312 313 /** Create the Portfolio Summary data string for the CR account for today. 314 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 315 * FriendlyName, Currency, BookValue, TradeCash, MarketValue, EquityValue, UnrealizedGainLoss, UnrealizedGainLossPercent, numHoldings, date<br> 316 * "#2V9669V6 - TFSA", 50.5, 10.1, 66.4, 15.9, 8, "10/27/2017" 317 * 318 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 319 **/ 320 public String getPortfolioSummaryString() 321 { 322 String retVal = ""; 323 boolean success = false; 324 JsonObject dataHome = null; 325 if(holdings==null) 326 { 327 if(!"".equals(getUsername()) && !"".equals(getPassword())) 328 success = doLogin(); 329 else 330 System.out.println("ERROR reading username and password"); 331 332 if(success) 333 { 334 String result = doScrape(); 335 JsonObject jsO = toJsonObject(result); 336 JsonArray jsA = jsO.getJsonArray("Data"); 337 dataHome = jsA.getJsonObject(0); 338 //holdings = dataHome.getJsonArray("Holdings"); 339 } 340 } 341 else 342 System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]"); 343 344 if(dataHome!=null) // && holdings!=null) 345 { 346 retVal+="\"friendlyName\""; 347 retVal+=COL_DELIM; 348 retVal+="\"Currency\""; 349 retVal+=COL_DELIM; 350 retVal+="\"BookValue\""; 351 retVal+=COL_DELIM; 352 retVal+="\"TradeCash\""; 353 retVal+=COL_DELIM; 354 retVal+="\"MarketValue\""; 355 retVal+=COL_DELIM; 356 retVal+="\"EquityValue\""; 357 retVal+=COL_DELIM; 358 retVal+="\"UnrealizedGainLoss\""; 359 retVal+=COL_DELIM; 360 retVal+="\"UnrealizedGainLossPercent\""; 361 retVal+=COL_DELIM; 362 retVal+="\"NumberOfHoldings\""; 363 retVal+=COL_DELIM; 364 retVal+="\"date\""; 365 retVal+="\n"; 366 367 retVal+="\""+dataHome.getString("FriendlyName")+"\""; 368 retVal+=COL_DELIM; 369 retVal+=""+dataHome.getJsonString("Currency"); 370 retVal+=COL_DELIM; 371 retVal+=""+dataHome.getJsonNumber("BookValue"); 372 retVal+=COL_DELIM; 373 retVal+=""+dataHome.getJsonNumber("TradeCash"); 374 retVal+=COL_DELIM; 375 retVal+=""+dataHome.getJsonNumber("MarketValue"); 376 retVal+=COL_DELIM; 377 retVal+=""+dataHome.getJsonNumber("EquityValue"); 378 retVal+=COL_DELIM; 379 retVal+=""+dataHome.getJsonNumber("UnrealizedGainLoss"); 380 retVal+=COL_DELIM; 381 retVal+=""+dataHome.getJsonNumber("UnrealizedGainLossPercent"); 382 retVal+=COL_DELIM; 383 retVal+=""+(holdings!=null?holdings.size():"0"); 384 retVal+=COL_DELIM; 385 retVal+="\""+dateStr_+"\""; 386 387 /* 388 for(int i=0; i<holdings.size();i++) 389 { 390 JsonObject stock = holdings.getJsonObject(i); 391 if(stock!=null) 392 { 393 String currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1); 394 System.out.println(currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+ 395 stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+ 396 " = "+stock.getJsonNumber("MarketValue").toString()); 397 } 398 } 399 */ 400 } 401 402 return retVal; 403 } 404 405 406 407 /** Create the Portfolio data string for the CR account for today. 408 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 409 * stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br> 410 * Hive Tech, HIVE, HIVE, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, "10/27/2017" 411 * 412 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 413 **/ 414 public String getPortfolioString(){return getPortfolioString(true);} 415 416 417 418 /** Create the Portfolio data string for the CR account for today. 419 * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br> 420 * stockName, symbol, CRExchangeSymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br> 421 * Hive Tech, HIVE, V, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, 10-27-2017 422 * 423 * @param resultSetOnly true to return only the default delimited resultSet OR false to send a longer more readable string 424 * @return the delimited resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 425 **/ 426 public String getPortfolioString(boolean resultSetOnly) 427 { 428 String retVal = "stockName"+COL_DELIM+"symbol"+COL_DELIM+"CRQuerySymbol"+COL_DELIM+"numShares"+COL_DELIM+"currentPrice"+COL_DELIM+ 429 "currency"+COL_DELIM+"bookValue"+COL_DELIM+"marketValue"+COL_DELIM+"gain"+COL_DELIM+"gainPercent"+COL_DELIM+"currentDate\n"; 430 if(!resultSetOnly) retVal = "\n\n---------------------------------\n Portfolio Summary\n---------------------------------\n"; 431 432 //DecimalFormat dfp = new DecimalFormat( "##0" ); 433 DecimalFormat dfe = new DecimalFormat( "##0.000" ); 434 boolean success = false; 435 436 if(holdings==null) 437 { 438 if(!"".equals(getUsername()) && !"".equals(getPassword())) 439 success = doLogin(); 440 else 441 System.out.println("ERROR reading username and password"); 442 443 if(success) 444 { 445 String result = doScrape(); 446 JsonObject jsO = toJsonObject(result); 447 JsonArray jsA = jsO.getJsonArray("Data"); 448 dataHome = jsA.getJsonObject(0); 449 holdings = dataHome.getJsonArray("Holdings"); 450 } 451 } 452 else 453 System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]"); 454 455 if(dataHome!=null && holdings!=null) 456 { 457 if(!resultSetOnly) 458 { 459 String friendlyName = dataHome.getString("FriendlyName"); 460 461 retVal+=" FriendlyName : "+friendlyName; 462 retVal+="\n"; 463 retVal+=" MarketValue : "+dataHome.getJsonNumber("MarketValue"); 464 retVal+="\n"; 465 retVal+=" TradeCash : "+dataHome.getJsonNumber("TradeCash"); 466 retVal+="\n"; 467 retVal+=" BookValue : "+dataHome.getJsonNumber("BookValue"); 468 retVal+="\n"; 469 retVal+=" ------------------ ------------------------"; 470 retVal+="\n"; 471 retVal+=" UnrealizedGainLoss : "+dataHome.getJsonNumber("UnrealizedGainLoss"); 472 retVal+="\n"; 473 retVal+=" UnrealizedGain% : "+ 474 dfe.format(dataHome.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0); 475 retVal+="\n"; 476 retVal+="\n Holdings Summary\n - - - - - - - - - - - - -\n"; 477 } 478 479 String currSymbol = ""; 480 String priceStr = ""; 481 String currencySymbol = ""; 482 JsonObject stock = null; 483 for(int i=0; i<holdings.size();i++) 484 { 485 stock = holdings.getJsonObject(i); 486 if(stock!=null) 487 { 488 currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1); 489 if(!resultSetOnly) 490 { 491 retVal+=" "+currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+ 492 stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+ 493 " = "+stock.getJsonNumber("MarketValue").toString(); 494 retVal+="\n"; 495 } 496 else 497 { 498 if(right(stock.getJsonString("Price").toString() ,2).equals("U\"")) 499 { 500 priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-2); 501 currencySymbol = "US$"; 502 } 503 else 504 { 505 priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1); 506 currencySymbol = "C$"; 507 } 508 //stockName, symbol, CRExchangeSymbol, numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate 509 retVal+=stock.getString("SymbolDescription").toString()+COL_DELIM+ 510 currSymbol+COL_DELIM+ 511 stock.getString("Exchange").toString()+COL_DELIM+ 512 stock.getJsonNumber("Quantity").toString()+COL_DELIM+ 513 priceStr+COL_DELIM+ 514 currencySymbol+COL_DELIM+ 515 stock.getJsonNumber("BookValue").toString()+COL_DELIM+ 516 stock.getJsonNumber("MarketValue").toString()+COL_DELIM+ 517 stock.getJsonNumber("UnrealizedGainLoss").toString()+COL_DELIM+ 518 dfe.format(stock.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0)+COL_DELIM+ 519 dateStr_; 520 retVal+="\n"; 521 } 522 } 523 } 524 if(!resultSetOnly) retVal+="\n---------------------------\nHoldings Details:\n---------------------------\n"+ 525 prettyJson(holdings.toString()); 526 } 527 528 return retVal; 529 } 530 531 532 public String right(String value, int length) 533 { 534 // To get right characters from a string, change the begin index. 535 return value.substring(value.length() - length); 536 } 537 538 539 /** Test method to do whatever tests I want. **/ 540 @Override 541 protected void test(String[] args) 542 { 543 super.debugOut_=true; 544 debugOut_=true; 545 546 if(true) 547 { 548 String symbol = "HIVE"; 549 String result = getQuoteString(symbol,"CA"); 550 System.out.println("Test Quote ResultSet For HIVE\n"+result); 551 writeStringToFile(result,"yQuote-"+symbol+".txt"); 552 } 553 else if(false) 554 { 555 System.out.println("..."); 556 /*try 557 { 558 559 } 560 catch (Exception ex) 561 { 562 System.out.println("NO GO..."); 563 ex.printStackTrace(); 564 } 565 */ 566 } 567 } 568 569 570 public static void main(String[] args) throws Exception 571 { 572 MyPortfolioScraper instance = new MyPortfolioScraper(); 573 574 if(args.length>0 && args[0].equalsIgnoreCase("-t")) 575 instance.test(args); 576 else 577 { 578 boolean success = false; 579 if(!"".equals(instance.getUsername()) && !"".equals(instance.getPassword())) 580 success = instance.doLogin(); 581 else 582 System.out.println("ERROR reading username and password"); 583 584 if(success) 585 { 586 String result = instance.doScrape(); 587 System.out.println("Saving Scraped page results to file: "+"scrapedPageContent-"+instance.dateStr_+".json"); 588 writeStringToFile(prettyJson(result),"scrapedPageContent-"+instance.dateStr_+".json"); 589 590 System.out.println(instance.getPortfolioString()); 591 } 592 } 593 } 594}