001/*
002 *
003 *  $Revision: 569 $
004 *  $Date: 2012-11-05 16:44:26 -0800 (Mon, 05 Nov 2012) $
005 *  $Locker:  $
006 *
007 */
008package ca.bc.webarts.tools;
009
010import java.util.HashMap;
011import java.util.Properties;
012import javax.json.JsonObject;
013import javax.json.JsonArray;
014import javax.json.JsonNumber;
015import javax.json.JsonString;
016
017import ca.bc.webarts.widgets.ResultSetConverter;
018
019import java.text.DecimalFormat;
020
021
022  /**
023    * The Credential Direct webpage scraper. It is a class that extends the UrlScraper class to wrap/abstract the login and
024    * credentials away and just focusses on the scraping of data from the restricted pages.<br><br>
025    * <a href="https://trading.credentialdirect.com">https://trading.credentialdirect.com</a> is a site requiring login/authentication, so this class provides autoLogin and scraping of sub-pages.<br>
026    * The site login credentials are NOT kept inside the code for this class; they are in a properties file called '<i>.credential</i>' in the '<i>user.home</i>' directory.
027    * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too.<br>
028    * <b>copyright (c) 2017-2018 Tom B. Gutwin</b>
029    **/
030public class CredentialDirectScraper extends UrlScraper
031{
032  /**  A holder for this clients System File Separator.  */
033  public final static String SYSTEM_FILE_SEPERATOR = java.io.File.separator;
034
035  /**  A holder for this clients System line termination separator.  */
036  public final static String SYSTEM_LINE_SEPERATOR =
037                                           System.getProperty("line.separator");
038
039  public final static String COL_DELIM = SqlQuery.DEFAULT_COLUMN_DELIMITOR;
040
041  /**  The users home ditrectory.  */
042  public static String USERHOME = System.getProperty("user.home");
043
044  private String credentialPropertiesFilename_ = USERHOME+SYSTEM_FILE_SEPERATOR+".credential";
045
046  HashMap <String, String> reqProps = new HashMap<String, String>();
047  String crLoginUrl = "https://trading.credentialdirect.com/login.aspx";
048  String crScrapePageUrl = "https://trading.credentialdirect.com/AccountInquiry/Holdings?id=2V9669||2V9669||2V9669V6||TFSA";
049  String crQuotemediaAuthWebservice = "https://app.quotemedia.com/auth/g/authenticate/dataTool/v0/102590/962909937cd51685d7a18b6e01d3ccea95e88b5ddb9384d0f9a890ed4db261e1";
050  String quoteSymbolToken = "^%SYMBOL%^";
051  String datatoolAuthToken = "^%AUTHTOKEN%^";
052  String datatoolAuthHashToken = "";
053  String crScrapeQuoteTokenizedUrl = "https://trading.credentialdirect.com/Markets/Quotes/Holding?symbol="+quoteSymbolToken+"&symbolType=equity";
054  String crScrapeDatatoolTokenizedUrl = "https://app.quotemedia.com/datatool/getLevel2Quote.json?timezone=true&tradesummary=true&currencyInd=true&countryInd=true&symbol="+
055                                         quoteSymbolToken+
056                                         "&token="+
057                                         datatoolAuthToken; //422c0194c3cff9fb4b0c2e03b9094c7b9f096e02dfa05e3a6cf270b10220ef77&sid=343604d8-81c8-4b78-a520-f4d39629727d";
058  String crScrapeQuoteStart =   "<div class=\"pure-g\">";
059  String crScrapeQuoteEnd =   "</div><!--rivets: if data.datatype | = 'forex'-->";
060
061  String crLoginFormElement = "loginForm";  /* id of the form element */
062  String crUserLoginElement = "UserID";
063  String crPasswordElement = "Password";
064  String crUsername = ""; // load from outside somewhere
065  String crPassword = ""; // load from outside somewhere
066
067  private JsonArray holdings = null;
068  private JsonObject dataHome = null;
069
070
071  /**
072    * Default constructor for the Credential Direct webpage scraper.
073    * <a href="https://trading.credentialdirect.com">https://trading.credentialdirect.com</a> is a site requiring login; this class wraps the login and
074    * credentials away and just focusses on the scraping of data from the restricted pages.<br><br>
075    *
076    * The constructor ONLY gets the class fields and properties setup to go, BUT does no web page access;
077    * that is left up to the doLogin and doScrape methods.
078    **/
079  public CredentialDirectScraper( )
080  {
081    super();
082
083    loadProperties();
084
085    reqProps.put("Accept","text/html,application/xhtml+xml,application/xml");
086    reqProps.put("Accept-Encoding       ","gzip, deflate, br");
087    reqProps.put("Accept-Language       ","en-US,en;q=0.5");
088    reqProps.put("Connection","keep-alive");
089    reqProps.put("Content-Type","application/x-www-form-urlencoded");
090    reqProps.put("Upgrade-Insecure-Requests","1");
091    reqProps.put("Host","trading.credentialdirect.com");
092
093    setLoginUrl(crLoginUrl);
094    setLoginFormID(crLoginFormElement);
095    setUsernameFormElementName(crUserLoginElement);
096    setPasswordFormElementName(crPasswordElement);
097    setUsername(crUsername);
098    setPassword(crPassword);
099    setScrapePageUrl(crScrapePageUrl);
100    setScrapeStart("{\"Data\":");
101    setScrapeEnd("}});});</script>");
102    setRequestProps(reqProps);
103  }
104
105
106  /**
107    * Load the private properties from an external/private file.
108    * Site login credentials are NOT kept in this class; they are in a properties file called .credential in the 'user.home' directory.
109    * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too.
110    *
111    * @return truee if successful, false if could not load from file
112    **/
113  private boolean loadProperties()
114  {
115    boolean retVal = false;
116    try
117    {
118      //get user/pass from ~/.credential
119      Properties crProps = new Properties();
120      crProps.load(new java.io.FileReader(credentialPropertiesFilename_));
121      crUsername = crProps.getProperty("username");
122      crPassword = crProps.getProperty("password");
123      retVal = true;
124    }
125    catch (Exception ex)
126    {
127      // use defaults or set before use
128      System.out.println("ERROR : did not read props from :"+credentialPropertiesFilename_);
129    }
130
131    return retVal;
132  }
133
134
135  /**
136    * Sends the POST to the login url  parameters from the classVars.
137    * It also caches the page response text into thew classVar loginPageResponse_.<br>
138    */
139  @Override
140  public  boolean  doLogin()
141  {
142    boolean retVal = alreadyLoggedIn_;
143    if(!alreadyLoggedIn_)
144      if(!"".equals(getUsername()) && !"".equals(getPassword()))
145        retVal = super.doLogin();
146      else
147        System.out.println("ERROR reading username and password");
148
149    return retVal;
150  }
151
152
153  private String authenticateQuoteMedia() throws Exception
154  {
155    String retVal = "";
156
157    // login to CR
158    doLogin();
159
160    // POST to the  authenticate web-service
161    HashMap <String, String> reqProps = null;
162
163    // if using the datatool api url, then a different set of requestProps are needed
164    reqProps = new HashMap<String, String>();
165    reqProps.put("Accept","*/*");
166    reqProps.put("Accept-Encoding       ","zip, deflate, br");
167    reqProps.put("Accept-Language       ","en-US,en;q=0.5");
168    reqProps.put("Connection","keep-alive");
169    reqProps.put("Host","app.quotemedia.com");
170    reqProps.put("Origin","https://trading.credentialdirect.com");
171    reqProps.put("Referer","https://trading.credentialdirect.com/AccountInquiry/Summary");
172
173    //System.out.println("Quotemedia webService call:\n");
174
175    int responseCode = sendPost(crQuotemediaAuthWebservice, "", reqProps);
176    System.out.println("Quotemedia RESPONSE CODE: "+responseCode);
177
178    //get back the JSON authToken
179    JsonObject jsO = toJsonObject(postPageResponse_);
180    System.out.println("QuoteMedia authentication JSON response:\n"+jsO);
181
182    return retVal;
183  }
184
185
186  /** Create the stock QUOTE data string for the given stock symbol for today.
187    * this data gets used as the data load into the InvestmentTrackerQuery database.<br>
188    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
189    *  open,  daysLow,  daysHigh, previousClose,  daysVolume, dividendPerShare,   dividendPayDate<br>
190    *  90.01, 89.13, 90.01, 90.34, 386147, 4.18, "10/27/2017"
191    *
192    * @param stockSymbol is the Symbol of the stock to lookup
193    * @param country is the stock market country (US, CA etc)
194    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited open,  daysLow,  daysHigh, previousClose,  daysVolume, dividendPerShare,   dividendPayDate OR '' if not found
195    **/
196  private String getQuoteString(String stockSymbol, String country)
197  {
198    //"https://trading.credentialdirect.com/Markets/Quotes/Holding?symbol=US:AMD&symbolType=equity"
199    //"https://trading.credentialdirect.com/Markets/Quotes/Holding?symbol=HIVE&symbolType=equity"
200    // start <div class="pure-g">
201    // end   </div><!--rivets: if data.datatype | = 'forex'-->
202
203    /*
204    https://app.quotemedia.com/datatool/getLevel2Quote.json?timezone=true&tradesummary=true&currencyInd=true&countryInd=true&symbol=HIVE%3ACA&token=422c0194c3cff9fb4b0c2e03b9094c7b9f096e02dfa05e3a6cf270b10220ef77&sid=343604d8-81c8-4b78-a520-f4d39629727d
205    Accept-Encoding     g  zip, deflate, br
206    Accept-Language     en
207    Connection  keep-alive
208    Host        app.quotemedia.com
209    Origin      https://trading.credentialdirect.com
210    Referer     https://trading.credentialdirect.com/Markets/Quotes/Holding?symbol=BLDP&symbolType=equity
211    User-Agent  Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/57.0
212    */
213    String retVal = "";
214    boolean success = false;
215    boolean useDataTool = true;
216
217    String currPrefix = (country.equalsIgnoreCase("us")?"US:":"");
218    String currSuffix = (country.equalsIgnoreCase("us")?"%3AUS":"%3ACA");
219    String tokenReplacedScrapeUrl = crScrapeQuoteTokenizedUrl.replace(quoteSymbolToken,currPrefix+stockSymbol);
220    tokenReplacedScrapeUrl = tokenReplacedScrapeUrl.replace(datatoolAuthToken,datatoolAuthHashToken);
221    setScrapePageUrl(crScrapeQuoteTokenizedUrl.replace(quoteSymbolToken,currPrefix+stockSymbol));
222
223    HashMap <String, String> reqProps = null;
224
225    if(useDataTool)
226    {
227      setScrapePageUrl(crScrapeDatatoolTokenizedUrl.replace(quoteSymbolToken,stockSymbol+currSuffix));
228      // if using the datatool api url, then a different set of requestProps are needed
229      reqProps = new HashMap<String, String>();
230      reqProps.put("Accept","*/*");
231      reqProps.put("Accept-Encoding     ","zip, deflate, br");
232      reqProps.put("Accept-Language     ","en");
233      reqProps.put("Connection","keep-alive");
234      reqProps.put("Origin","https://trading.credentialdirect.com");
235      reqProps.put("Referer",getScrapePageUrl());
236      reqProps.put("Host","app.quotemedia.com");
237    }
238
239    if(holdings==null)
240    {
241      if(!"".equals(getUsername()) && !"".equals(getPassword()))
242        success = doLogin();
243      else
244        System.out.println("ERROR reading username and password");
245
246      if(success)
247      {
248        String result = doScrape(getScrapePageUrl(), reqProps); // without the trimming
249        retVal = result;
250      }
251    }
252    return retVal;
253  }
254
255
256
257  /** Create the STOCK_DAILY data string  for the given stock symbol for today.
258    * this data gets used as the data load into the InvestmentTrackerQuery database.<br>
259    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
260    *  open,  daysLow,  daysHigh, close, previousClose,  daysVolume, date, dividendPerShare,   dividendPayDate<br>
261    *  90.01, 89.13, 90.01, 90.00, 90.34, 386147, "11/08/2017", 4.18, "10/27/2017"
262    *
263    * @param stockSymbol is the Symbol of the stock to lookup    *
264    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited open,  daysLow,  daysHigh, previousClose,  daysVolume, dividendPerShare,   dividendPayDate OR '' if not found
265    **/
266  public String getStockDailyString(String stockSymbol)
267  {
268    String retVal = "";
269    boolean success = false;
270    JsonObject dataHome = null;
271    if(holdings==null)
272    {
273      if(!"".equals(getUsername()) && !"".equals(getPassword()))
274        success = doLogin();
275      else
276        System.out.println("ERROR reading username and password");
277
278      if(success)
279      {
280        // scape the summary first to get a list of holdings
281        String result = doScrape();
282        JsonObject jsO = toJsonObject(result);
283        JsonArray jsA = jsO.getJsonArray("Data");
284        dataHome = jsA.getJsonObject(0);
285        holdings = dataHome.getJsonArray("Holdings");
286      }
287    }
288    else
289      System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]");
290
291    if(dataHome!=null && holdings!=null)
292    {
293      retVal+="\"Open\"";
294      retVal+=COL_DELIM;
295      retVal+="\"daysLow\"";
296      retVal+=COL_DELIM;
297      retVal+="\"daysHigh\"";
298      retVal+=COL_DELIM;
299      retVal+="\"close\"";
300      retVal+=COL_DELIM;
301      retVal+="\"previousClose\"";
302      retVal+=COL_DELIM;
303      retVal+="\"date\"";
304      retVal+=COL_DELIM;
305      retVal+="\"dividendPerShare\"";
306      retVal+=COL_DELIM;
307      retVal+="\"dividendPayDate\"";
308      retVal+="\n";
309
310      setScrapeStart(crScrapeQuoteStart);
311      setScrapeEnd(crScrapeQuoteEnd);
312      String currSymbol = "";
313      String currPrefix = "";
314      JsonObject stock =null;
315
316      for(int i=0; i<holdings.size();i++)
317      {
318        stock = holdings.getJsonObject(i);
319        if(stock!=null)
320        {
321          currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1);
322          currPrefix = stock.getJsonString("CountryPrefix").toString().substring(1,stock.getJsonString("CountryPrefix").toString().length()-1);
323
324          if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol.toString()))
325          {
326
327            //get the daily quote for each individual holding
328            setScrapePageUrl(crScrapeQuoteTokenizedUrl.replace(quoteSymbolToken,currPrefix+currSymbol));
329            String result = doScrape();
330            System.out.println(" DEBUG Stock Quote Result \n----------------------------\n");
331            System.out.println(result);
332
333            /*
334            JsonObject jsO = toJsonObject(result);
335            JsonString currSymbol = stock.getJsonString("Symbol");
336            if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol.toString()))
337            {
338              JsonNumber  open,  daysLow,  daysHigh, previousClose,  daysVolume, dividendPerShare;
339              String   dividendPayDate;
340              //open=stock.getJsonString("Price");
341              retVal+="\""+dataHome.getString("FriendlyName")+"\"";
342              retVal+=COL_DELIM;
343              retVal+=""+dataHome.getJsonString("Currency");
344              retVal+=COL_DELIM;
345
346            }
347            */
348            i=holdings.size();
349          }
350        }
351      }
352    }
353
354    return retVal;
355  }
356
357
358  /** Create the Portfolio Summary data string for the CR account for today.
359    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
360    *  FriendlyName, Currency, BookValue, TradeCash,  MarketValue, EquityValue, UnrealizedGainLoss, UnrealizedGainLossPercent, numHoldings, date<br>
361    *  "#2V9669V6 - TFSA", 50.5, 10.1, 66.4, 15.9, 8, "10/27/2017"
362    *
363    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate
364    **/
365  public String getPortfolioSummaryString()
366  {
367    String retVal = "";
368    boolean success = false;
369    JsonObject dataHome = null;
370    if(holdings==null)
371    {
372      if(!"".equals(getUsername()) && !"".equals(getPassword()))
373        success = doLogin();
374      else
375        System.out.println("ERROR reading username and password");
376
377      if(success)
378      {
379        String result = doScrape();
380        JsonObject jsO = toJsonObject(result);
381        JsonArray jsA = jsO.getJsonArray("Data");
382        dataHome = jsA.getJsonObject(0);
383        //holdings = dataHome.getJsonArray("Holdings");
384      }
385    }
386    else
387      System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]");
388
389    if(dataHome!=null) // && holdings!=null)
390    {
391      retVal+="\"friendlyName\"";
392      retVal+=COL_DELIM;
393      retVal+="\"Currency\"";
394      retVal+=COL_DELIM;
395      retVal+="\"BookValue\"";
396      retVal+=COL_DELIM;
397      retVal+="\"TradeCash\"";
398      retVal+=COL_DELIM;
399      retVal+="\"MarketValue\"";
400      retVal+=COL_DELIM;
401      retVal+="\"EquityValue\"";
402      retVal+=COL_DELIM;
403      retVal+="\"UnrealizedGainLoss\"";
404      retVal+=COL_DELIM;
405      retVal+="\"UnrealizedGainLossPercent\"";
406      retVal+=COL_DELIM;
407      retVal+="\"NumberOfHoldings\"";
408      retVal+=COL_DELIM;
409      retVal+="\"date\"";
410      retVal+="\n";
411
412      retVal+="\""+dataHome.getString("FriendlyName")+"\"";
413      retVal+=COL_DELIM;
414      retVal+=""+dataHome.getJsonString("Currency");
415      retVal+=COL_DELIM;
416      retVal+=""+dataHome.getJsonNumber("BookValue");
417      retVal+=COL_DELIM;
418      retVal+=""+dataHome.getJsonNumber("TradeCash");
419      retVal+=COL_DELIM;
420      retVal+=""+dataHome.getJsonNumber("MarketValue");
421      retVal+=COL_DELIM;
422      retVal+=""+dataHome.getJsonNumber("EquityValue");
423      retVal+=COL_DELIM;
424      retVal+=""+dataHome.getJsonNumber("UnrealizedGainLoss");
425      retVal+=COL_DELIM;
426      retVal+=""+dataHome.getJsonNumber("UnrealizedGainLossPercent");
427      retVal+=COL_DELIM;
428      retVal+=""+(holdings!=null?holdings.size():"0");
429      retVal+=COL_DELIM;
430      retVal+="\""+dateStr_+"\"";
431
432      /*
433      for(int i=0; i<holdings.size();i++)
434      {
435        JsonObject stock = holdings.getJsonObject(i);
436        if(stock!=null)
437        {
438           String currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1);
439            System.out.println(currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+
440                               stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+
441                               " = "+stock.getJsonNumber("MarketValue").toString());
442        }
443      }
444      */
445    }
446
447    return retVal;
448  }
449
450
451
452  /** Create the Portfolio data string for the CR account for today.
453    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
454    *  stockName, symbol, CRQuerySymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br>
455    *  Hive Tech, HIVE, HIVE, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, "10/27/2017"
456    *
457    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate
458    **/
459  public String getPortfolioString(){return getPortfolioString(true);}
460
461
462
463  /** Create the Portfolio data string for the CR account for today.
464    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
465    *  stockName, symbol, CRExchangeSymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br>
466    *  Hive Tech, HIVE, V, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, 10-27-2017
467    *
468    * @param resultSetOnly true to return only the default delimited resultSet OR false to send a longer more readable string
469    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate
470    **/
471  public String getPortfolioString(boolean resultSetOnly)
472  {
473    String retVal = "stockName"+COL_DELIM+"symbol"+COL_DELIM+"CRQuerySymbol"+COL_DELIM+"numShares"+COL_DELIM+"currentPrice"+COL_DELIM+
474                    "currency"+COL_DELIM+"bookValue"+COL_DELIM+"marketValue"+COL_DELIM+"gain"+COL_DELIM+"gainPercent"+COL_DELIM+"currentDate\n";
475    if(!resultSetOnly) retVal = "\n\n---------------------------------\n    Portfolio Summary\n---------------------------------\n";
476
477    DecimalFormat dfp = new DecimalFormat( "##0" );
478    DecimalFormat dfe = new DecimalFormat( "##0.000" );
479    boolean success = false;
480
481    if(holdings==null)
482    {
483      if(!"".equals(getUsername()) && !"".equals(getPassword()))
484        success = doLogin();
485      else
486        System.out.println("ERROR reading username and password");
487
488      if(success)
489      {
490        String result = doScrape();
491        JsonObject jsO = toJsonObject(result);
492        JsonArray jsA = jsO.getJsonArray("Data");
493        dataHome = jsA.getJsonObject(0);
494        holdings = dataHome.getJsonArray("Holdings");
495      }
496    }
497    else
498      System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]");
499
500    if(dataHome!=null && holdings!=null)
501    {
502      if(!resultSetOnly)
503      {
504        String friendlyName = dataHome.getString("FriendlyName");
505
506        retVal+="    FriendlyName       : "+friendlyName;
507        retVal+="\n";
508        retVal+="    MarketValue        : "+dataHome.getJsonNumber("MarketValue");
509        retVal+="\n";
510        retVal+="    TradeCash          : "+dataHome.getJsonNumber("TradeCash");
511        retVal+="\n";
512        retVal+="    BookValue          : "+dataHome.getJsonNumber("BookValue");
513        retVal+="\n";
514        retVal+="    ------------------   ------------------------";
515        retVal+="\n";
516        retVal+="    UnrealizedGainLoss : "+dataHome.getJsonNumber("UnrealizedGainLoss");
517        retVal+="\n";
518        retVal+="    UnrealizedGain%    : "+
519                dfe.format(dataHome.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0);
520        retVal+="\n";
521        retVal+="\n Holdings Summary\n - - - - - - - - - - - - -\n";
522      }
523
524      String currSymbol = "";
525      String priceStr = "";
526      String currencySymbol = "";
527      JsonObject stock = null;
528      for(int i=0; i<holdings.size();i++)
529      {
530        stock = holdings.getJsonObject(i);
531        if(stock!=null)
532        {
533          currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1);
534          if(!resultSetOnly)
535          {
536            retVal+="   "+currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+
537                  stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+
538                  " = "+stock.getJsonNumber("MarketValue").toString();
539            retVal+="\n";
540          }
541          else
542          {
543            if(right(stock.getJsonString("Price").toString() ,2).equals("U\""))
544            {
545              priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-2);
546              currencySymbol = "US$";
547            }
548            else
549            {
550              priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1);
551              currencySymbol = "C$";
552            }
553            //stockName, symbol, CRExchangeSymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate
554            retVal+=stock.getString("SymbolDescription").toString()+COL_DELIM+
555                    currSymbol+COL_DELIM+
556                    stock.getString("Exchange").toString()+COL_DELIM+
557                    stock.getJsonNumber("Quantity").toString()+COL_DELIM+
558                    priceStr+COL_DELIM+
559                    currencySymbol+COL_DELIM+
560                    stock.getJsonNumber("BookValue").toString()+COL_DELIM+
561                    stock.getJsonNumber("MarketValue").toString()+COL_DELIM+
562                    stock.getJsonNumber("UnrealizedGainLoss").toString()+COL_DELIM+
563                    dfe.format(stock.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0)+COL_DELIM+
564                    dateStr_;
565            retVal+="\n";
566          }
567        }
568      }
569      if(!resultSetOnly) retVal+="\n---------------------------\nHoldings Details:\n---------------------------\n"+
570                                  prettyJson(holdings.toString());
571    }
572
573    return retVal;
574  }
575
576
577  public  String right(String value, int length)
578  {
579    // To get right characters from a string, change the begin index.
580    return value.substring(value.length() - length);
581  }
582
583
584  /** Test method to do whatever tests I want. **/
585  @Override
586  protected void test(String[] args)
587  {
588    if(false)
589    {
590      String symbol = "HIVE";
591      String result = getQuoteString(symbol,"CA");
592      System.out.println("Test Quote ResultSet For HIVE\n"+result);
593      writeStringToFile(result,"crQuote-"+symbol+".txt");
594    }
595    else if(true)
596    {
597      System.out.println("Testing the QuoteMedia authenication web-service...");
598      try
599      {
600        authenticateQuoteMedia();
601      }
602      catch (Exception ex)
603      {
604        System.out.println("NO GO...");
605        ex.printStackTrace();
606      }
607    }
608  }
609
610
611  public static void main(String[] args)
612  {
613    CredentialDirectScraper instance = new CredentialDirectScraper();
614
615    if(args.length>0 && args[0].toLowerCase().equals("-t"))
616      instance.test(args);
617    else
618    {
619      boolean success = false;
620      if(!"".equals(instance.getUsername()) && !"".equals(instance.getPassword()))
621        success = instance.doLogin();
622      else
623        System.out.println("ERROR reading username and password");
624
625      if(success)
626      {
627        String result = instance.doScrape();
628        System.out.println("Saving Scraped page results to file: "+"crPageContent-"+instance.dateStr_+".json");
629        writeStringToFile(prettyJson(result),"crPageContent-"+instance.dateStr_+".json");
630
631        System.out.println(instance.getPortfolioString());
632      }
633    }
634  }
635}