/*
 *  $URL: svn://svn.webarts.bc.ca/open/trunk/projects/WebARTS/ca/bc/webarts/tools/GoogleFinanceScraper.java $
 *  $Author: tgutwin $
 *  $Revision: 1255 $
 *  $Date: 2018-03-17 20:34:04 -0700 (Sat, 17 Mar 2018) $
 */
package ca.bc.webarts.tools;

import java.util.HashMap;
import java.util.Properties;
import javax.json.JsonObject;
import javax.json.JsonArray;
import javax.json.JsonNumber;
import javax.json.JsonString;

import ca.bc.webarts.widgets.ResultSetConverter;

import java.text.DecimalFormat;


  /**
    * The Google finance webpage scraper. It is a class that extends the UrlScraper class to wrap/abstract the login and
    * credentials away and just focusses on the scraping of data from the restricted pages.<br><br>
    * <a href="https://trading.credentialdirect.com">https://trading.credentialdirect.com</a> is a site requiring login/authentication, so this class provides autoLogin and scraping of sub-pages.<br>
    * The site login credentials are NOT kept inside the code for this class; they are in a properties file called '<i>.credential</i>' in the '<i>user.home</i>' directory.
    * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too.<br>
    * <b>copyright (c) 2017-2018 Tom B. Gutwin</b>
    **/
public class GoogleFinanceScraper extends UrlScraper
{
  /**  A holder for this clients System File Separator.  */
  public final static String SYSTEM_FILE_SEPERATOR = java.io.File.separator;

  /**  A holder for this clients System line termination separator.  */
  public final static String SYSTEM_LINE_SEPERATOR =
                                           System.getProperty("line.separator");

  public final static String COL_DELIM = SqlQuery.DEFAULT_COLUMN_DELIMITOR;
  public static final String QUOTE_SYMBOL_TOKEN = "^%SYMBOL%^";
  public static final String QUOTE_MARKET_TOKEN = "^%MARKET%^";

  /**  The users home ditrectory.  */
  public static String USERHOME = System.getProperty("user.home");

  private String googleFinanceFilename_ = USERHOME+SYSTEM_FILE_SEPERATOR+".googleFinance";

  HashMap <String, String> reqProps = new HashMap<String, String>();

  //https://login.google.com/?.src=fpctx&.intl=ca&.lang=en-CA&authMechanism=primary&yid=&done=https%3A%2F%2Fca.google.com%2F&eid=100&as=1&login=tgutwin&crumb=9D8I.8WDkTW
  String gLoginUrl_plong = "https://login.google.com/account/challenge/password?.src=fpctx&.intl=ca&.lang=en-CA&authMechanism=primary&yid=tgutwin&done=https%3A%2F%2Ffinance.google.com%2F&as=1&login=tgutwin&crumb=1EAn939fa%2Ft&display=login&s=QQ--&sessionIndex=QQ--&acrumb=FBRp8xFN";
  String gLoginUrl_pshort = "https://login.google.com/?.intl=ca&.lang=en-CA&login=tgutwin&.src=fpctx&done=https%3A%2F%2Ffinance.google.com%2F&prefill=0";  //"https://login.google.com/account/challenge/password?.src=fpctx&.intl=ca&.lang=en-CA&authMechanism=primary&yid=tgutwin&done=https%3A%2F%2Ffinance.google.com%2F&as=1&login=tgutwin";
  String gLoginUrl = "https://login.google.com/?display=login&login=tgutwin&done=https%3A%2F%2Ffinance.google.com%2F&prefill=1";  //https://login.google.com/?authMechanism=primary&yid=&done=https%3A%2F%2Ffinance.google.com%2F&eid=100&as=1&login=tgutwin&crumb=9D8I.8WDkTW
  String gScrapePageUrl = "https://finance.google.ca/finance?q=NASDAQ:ITRI&ei=KZITWpDrINihjAHU9JWIAg";
  static String gScrapeQuoteTokenizedUrl = "https://finance.google.ca/finance?q="+QUOTE_MARKET_TOKEN+":"+QUOTE_SYMBOL_TOKEN;

  String gLoginFormElement = "pure-form pure-form-stacked";  /* id of the form element */
  String gUserLoginElement = "displayName";
  String gPasswordElement = "login-passwd";
  String gUsername = ""; // load from outside somewhere
  String gPassword = ""; // load from outside somewhere

  String gScrapeQuoteStart= "<div id=market-data-div";
  String gScrapeQuoteStart_= "<table><tr><td class=\"JgXcPd\">";
  String gScrapeQuoteEnd= "<script>google.finance.renderMarketData();</script>";
  String gScrapeQuoteEnd_= "</td></tr></table></div></div></div></g-card-section></div></div></div>";

  private JsonArray holdings = null;
  private JsonObject dataHome = null;

  protected String quoteStringCache_ = "";

  /**
    * Default constructor for the Yahoo! Finance webpage scraper.
    * <a href="https://finance.google.com">https://finance.google.com</a> is a site requiring login; this class wraps the login and
    * credentials away and just focusses on the scraping of data from the restricted pages.<br><br>
    *
    * The constructor ONLY gets the class fields and properties setup to go, BUT does no web page access;
    * that is left up to the doLogin and   methods.
    **/
  public GoogleFinanceScraper( )
  {
    super();

    alreadyLoggedIn_ = true;
    loadProperties();

    reqProps.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
    reqProps.put("Accept-Encoding       ","gzip, deflate, br");
    reqProps.put("Accept-Language       ","en-US,en;q=0.5");
    reqProps.put("Connection","keep-alive");
    reqProps.put("Cookie","_ga=GA1.2-2.2126383985.1485192339; SC=RV=:ED=ca; _ga=GA1.3-2.2126383985.1485192339; _gid=GA1.3-2.1169777638.1511223678; NID=117=XCFBclyBN8yMrHodqAHGAnzZqTHHgdgIC7ZMLZEstpmmhvehgkI0QeHLVkvWLG0uySKvItWNuKphGmBAJe4QzCD6MBGfgLclSeW27xySZ5L_M-iSNTfeVfQHYCkl0tqP15yuhEvq7f24oyMTTyVT_QN9B3y9o8EN1Anhb9JZ; OGPC=5061451-2:5061821-3:; 1P_JAR=2017-11-21-2; OGP=-5061451:-5061821:; __gads=ID=71d1c22d471495fa:T=1508459738:S=ALNI_MbCde9o7niI4fDhlWp2dJCbeAAdNQ; S=quotestreamer=PY9m_pUIUhUQlj-rEAMVepsfJzJoAXRV");
    reqProps.put("Upgrade-Insecure-Requests","1");
    reqProps.put("Host","finance.google.ca");

    setLoginUrl(gLoginUrl_pshort);
    setLoginFormID(gLoginFormElement);
    setUsernameFormElementName(gUserLoginElement);
    setPasswordFormElementName(gPasswordElement);
    setUsername(gUsername);
    setPassword(gPassword);
    setScrapePageUrl(gScrapePageUrl);
    setScrapeStart(gScrapeQuoteStart);
    setScrapeEnd(gScrapeQuoteEnd);
    setRequestProps(reqProps);
  }


  /**
    * Load the private properties from an external/private file.
    * Site login credentials are NOT kept in this class; they are in a properties file called .credential in the 'user.home' directory.
    * <br>The properites that should be in this file:<ul><li>username</li><li>password</li></ul>Maybe the request properties for the website could go in here too.
    *
    * @return truee if successful, false if could not load from file
    **/
  private boolean loadProperties()
  {
    boolean retVal = false;
    try
    {
      //get user/pass from ~/.credential
      Properties gProps = new Properties();
      gProps.load(new java.io.FileReader(googleFinanceFilename_));
      gUsername = gProps.getProperty("username");
      gPassword = gProps.getProperty("password");
      retVal = true;
    }
    catch (Exception ex)
    {
      // use defaults or set before use
      System.out.println("ERROR : did not read props from :"+googleFinanceFilename_);
    }

    return retVal;
  }


  /**
    * Not needed because this is a open webpage.
    **/
  @Override
  public  boolean  doLogin()
  {
    boolean retVal = alreadyLoggedIn_;
    reqProps.put("Content-Type","application/x-www-form-urlencoded");

    if(!alreadyLoggedIn_)
      if(!"".equals(getUsername()) && !"".equals(getPassword()))
        retVal = super.doLogin();
      else
        System.out.println("ERROR reading username and password");

    return retVal;
  }


  /** Get the latest stock QUOTE using old website.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the quote OR 0.0 if not found
    **/
  private double getQuoteOld(String stockSymbol, String marketSymbol)
  {

    double retVal = 0.0;
    String currPriceStr = null;
        String result = getQuoteString(stockSymbol, marketSymbol);
        writeStringToFile(result,"gQuote-"+stockSymbol+".txt");
        //Looking for
        // <span class="pr">
        // <span id="ref_630737340410842_l">
        //  3.72</span>
        // </span>
    try
    {
      int offset = "class=\"pr\">".length();
      int s= result.indexOf("class=\"pr\">");
      if (s==-1)
      {
        s = result.indexOf("class=pr>");
        currPriceStr = result.substring( s);
        currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>')+1); // should be from 3.72</span
        currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('<'));
        retVal = Double.parseDouble(currPriceStr);
      }
      else
      {
        currPriceStr = result.substring( s);
        currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>', offset)+1); // should be from 3.72</span
        currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('<'));
        retVal = Double.parseDouble(currPriceStr);
      }
    }
    catch(java.lang.StringIndexOutOfBoundsException oobEx)
    {
      System.out.println("ERROR: get quote parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt");
      oobEx.printStackTrace();
      System.out.println(currPriceStr);
    }
    return retVal;
  }


  /** Get the latest stock QUOTE from new Website Circa 2018.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the quote OR 0.0 if not found
    **/
  private double getQuoteNew(String stockSymbol, String marketSymbol)
  {

    double retVal = 0.0;
    String currPriceStr = null;
        String result = getQuoteString(stockSymbol, marketSymbol);
        writeStringToFile(result,"gQuote-"+stockSymbol+".txt");
    try
    {

        String str = "<div><span><span jsl=\"$t t-cLlm2e1WimE;$x 0;\"";
        String str2 = "<span class=\"IsqQVc NprOob inM7_hGITiio-zJFzKq8ukm8\">";  // needed if there is a 2nd option
        String str3 = "><span class=\"";
        int offset = str.length();
        int s= result.indexOf(str);
        if (s==-1)
        {
          offset = str2.length();
          s = result.indexOf(str2);
          if (s!=-1)
          {
            currPriceStr = result.substring( s+offset);
            int offset2 = str3.length();
            int s2= currPriceStr.indexOf(str3);
            currPriceStr = currPriceStr.substring( s2+offset2);
            int s3= currPriceStr.indexOf("\">");
            currPriceStr = currPriceStr.substring( s3+2);
            currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</span>"));
            retVal = Double.parseDouble(currPriceStr.replace(",",""));
          }
        }
        else
        {
          currPriceStr = result.substring( s+offset);
          int offset2 = str3.length();
          int s2= currPriceStr.indexOf(str3);
          currPriceStr = currPriceStr.substring( s2+offset2);
          int s3= currPriceStr.indexOf("\">");
          currPriceStr = currPriceStr.substring( s3+2);
          currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</span>"));
          retVal = Double.parseDouble(currPriceStr.replace(",",""));
        }
    }
    catch(java.lang.StringIndexOutOfBoundsException oobEx)
    {
      System.out.println("ERROR: get quote parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt");
      oobEx.printStackTrace();
      System.out.println(currPriceStr);
    }
    return retVal;
  }


    /** Get the latest stock QUOTE .
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the quote OR 0.0 if not found
    **/
  public double getQuote(String stockSymbol, String marketSymbol)
  {
    return getQuoteNew(stockSymbol, marketSymbol);
  }


  public String getTodaysTSXChartAndTableHtmlStr()
  {
    return  getTodaysTSXChartAndTableHtmlStr(true);
  }


  /** Get the days TSX Index chart and data .
    *
    * @return html containing todays TSX indices chart
    **/
  public String getTodaysTSXChartAndTableHtmlStr(boolean includeTable)
  {
    String retVal = "";
    boolean success = false;
    boolean doLogin = false;
    setScrapePageUrl("https://finance.google.ca/finance");
    setScrapeStart("<div class=id-summary-chart>");
    setScrapeEnd("</div></div><div class=\"sfe-section clf\">");
    //setScrapeStart("BlahBlahBlah");
    //setScrapeEnd("BlahBlahBlah");

    HashMap <String, String> reqProps = null;

    setDebugOut();

    if(doLogin)
    {
      if(!"".equals(getUsername()) && !"".equals(getPassword()))
      {
        success = doLogin();
        //System.out.println("\nLogin Response:\n"+postPageResponse_);
      }
      else
        System.out.println("ERROR reading username and password");
    }
    else
      success = true;

    if(success)
    {
      String result =  doScrape(false);
      retVal = result;

      //System.out.println("Scraped Chart:\n"+retVal);
    }

    /*
      //Looking for
      // <span class="pr">
      // <span id="ref_630737340410842_l">
      //  3.72</span>
      // </span>
      try
      {
        int offset = "Vol / Avg.</td>".length();
        int s= result.indexOf("Vol / Avg.</td>");
        if (s==-1)
        {
          s = result.indexOf("class=pr>");
          if (s!=-1)
          {
            currPriceStr = result.substring( s);
            currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>')+1); // should be from 3.72</span
            if(currPriceStr.indexOf('M')!=-1 && currPriceStr.indexOf('M')<8)
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('M'));
              retVal = Double.parseDouble(currPriceStr.replace(",",""))*1000000.0;
            }
            else
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('/'));
              retVal = Double.parseDouble(currPriceStr.replace(",",""));
            }
          }
        }
        else
        {
          currPriceStr = result.substring( s);
          //System.out.println("\nDEBUG:"+currPriceStr);
          currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>', offset)+1); // should be from 3.72</span
          //System.out.println("\nDEBUG:"+currPriceStr);
            if(currPriceStr.indexOf('M')!=-1 && currPriceStr.indexOf('M')<8)
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('M'));
              retVal = Double.parseDouble(currPriceStr.replace(",",""))*1000000.0;
            }
            else
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('/'));
              retVal = Double.parseDouble(currPriceStr.replace(",",""));
            }
        }
      }
      catch(java.lang.StringIndexOutOfBoundsException oobEx)
      {
        System.out.println("ERROR: get volume parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt");
        oobEx.printStackTrace();
        System.out.println(currPriceStr);
      }
    */
    return retVal;
  }


    /** Get the latest stock stat for the passed in valueName from the NEW GFinance pages Circa 2018.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @param valueName is the stat to scrape - Open, Close, High, Low, 52-wk high, 52-wk low, Div yield
    * @return the price OR -1 if not found;
    **/
  private double scrapeValueNew(String stockSymbol, String marketSymbol, String valueName)
  {
    double retVal = -1.0;
    String currPriceStr = null;
    if ("Close".equalsIgnoreCase(valueName)) valueName = "Prev close";
    if ("Prev close".equalsIgnoreCase(valueName)) valueName = "Prev close";
    if ("Div".equalsIgnoreCase(valueName)) valueName = "Div yield";
    if ("Dividend".equalsIgnoreCase(valueName)) valueName = "Div yield";
    if ("Div yield".equalsIgnoreCase(valueName)) valueName = "Div yield";
    if(!"".equalsIgnoreCase(marketSymbol))
    {
      retVal = 0.0;
      String result = getQuoteString(stockSymbol, marketSymbol);
      writeStringToFile(result,"gQuote-"+stockSymbol+".txt");
      //Looking for
      // <div class="ZSM8k"><table><tr>
      //   <td class="JgXcPd">Open</td><td class="iyjjgb">21.50</td></tr>
      //   <tr><td class="JgXcPd">High</td><td class="iyjjgb">21.93</td></tr>
      //   <tr><td class="JgXcPd">Low</td><td class="iyjjgb">20.65</td></tr>
      //   <tr><td class="JgXcPd">Mkt cap</td><td class="iyjjgb">567.04M</td></tr>
      //   <tr><td class="JgXcPd">P/E ratio</td><td class="iyjjgb">10.42</td></tr>
      //   </table></div>
      //   <div class="ZSM8k"><table><tr>
      //     <td class="JgXcPd">Div yield</td><td class="iyjjgb">1.94%</td></tr>
      //     <tr><td class="JgXcPd">Prev close</td><td class="iyjjgb">20.87</td></tr>
      //     <tr><td class="JgXcPd">52-wk high</td><td class="iyjjgb">26.11</td></tr>
      //     <tr><td class="JgXcPd">52-wk low</td><td class="iyjjgb">17.46</td></tr>
      // </table></div>
      try
      {
        String str = "<td class=\"JgXcPd\">"+valueName+"</td><td class=\"iyjjgb\">";
        String str2 = "<td class=JgXcPd>"+valueName+"</td><td class=iyjjgb>";
        int offset = str.length();
        int s= result.indexOf(str);
        if (s==-1)
        {
          offset = str2.length();
          s = result.indexOf(str2);
          if (s!=-1)
          {
            currPriceStr = result.substring( s+offset);
            if(valueName == "Div yield")
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("%</"));
              if(currPriceStr.startsWith("-")) currPriceStr = "0.0";
            }
            else
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</"));
            retVal = Double.parseDouble(currPriceStr.replace(",",""));
          }
        }
        else
        {
            currPriceStr = result.substring( s+offset);
            if(valueName == "Div yield")
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("%</"));
              if(currPriceStr.startsWith("-")) currPriceStr = "0.0";
            }
            else
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</"));
            retVal = Double.parseDouble(currPriceStr.replace(",",""));
        }
      }
      catch(java.lang.StringIndexOutOfBoundsException oobEx)
      {
        System.out.println("ERROR: scraping "+valueName+". See result html in file "+ "gQuote-"+stockSymbol+".txt");
        oobEx.printStackTrace();
        System.out.println(currPriceStr);
      }
      catch(java.lang.NumberFormatException mfEx)
      {
        System.out.println("ERROR: number Format error scraping "+valueName+". See result html in file "+ "gQuote-"+stockSymbol+".txt");
        mfEx.printStackTrace();
        System.out.println(currPriceStr);
      }
    }
    return retVal;
  }


  /** Get the latest stock Open price from the OLD GFinance Pages .
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  private double getOpenOld(String stockSymbol, String marketSymbol)
  {

    double retVal = -1.0;
    String currPriceStr = null;
    if(!"".equalsIgnoreCase(marketSymbol))
    {
      retVal = 0.0;
      String result = getQuoteString(stockSymbol, marketSymbol);
      writeStringToFile(result,"gQuote-"+stockSymbol+".txt");
      //Looking for
      // <span class="pr">
      // <span id="ref_630737340410842_l">
      //  3.72</span>
      // </span>
      try
      {
        int offset = "Open</td><td class=\"val\">".length();
        int s= result.indexOf("Open</td><td class=\"val\">");
        if (s==-1)
        {
          offset = "Open</td><td class=val>".length();
          s = result.indexOf("Open</td><td class=val>");
          if (s!=-1)
          {
            currPriceStr = result.substring( s+offset);
            currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</"));
            retVal = Double.parseDouble(currPriceStr.replace(",",""));
          }
        }
        else
        {
          currPriceStr = result.substring( s+offset);
          currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf("</"));
          retVal = Double.parseDouble(currPriceStr.replace(",",""));
        }
      }
      catch(java.lang.StringIndexOutOfBoundsException oobEx)
      {
        System.out.println("ERROR: get open parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt");
        oobEx.printStackTrace();
        System.out.println(currPriceStr);
      }
    }
    return retVal;
  }


  /** Get the latest stock Open price from the NEW GFinance pages Circa 2018.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  private double getOpenNew(String stockSymbol, String marketSymbol)
  {
    return scrapeValueNew(stockSymbol, marketSymbol, "Open");
  }


    /** Get the latest stock Open price.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  public double getOpen(String stockSymbol, String marketSymbol)
  {
    return getOpenNew(stockSymbol, marketSymbol);
  }


  /** Get the latest stock Days High price  from the Old GFinance pages before  2018.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  private double getDaysHighOld(String stockSymbol, String marketSymbol)
  {

    double retVal = -1.0;
    String currLowStr = null;
    String currHighStr = null;
    if(!"".equalsIgnoreCase(marketSymbol))
    {
      retVal = 0.0;
      String result = getQuoteString(stockSymbol, marketSymbol);
      writeStringToFile(result,"gQuote-"+stockSymbol+".txt");
      //Looking for
      // Range</td><td class="val">
      try
      {
        int offset = "Range</td><td class=\"val\">".length();
        int s= result.indexOf("Range</td><td class=\"val\">");
        if (s==-1)
        {
          offset = "Range</td><td class=val>".length();
          s = result.indexOf("Range</td><td class=val>");
          if (s!=-1)
          {
            currLowStr = result.substring( s+offset);
            currHighStr = currLowStr.substring( currLowStr.indexOf(" - ")+3);
            currHighStr = currHighStr.substring( 0, currHighStr.indexOf("</td>"));
            currLowStr = currLowStr.substring( 0, currLowStr.indexOf(" - "));
            retVal = Double.parseDouble(currHighStr.replace(",",""));
          }
        }
        else
        {
            currLowStr = result.substring( s+offset);
            currHighStr = currLowStr.substring( currLowStr.indexOf(" - ")+3);
            currHighStr = currHighStr.substring( 0, currHighStr.indexOf("</td>"));
            currLowStr = currLowStr.substring( 0, currLowStr.indexOf(" - "));
            retVal = Double.parseDouble(currHighStr.replace(",",""));
        }
      }
      catch(java.lang.StringIndexOutOfBoundsException oobEx)
      {
        System.out.println("ERROR: get days high parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt");
        oobEx.printStackTrace();
        System.out.println(currHighStr);
      }
    }
    return retVal;
  }


  /** Get the latest stock Days High price  from the NEW GFinance pages Circa 2018.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  private double getDaysHighNew(String stockSymbol, String marketSymbol)
  {
    return scrapeValueNew(stockSymbol, marketSymbol, "High");
  }


  /** Get the latest stock Days High price .
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  public double getDaysHigh(String stockSymbol, String marketSymbol)
  {return getDaysHighNew( stockSymbol,  marketSymbol);}


  /** Get the latest stock Days Low price .
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  private double getDaysLowOld(String stockSymbol, String marketSymbol)
  {

    double retVal = -1.0;
    String currLowStr = null;
    String currHighStr = null;
    if(!"".equalsIgnoreCase(marketSymbol))
    {
      retVal = 0.0;
      String result = getQuoteString(stockSymbol, marketSymbol);
      writeStringToFile(result,"gQuote-"+stockSymbol+".txt");
      //Looking for
      // Range</td><td class="val">
      try
      {
        int offset = "Range</td><td class=\"val\">".length();
        int s= result.indexOf("Range</td><td class=\"val\">");
        if (s==-1)
        {
          offset = "Range</td><td class=val>".length();
          s = result.indexOf("Range</td><td class=val>");
          if (s!=-1)
          {
            currLowStr = result.substring( s+offset);
            currHighStr = currLowStr.substring( currLowStr.indexOf(" - ")+3);
            currHighStr = currHighStr.substring( 0, currHighStr.indexOf("</td>"));
            currLowStr = currLowStr.substring( 0, currLowStr.indexOf(" - "));
            retVal = Double.parseDouble(currLowStr.replace(",",""));
          }
        }
        else
        {
            currLowStr = result.substring( s+offset);
            currHighStr = currLowStr.substring( currLowStr.indexOf(" - ")+3);
            currHighStr = currHighStr.substring( 0, currHighStr.indexOf("</td>"));
            currLowStr = currLowStr.substring( 0, currLowStr.indexOf(" - "));
            retVal = Double.parseDouble(currLowStr.replace(",",""));
        }
      }
      catch(java.lang.StringIndexOutOfBoundsException oobEx)
      {
        System.out.println("ERROR: get days high parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt");
        oobEx.printStackTrace();
        System.out.println(currLowStr);
      }
    }
    return retVal;
  }


  /** Get the latest stock Days Low price  from the NEW GFinance pages Circa 2018.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  private double getDaysLowNew(String stockSymbol, String marketSymbol)
  {
    return scrapeValueNew(stockSymbol, marketSymbol, "Low");
  }


  /** Get the latest stock Days Low price .
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  public double getDaysLow(String stockSymbol, String marketSymbol)
  {return getDaysLowNew( stockSymbol,  marketSymbol);}


  /** Get the latest stock Dividend Yield % .
    * Latest dividend/dividend yield<br><br>
    * Latest dividend is dividend per share paid to shareholders in the most recent quarter.<br>
    * Dividend yield is the value of the latest dividend, multiplied by the number of times dividends are typically paid per year, divided by the stock price.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the % OR -1 if not found;
    **/
  private double getLatestDividendOld(String stockSymbol, String marketSymbol)
  {

    double retVal = -1.0;
    String findStr = "Div/yield</td><td class=\"val\">";
    String findStr2 = "Div/yield</td><td class=val>";
    String dividendStr = null;
    String divYieldStr = null;
    if(!"".equalsIgnoreCase(marketSymbol))
    {
      retVal = 0.0;
      String result = getQuoteString(stockSymbol, marketSymbol);
      writeStringToFile(result,"gQuote-"+stockSymbol+".txt");
      //Looking for
      // Range</td><td class="val">
      try
      {
        int offset = findStr.length();
        int s= result.indexOf(findStr);
        if (s==-1)
        {
          offset = findStr2.length();
          s = result.indexOf(findStr2);
          if (s!=-1)
          {
            dividendStr = result.substring( s+offset);
            if(!dividendStr.startsWith("&"))
            {
              divYieldStr = dividendStr.substring( dividendStr.indexOf("/")+1);
              divYieldStr = divYieldStr.substring( 0, divYieldStr.indexOf("</td>"));
              dividendStr = dividendStr.substring( 0, dividendStr.indexOf("/"));
              retVal = Double.parseDouble(dividendStr.replace(",",""));
            }
          }
        }
        else
        {
          dividendStr = result.substring( s+offset);
          if(!dividendStr.startsWith("&"))
          {
            divYieldStr = dividendStr.substring( dividendStr.indexOf("/")+1);
            divYieldStr = divYieldStr.substring( 0, divYieldStr.indexOf("</td>"));
            dividendStr = dividendStr.substring( 0, dividendStr.indexOf("/"));
            retVal = Double.parseDouble(dividendStr.replace(",",""));
          }
        }
      }
      catch(java.lang.StringIndexOutOfBoundsException oobEx)
      {
        System.out.println("ERROR: get  dividend parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt");
        oobEx.printStackTrace();
        System.out.println(dividendStr);
      }
    }
    return retVal;
  }


  /** Get the latest stock Div Yield %  from the NEW GFinance pages Circa 2018.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the div % OR -1 if not found;
    **/
  private double getLatestDividendNew(String stockSymbol, String marketSymbol)
  {
    return scrapeValueNew(stockSymbol, marketSymbol, "Div yield");
  }


  /** Get the latest stock Div Yield %.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the div % OR -1 if not found;
    **/
  public double getLatestDividend(String stockSymbol, String marketSymbol)
  {return getLatestDividendNew( stockSymbol,  marketSymbol);}


  /** Get the latest stock Volume .
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the most recent daily volume OR -1 if not found;
    **/
  private double getVolumeOld(String stockSymbol, String marketSymbol)
  {

    double retVal = -1.0;
    String currPriceStr = null;
    if(!"MUTF_CA".equalsIgnoreCase(marketSymbol))
    {
      retVal = 0.0;
      String result = getQuoteString(stockSymbol, marketSymbol);
      writeStringToFile(result,"gQuote-"+stockSymbol+".txt");
      //Looking for
      // <span class="pr">
      // <span id="ref_630737340410842_l">
      //  3.72</span>
      // </span>
      try
      {
        int offset = "Vol / Avg.</td>".length();
        int s= result.indexOf("Vol / Avg.</td>");
        if (s==-1)
        {
          s = result.indexOf("class=pr>");
          if (s!=-1)
          {
            currPriceStr = result.substring( s);
            currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>')+1); // should be from 3.72</span
            if(currPriceStr.indexOf('M')!=-1 && currPriceStr.indexOf('M')<8)
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('M'));
              retVal = Double.parseDouble(currPriceStr.replace(",",""))*1000000.0;
            }
            else
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('/'));
              retVal = Double.parseDouble(currPriceStr.replace(",",""));
            }
          }
        }
        else
        {
          currPriceStr = result.substring( s);
          //System.out.println("\nDEBUG:"+currPriceStr);
          currPriceStr = currPriceStr.substring( currPriceStr.indexOf('>', offset)+1); // should be from 3.72</span
          //System.out.println("\nDEBUG:"+currPriceStr);
            if(currPriceStr.indexOf('M')!=-1 && currPriceStr.indexOf('M')<8)
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('M'));
              retVal = Double.parseDouble(currPriceStr.replace(",",""))*1000000.0;
            }
            else
            {
              currPriceStr = currPriceStr.substring( 0, currPriceStr.indexOf('/'));
              retVal = Double.parseDouble(currPriceStr.replace(",",""));
            }
        }
      }
      catch(java.lang.StringIndexOutOfBoundsException oobEx)
      {
        System.out.println("ERROR: get volume parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt");
        oobEx.printStackTrace();
        System.out.println(currPriceStr);
      }
    }
    return retVal;
  }


  /** Get the latest stock Volume from the NEW GFinance pages Circa 2018 - IF IT EXISTED.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the Volume from most resent day -1 if not found;
    **/
  private double getVolumeNew(String stockSymbol, String marketSymbol)
  {
    return -1.0; // does not exist in new Google... scrapeValueNew(stockSymbol, marketSymbol, "Div yield");
  }


  /** Get the latest stock Volume.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the div % OR -1 if not found;
    **/
  public double getVolume(String stockSymbol, String marketSymbol)
  {return getVolumeNew( stockSymbol,  marketSymbol);}


  /** Get the latest stock Dividend Yield.
    * Latest dividend/dividend yield<br><br>
    * Latest dividend is dividend per share paid to shareholders in the most recent quarter.<br>
    * Dividend yield is the value of the latest dividend, multiplied by the number of times dividends are typically paid per year, divided by the stock price.
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the price OR -1 if not found;
    **/
  public double getDividendYieldOld(String stockSymbol, String marketSymbol)
  {

    double retVal = -1.0;
    String findStr = "Div/yield</td><td class=\"val\">";
    String findStr2 = "Div/yield</td><td class=val>";
    String dividendStr = null;
    String divYieldStr = null;
    if(!"".equalsIgnoreCase(marketSymbol))
    {
      retVal = 0.0;
      String result = getQuoteString(stockSymbol, marketSymbol);
      writeStringToFile(result,"gQuote-"+stockSymbol+".txt");
      //Looking for
      // Range</td><td class="val">
      try
      {
        int offset = findStr.length();
        int s= result.indexOf(findStr);
        if (s==-1)
        {
          offset = findStr2.length();
          s = result.indexOf(findStr2);
          if (s!=-1)
          {
            dividendStr = result.substring( s+offset);
            if(!dividendStr.startsWith("&"))
            {
              divYieldStr = dividendStr.substring( dividendStr.indexOf("/")+1);
              divYieldStr = divYieldStr.substring( 0, divYieldStr.indexOf("</td>"));
              dividendStr = dividendStr.substring( 0, dividendStr.indexOf("/"));
              retVal = Double.parseDouble(divYieldStr.replace(",",""));
            }
          }
        }
        else
        {
          dividendStr = result.substring( s+offset);
          if(!dividendStr.startsWith("&"))
          {
            divYieldStr = dividendStr.substring( dividendStr.indexOf("/")+1);
            divYieldStr = divYieldStr.substring( 0, divYieldStr.indexOf("</td>"));
            dividendStr = dividendStr.substring( 0, dividendStr.indexOf("/"));
            retVal = Double.parseDouble(divYieldStr.replace(",",""));
          }
        }
      }
      catch(java.lang.StringIndexOutOfBoundsException oobEx)
      {
        System.out.println("ERROR: get dividend Yield parse error. See result html in file "+ "gQuote-"+stockSymbol+".txt");
        oobEx.printStackTrace();
        System.out.println(divYieldStr);
      }
    }
    return retVal;
  }


  /**
    * Returns the URL string to the Google page for the specified stock.
  **/
  public static String getStockPageUrlStr(String stockSymbol, String marketSymbol)
  {
    String retVal = "";
    retVal = gScrapeQuoteTokenizedUrl.replace(QUOTE_SYMBOL_TOKEN, stockSymbol);
    retVal = retVal.replace(QUOTE_MARKET_TOKEN, googleMarketSymbol(marketSymbol));

    return retVal;
  }


  /** Requests, scrapes and creates the stock QUOTE data string for the given stock symbol for today.
    * this data gets used as the data load into the InvestmentTrackerQuery database.<br>
    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
    *  open,  daysLow,  daysHigh, previousClose,  daysVolume, dividendPerShare,   dividendPayDate<br>
    *  90.01, 89.13, 90.01, 90.34, 386147, 4.18, "10/27/2017"
    *
    * @param stockSymbol is the Symbol of the stock to lookup
    * @param marketSymbol is the stock market symbol that Google uses in the quoteUrl
    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited open,  daysLow,  daysHigh, previousClose,  daysVolume, dividendPerShare,   dividendPayDate OR '' if not found
    **/
  private String getQuoteString(String stockSymbol, String marketSymbol)  { return getQuoteString(stockSymbol, marketSymbol, true);}
  private String getQuoteString(String stockSymbol, String marketSymbol, boolean useCache)
  {
    String retVal = "";
    boolean success = false;
    boolean useDataTool = false;
    //NASDAQ:FCEL   CVE:HIVE   MUTF_CA:AGF201 TSE:LUN

    if(useCache && stockSymbolCache_ == stockSymbol && stockSymbolCache_ == stockSymbol)
    {
      retVal = quoteStringCache_;
    }
    else
    {
      String tokenReplacedScrapeUrl = gScrapeQuoteTokenizedUrl.replace(QUOTE_SYMBOL_TOKEN,stockSymbol);
      tokenReplacedScrapeUrl = tokenReplacedScrapeUrl.replace(QUOTE_MARKET_TOKEN,marketSymbol);
      setScrapePageUrl(tokenReplacedScrapeUrl);

      HashMap <String, String> reqProps = null;

      if(holdings==null && !marketSymbol.equals("MUTF_CA")) // Google is banning my MUTF_CA queries
      {
        if(!"".equals(getUsername()) && !"".equals(getPassword()))
        {
          success = doLogin();
          //System.out.println("\nLogin Response:\n"+postPageResponse_);
        }
        else
          System.out.println("ERROR reading username and password");

        if(success)
        {
          String result = doScrape(reqProps, true); // without any further trimming, only the start/end trimming
          retVal = result;
        }
        stockSymbolCache_ = stockSymbol;
        marketSymbolCache_ = marketSymbol;
        quoteStringCache_ = retVal;
      }
    }
    return retVal;
  }



  /** Create the STOCK_DAILY data string  for the given stock symbol for today.
    * this data gets used as the data load into the InvestmentTrackerQuery database.<br>
    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
    *  open,  daysLow,  daysHigh, close, previousClose,  daysVolume, date, dividendPerShare,   dividendPayDate<br>
    *  90.01, 89.13, 90.01, 90.00, 90.34, 386147, "11/08/2017", 4.18, "10/27/2017"
    *
    * @param stockSymbol is the Symbol of the stock to lookup    *
    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited open,  daysLow,  daysHigh, previousClose,  daysVolume, dividendPerShare,   dividendPayDate OR '' if not found
    **/
  public String getStockDailyString(String stockSymbol)
  {
    String retVal = "";
    boolean success = false;
    JsonObject dataHome = null;
    if(holdings==null)
    {
      if(!"".equals(getUsername()) && !"".equals(getPassword()))
        success = doLogin();
      else
        System.out.println("ERROR reading username and password");

      if(success)
      {
        // scape the summary first to get a list of holdings
        String result = doScrape();
        JsonObject jsO = toJsonObject(result);
        JsonArray jsA = jsO.getJsonArray("Data");
        dataHome = jsA.getJsonObject(0);
        holdings = dataHome.getJsonArray("Holdings");
      }
    }
    else
      System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]");

    if(dataHome!=null && holdings!=null)
    {
      retVal+="\"Open\"";
      retVal+=COL_DELIM;
      retVal+="\"daysLow\"";
      retVal+=COL_DELIM;
      retVal+="\"daysHigh\"";
      retVal+=COL_DELIM;
      retVal+="\"close\"";
      retVal+=COL_DELIM;
      retVal+="\"previousClose\"";
      retVal+=COL_DELIM;
      retVal+="\"date\"";
      retVal+=COL_DELIM;
      retVal+="\"dividendPerShare\"";
      retVal+=COL_DELIM;
      retVal+="\"dividendPayDate\"";
      retVal+="\n";

      setScrapeStart(gScrapeQuoteStart);
      setScrapeEnd(gScrapeQuoteEnd);
      String currSymbol = "";
      String currPrefix = "";
      JsonObject stock =null;

      for(int i=0; i<holdings.size();i++)
      {
        stock = holdings.getJsonObject(i);
        if(stock!=null)
        {
          currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1);
          currPrefix = stock.getJsonString("CountryPrefix").toString().substring(1,stock.getJsonString("CountryPrefix").toString().length()-1);

          if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol.toString()))
          {

            //get the daily quote for each individual holding
            setScrapePageUrl(gScrapeQuoteTokenizedUrl.replace(QUOTE_SYMBOL_TOKEN,currPrefix+currSymbol));
            String result = doScrape();
            System.out.println(" DEBUG Stock Quote Result \n----------------------------\n");
            System.out.println(result);

            /*
            JsonObject jsO = toJsonObject(result);
            JsonString currSymbol = stock.getJsonString("Symbol");
            if(("\""+stockSymbol+"\"").equalsIgnoreCase(currSymbol.toString()))
            {
              JsonNumber  open,  daysLow,  daysHigh, previousClose,  daysVolume, dividendPerShare;
              String   dividendPayDate;
              //open=stock.getJsonString("Price");
              retVal+="\""+dataHome.getString("FriendlyName")+"\"";
              retVal+=COL_DELIM;
              retVal+=""+dataHome.getJsonString("Currency");
              retVal+=COL_DELIM;

            }
            */
            i=holdings.size();
          }
        }
      }
    }

    return retVal;
  }


  public static String googleMarketSymbol(String mkt)
  {
    String retVal = mkt;

    if("TSXV".equalsIgnoreCase(mkt)) retVal = "CVE";
    else if("TSX".equalsIgnoreCase(mkt)) retVal = "TSE";
    else if("MUT".equalsIgnoreCase(mkt)) retVal = "MUTF_CA";
    else if("MUTF".equalsIgnoreCase(mkt)) retVal = "MUTF_CA";

    return retVal;
  }


  /** Create the Portfolio Summary data string for the CR account for today.
    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
    *  FriendlyName, Currency, BookValue, TradeCash,  MarketValue, EquityValue, UnrealizedGainLoss, UnrealizedGainLossPercent, numHoldings, date<br>
    *  "#2V9669V6 - TFSA", 50.5, 10.1, 66.4, 15.9, 8, "10/27/2017"
    *
    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate
    **/
  public String getPortfolioSummaryString()
  {
    String retVal = "";
    boolean success = false;
    JsonObject dataHome = null;
    if(holdings==null)
    {
      if(!"".equals(getUsername()) && !"".equals(getPassword()))
        success = doLogin();
      else
        System.out.println("ERROR reading username and password");

      if(success)
      {
        String result = doScrape();
        JsonObject jsO = toJsonObject(result);
        JsonArray jsA = jsO.getJsonArray("Data");
        dataHome = jsA.getJsonObject(0);
        //holdings = dataHome.getJsonArray("Holdings");
      }
    }
    else
      System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]");

    if(dataHome!=null) // && holdings!=null)
    {
      retVal+="\"friendlyName\"";
      retVal+=COL_DELIM;
      retVal+="\"Currency\"";
      retVal+=COL_DELIM;
      retVal+="\"BookValue\"";
      retVal+=COL_DELIM;
      retVal+="\"TradeCash\"";
      retVal+=COL_DELIM;
      retVal+="\"MarketValue\"";
      retVal+=COL_DELIM;
      retVal+="\"EquityValue\"";
      retVal+=COL_DELIM;
      retVal+="\"UnrealizedGainLoss\"";
      retVal+=COL_DELIM;
      retVal+="\"UnrealizedGainLossPercent\"";
      retVal+=COL_DELIM;
      retVal+="\"NumberOfHoldings\"";
      retVal+=COL_DELIM;
      retVal+="\"date\"";
      retVal+="\n";

      retVal+="\""+dataHome.getString("FriendlyName")+"\"";
      retVal+=COL_DELIM;
      retVal+=""+dataHome.getJsonString("Currency");
      retVal+=COL_DELIM;
      retVal+=""+dataHome.getJsonNumber("BookValue");
      retVal+=COL_DELIM;
      retVal+=""+dataHome.getJsonNumber("TradeCash");
      retVal+=COL_DELIM;
      retVal+=""+dataHome.getJsonNumber("MarketValue");
      retVal+=COL_DELIM;
      retVal+=""+dataHome.getJsonNumber("EquityValue");
      retVal+=COL_DELIM;
      retVal+=""+dataHome.getJsonNumber("UnrealizedGainLoss");
      retVal+=COL_DELIM;
      retVal+=""+dataHome.getJsonNumber("UnrealizedGainLossPercent");
      retVal+=COL_DELIM;
      retVal+=""+(holdings!=null?holdings.size():"0");
      retVal+=COL_DELIM;
      retVal+="\""+dateStr_+"\"";

      /*
      for(int i=0; i<holdings.size();i++)
      {
        JsonObject stock = holdings.getJsonObject(i);
        if(stock!=null)
        {
           String currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1);
            System.out.println(currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+
                               stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+
                               " = "+stock.getJsonNumber("MarketValue").toString());
        }
      }
      */
    }

    return retVal;
  }



  /** Create the Portfolio data string for the CR account for today.
    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
    *  stockName, symbol, CRQuerySymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br>
    *  Hive Tech, HIVE, HIVE, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, "10/27/2017"
    *
    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate
    **/
  public String getPortfolioString(){return getPortfolioString(true);}



  /** Create the Portfolio data string for the CR account for today.
    * It returns a multi-line result-set string with the first line as the column headings and the remaining lies of data<br>
    *  stockName, symbol, CRExchangeSymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate<br>
    *  Hive Tech, HIVE, V, 500, 43.4, CA, 1502.40, 2900.84, 1397.44, 83.4, 10-27-2017
    *
    * @param resultSetOnly true to return only the default delimited resultSet OR false to send a longer more readable string
    * @return the delimited  resultSet OR '' if not found; the resultset is a comma delimited stockName, symbol, CRQuerySymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate
    **/
  public String getPortfolioString(boolean resultSetOnly)
  {
    String retVal = "stockName"+COL_DELIM+"symbol"+COL_DELIM+"CRQuerySymbol"+COL_DELIM+"numShares"+COL_DELIM+"currentPrice"+COL_DELIM+
                    "currency"+COL_DELIM+"bookValue"+COL_DELIM+"marketValue"+COL_DELIM+"gain"+COL_DELIM+"gainPercent"+COL_DELIM+"currentDate\n";
    if(!resultSetOnly) retVal = "\n\n---------------------------------\n    Portfolio Summary\n---------------------------------\n";

    DecimalFormat dfp = new DecimalFormat( "##0" );
    DecimalFormat dfe = new DecimalFormat( "##0.000" );
    boolean success = false;

    if(holdings==null)
    {
      if(!"".equals(getUsername()) && !"".equals(getPassword()))
        success = doLogin();
      else
        System.out.println("ERROR reading username and password");

      if(success)
      {
        String result = doScrape();
        JsonObject jsO = toJsonObject(result);
        JsonArray jsA = jsO.getJsonArray("Data");
        dataHome = jsA.getJsonObject(0);
        holdings = dataHome.getJsonArray("Holdings");
      }
    }
    else
      System.out.println("Already Logged in, using existing holdings["+holdings.size()+"]");

    if(dataHome!=null && holdings!=null)
    {
      if(!resultSetOnly)
      {
        String friendlyName = dataHome.getString("FriendlyName");

        retVal+="    FriendlyName       : "+friendlyName;
        retVal+="\n";
        retVal+="    MarketValue        : "+dataHome.getJsonNumber("MarketValue");
        retVal+="\n";
        retVal+="    TradeCash          : "+dataHome.getJsonNumber("TradeCash");
        retVal+="\n";
        retVal+="    BookValue          : "+dataHome.getJsonNumber("BookValue");
        retVal+="\n";
        retVal+="    ------------------   ------------------------";
        retVal+="\n";
        retVal+="    UnrealizedGainLoss : "+dataHome.getJsonNumber("UnrealizedGainLoss");
        retVal+="\n";
        retVal+="    UnrealizedGain%    : "+
                dfe.format(dataHome.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0);
        retVal+="\n";
        retVal+="\n Holdings Summary\n - - - - - - - - - - - - -\n";
      }

      String currSymbol = "";
      String priceStr = "";
      String currencySymbol = "";
      JsonObject stock = null;
      for(int i=0; i<holdings.size();i++)
      {
        stock = holdings.getJsonObject(i);
        if(stock!=null)
        {
          currSymbol = stock.getJsonString("Symbol").toString().substring(1,stock.getJsonString("Symbol").toString().length()-1);
          if(!resultSetOnly)
          {
            retVal+="   "+currSymbol+" : "+stock.getJsonNumber("Quantity").toString()+" @ "+
                  stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1)+
                  " = "+stock.getJsonNumber("MarketValue").toString();
            retVal+="\n";
          }
          else
          {
            if(right(stock.getJsonString("Price").toString() ,2).equals("U\""))
            {
              priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-2);
              currencySymbol = "US$";
            }
            else
            {
              priceStr = stock.getJsonString("Price").toString().substring(1,stock.getJsonString("Price").toString().length()-1);
              currencySymbol = "C$";
            }
            //stockName, symbol, CRExchangeSymbol,  numShares, currentPrice, currency, bookValue, marketValue, gain, gainPercent, currentDate
            retVal+=stock.getString("SymbolDescription").toString()+COL_DELIM+
                    currSymbol+COL_DELIM+
                    stock.getString("Exchange").toString()+COL_DELIM+
                    stock.getJsonNumber("Quantity").toString()+COL_DELIM+
                    priceStr+COL_DELIM+
                    currencySymbol+COL_DELIM+
                    stock.getJsonNumber("BookValue").toString()+COL_DELIM+
                    stock.getJsonNumber("MarketValue").toString()+COL_DELIM+
                    stock.getJsonNumber("UnrealizedGainLoss").toString()+COL_DELIM+
                    dfe.format(stock.getJsonNumber("UnrealizedGainLossPercent").doubleValue()*100.0)+COL_DELIM+
                    dateStr_;
            retVal+="\n";
          }
        }
      }
      if(!resultSetOnly) retVal+="\n---------------------------\nHoldings Details:\n---------------------------\n"+
                                  prettyJson(holdings.toString());
    }

    return retVal;
  }


  public  String right(String value, int length)
  {
    // To get right characters from a string, change the begin index.
    return value.substring(value.length() - length);
  }


  /** Test method to do whatever tests I want. **/
  @Override
  protected void test(String[] args)
  {
    super.debugOut_=true;
    debugOut_=true;

    //NASDAQ:FCEL   CVE:HIVE   MUTF_CA:AGF201 TSE:LUN
    if(true)
    {
      String symbol = "AGF201";
      String market = "MUTF_CA";
      double result = getQuote(symbol,market);
      System.out.println("Test Quote For "+symbol+"="+result);
      //writeStringToFile(result,"gQuote-"+symbol+".txt");
    }
    else if(false)
    {
      System.out.println("...");
      try
      {

      }
      catch (Exception ex)
      {
        System.out.println("NO GO...");
        ex.printStackTrace();
      }
    }
  }


  public static void main(String[] args)
  {
    GoogleFinanceScraper instance = new GoogleFinanceScraper();

    if(args.length>0 && args[0].toLowerCase().equals("-t"))
      instance.test(args);
    else if(args.length>1 && args[0].equalsIgnoreCase("quote"))
    {
      System.out.println("\n\nQuote For market:symbol "+args[1]+":"+args[2]+" = "+instance.getQuote(args[2],args[1]));
      System.out.println("Volume For market:symbol "+args[1]+":"+args[2]+" = "+instance.getVolume(args[2],args[1]));
      System.out.println("Open Price For market:symbol "+args[1]+":"+args[2]+" = "+instance.getOpen(args[2],args[1]));
      System.out.println("Days LOW Price For market:symbol "+args[1]+":"+args[2]+" = "+instance.getDaysLow(args[2],args[1]));
      System.out.println("Days High Price For market:symbol "+args[1]+":"+args[2]+" = "+instance.getDaysHigh(args[2],args[1]));
      System.out.println("Dividend Yield % For market:symbol "+args[1]+":"+args[2]+" = "+instance.getLatestDividend(args[2],args[1]));
  }
    else
    {
      boolean success = false;
      if(!"".equals(instance.getUsername()) && !"".equals(instance.getPassword()))
        success = instance.doLogin();
      else
        System.out.println("ERROR reading username and password");

      if(success)
      {
        String result = instance.doScrape();
        System.out.println("Saving Scraped page results to file: "+"gPageContent-"+instance.dateStr_+".json");
        writeStringToFile(prettyJson(result),"gPageContent-"+instance.dateStr_+".json");

        System.out.println(instance.getPortfolioString());
      }
    }
  }
}