/*
 *  $URL: svn://svn.webarts.bc.ca/open/trunk/projects/WebARTS/ca/bc/webarts/tools/UrlScraper.java $
 *  $Author: tgutwin $
 *  $Revision: 1255 $
 *  $Date: 2018-03-17 20:34:04 -0700 (Sat, 17 Mar 2018) $
 */
/*
 *  $Rev: 1255 $:     Revision of last commit
 *  $Author: tgutwin $:  Author of last commit
 *  $Date: 2018-03-17 20:34:04 -0700 (Sat, 17 Mar 2018) $:    Date of last commit
 *  Copyright (C) 2017-2018    WebARTS Design,
 *  North Vancouver Canada. All Rights Reserved.
 *
 *  Written by Tom Gutwin - WebARTS Design.
 *  http://www.webarts.bc.ca
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
package ca.bc.webarts.tools;


// import ca.bc.webarts.tools.MyCookieHandler;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.List;

import javax.json.*;
import javax.net.ssl.HttpsURLConnection;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


/**
 * A very basic tool using JSoup to login to a webPage, get authniticated and then load another page.
 **/
public class UrlScraper
{

  /**  A holder for this clients System File Separator.  */
  public static final String SYSTEM_FILE_SEPERATOR = File.separator;
  /**  A holder for this clients System line termination separator.  */
  public static final String SYSTEM_LINE_SEPERATOR =
  System.getProperty("line.separator");
  protected static final String USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0";
  protected static Calendar rightNow_ = Calendar.getInstance();
  protected String dateStr_ = rightNow_.get(rightNow_.YEAR) + "-" + (rightNow_.get(rightNow_.MONTH) + 1 < 10 ? "0" + rightNow_.get(rightNow_.MONTH) + 1 : String.valueOf(rightNow_.get(rightNow_.MONTH) + 1)) + "-" + (rightNow_.get(rightNow_.DAY_OF_MONTH) < 10 ? "0" + rightNow_.get(rightNow_.DAY_OF_MONTH) : rightNow_.get(rightNow_.DAY_OF_MONTH));
  protected boolean debugOut_ = false;
  protected MyCookieHandler cm_ = new MyCookieHandler();
  protected List<String> cookies_;
  protected HttpsURLConnection connection_;
  protected String baseUrl_ = "/";
  protected boolean alreadyLoggedIn_ = false;
  private String loginUrl_ = "";
  /** Cache of the response text returned from the login page post. **/
  protected String postPageResponse_ = "";
  private HashMap<String, String> requestProps_ = new HashMap<String, String>();
  private String loginFormID_ = "";
  private String usernameFormElementName_ = "";
  private String passwordFormElementName_ = "";
  private String username_ = "";
  private String password_ = "";
  private String scrapePageUrl_ = "";
  private String scrapeStart_ = "";
  private String scrapeEnd_ = "";

  /** Holds the most recent page scrape UNchopped. **/
  private String pageContentCache_ = "";
  protected String stockSymbolCache_ = "";
  protected String marketSymbolCache_ = "";


  /** default constructor does nothing. **/
  public UrlScraper()
  {
    // make sure cookies are turn on
    CookieHandler.setDefault(new CookieManager());
  }


  /** All In One constructor. **/
  public UrlScraper(String loginUrl,
                    HashMap<String, String> requestProps,
                    String loginFormID,
                    String usernameFormElementName,
                    String passwordFormElementName,
                    String username,
                    String password,
                    String scrapePageUrl,
                    String scrapeStart,
                    String scrapeEnd)
  {
    loginUrl_ = loginUrl;requestProps_ = requestProps;loginFormID_ = loginFormID;usernameFormElementName_ = usernameFormElementName;
    passwordFormElementName_ = passwordFormElementName;username_ = username;password_ = password;scrapePageUrl_ = scrapePageUrl;
    scrapeStart_ = scrapeStart;scrapeEnd_ = scrapeEnd;

    // make sure cookies are turn on
    CookieHandler.setDefault(new CookieManager());
  }


  /**
   * Set Method for class field 'debugOut_' to true. Turns on extra debugging System.out stuff.
   *
   **/
  public void setDebugOut()
  {
    this.debugOut_ = true;
  }  // setDebugOut_ Method


  /**
   * Set Method for class field 'debugOut_'. Turns on/off extra debugging System.out stuff.
   *
   * @param debugOut_ is the value to set this class field to.
   *
   **/
  public void setDebugOut(boolean debugOut)
  {
    this.debugOut_ = debugOut;
  }  // setDebugOut_ Method


  /**
   * Get Method for class field 'debugOut_'.
   *
   * @return boolean - The value the class field 'debugOut_'.
   *
   **/
  public boolean getDebugOut()
  {
    return debugOut_;
  }  // getDebugOut Method


  /**
   * Set Method for class field 'baseUrl_'.
   *
   * @param baseUrl_ is the value to set this class field to.
   *
   **/
  public void setBaseUrl(String baseUrl)
  {
    this.baseUrl_ = baseUrl;
  }  // setBaseUrl Method


  /**
   * Get Method for class field 'baseUrl_'.
   *
   * @return String - The value the class field 'baseUrl_'.
   *
   **/
  public String getBaseUrl()
  {
    return baseUrl_;
  }  // getBaseUrl Method


  /**
   * Set Method for class field 'loginUrl_'.
   *
   * @param loginUrl is the value to set this class field to.
   *
   **/
  public void setLoginUrl(String loginUrl)
  {
    this.loginUrl_ = loginUrl;
  }  // setLoginUrl Method


  /**
   * Get Method for class field 'loginUrl_'.
   *
   * @return String - The value the class field 'loginUrl_'.
   *
   **/
  public String getLoginUrl()
  {
    return loginUrl_;
  }  // getLoginUrl Method


  /**
   * Set Method for class field 'requestProps_'.
   *
   * @param requestProps is the value to set this class field to.
   *
   **/
  public void setRequestProps(HashMap<String, String> requestProps)
  {
    this.requestProps_ = requestProps;
  }  // setRequestProps Method


  /**
   * Get Method for class field 'requestProps_'.
   *
   * @return HashMap<String, String> - The value the class field 'requestProps_'.
   *
   **/
  public HashMap<String, String> getRequestProps()
  {
    return requestProps_;
  }  // getRequestProps Method


  /**
   * Set Method for class field 'loginFormID_'.
   *
   * @param loginFormID is the value to set this class field to.
   *
   **/
  public void setLoginFormID(String loginFormID)
  {
    this.loginFormID_ = loginFormID;
  }  // setLoginFormID Method


  /**
   * Get Method for class field 'loginFormID_'.
   *
   * @return String - The value the class field 'loginFormID_'.
   *
   **/
  public String getLoginFormID()
  {
    return loginFormID_;
  }  // getLoginFormID Method


  /**
   * Set Method for class field 'usernameFormElementName_'.
   *
   * @param usernameFormElementName_ is the value to set this class field to.
   *
   **/
  public void setUsernameFormElementName(String usernameFormElementName)
  {
    this.usernameFormElementName_ = usernameFormElementName;
  }  // setUsernameFormElementName_ Method


  /**
   * Get Method for class field 'usernameFormElementName_'.
   *
   * @return String - The value the class field 'usernameFormElementName_'.
   *
   **/
  public String getUsernameFormElementName()
  {
    return usernameFormElementName_;
  }  // getUsernameFormElementName Method


  /**
   * Set Method for class field 'passwordFormElementName_'.
   *
   * @param passwordFormElementName_ is the value to set this class field to.
   *
   **/
  public void setPasswordFormElementName(String passwordFormElementName)
  {
    this.passwordFormElementName_ = passwordFormElementName;
  }  // setPasswordFormElementName Method


  /**
   * Get Method for class field 'passwordFormElementName_'.
   *
   * @return String - The value the class field 'passwordFormElementName_'.
   *
   **/
  public String getPasswordFormElementName()
  {
    return passwordFormElementName_;
  }  // getPasswordFormElementName Method


  /**
   * Set Method for class field 'username_'.
   *
   * @param username_ is the value to set this class field to.
   *
   **/
  public void setUsername(String username)
  {
    this.username_ = username;
  }  // setUsername Method


  /**
   * Get Method for class field 'username_'.
   *
   * @return String - The value the class field 'username_'.
   *
   **/
  public String getUsername()
  {
    return username_;
  }  // getUsername Method


  /**
   * Set Method for class field 'password_'.
   *
   * @param password_ is the value to set this class field to.
   *
   **/
  public void setPassword(String password)
  {
    this.password_ = password;
  }  // setPassword Method


  /**
   * Get Method for class field 'password_'.
   *
   * @return String - The value the class field 'password_'.
   *
   **/
  public String getPassword()
  {
    return password_;
  }  // getPassword Method


  /**
   * Set Method for class field 'scrapePageUrl_'.
   *
   * @param scrapePageUrl_ is the value to set this class field to.
   *
   **/
  public void setScrapePageUrl(String scrapePageUrl)
  {
    this.scrapePageUrl_ = scrapePageUrl;
  }  // setScrapePageUrl_ Method


  /**
   * Get Method for class field 'scrapePageUrl_'.
   *
   * @return String - The value the class field 'scrapePageUrl_'.
   *
   **/
  public String getScrapePageUrl()
  {
    return scrapePageUrl_;
  }  // getScrapePageUrl Method


  /**
   * Set Method for class field 'scrapeStart_'.
   * This field is used as the substring start tag for the return when searching the response string.
   *
   * @param scrapeStart_ is the value to set this class field to.
   *
   **/
  public void setScrapeStart(String scrapeStart)
  {
    this.scrapeStart_ = scrapeStart;
  }  // setScrapeStart Method


  /**
   * Get Method for class field 'scrapeStart_'.
   * This field is used as the substring start tag for the return when searching the response string.
   *
   * @return String - The value the class field 'scrapeStart_'.
   *
   **/
  public String getScrapeStart()
  {
    return scrapeStart_;
  }  // getScrapeStart Method


  /**
   * Set Method for class field 'scrapeEnd_'.
   * This field is used as the substring end tag for the return when searching the response string.
   *
   * @param scrapeEnd_ is the value to set this class field to.
   *
   **/
  public void setScrapeEnd(String scrapeEnd)
  {
    this.scrapeEnd_ = scrapeEnd;
  }  // setScrapeEnd Method


  /**
   * Get Method for class field 'scrapeEnd_'.
   * This field is used as the substring end tag for the return when searching the response string.
   *
   * @return String - The value the class field 'scrapeEnd_'.
   *
   **/
  public String getScrapeEnd()
  {
    return scrapeEnd_;
  }  // getScrapeEnd Method


  public List<String> getCookies()
  {
    return cookies_;
  }


  public void setCookies(List<String> cookies)
  {
    if (debugOut_)
    {
      System.out.println("New cookies: " + cookies);
      for (String cookie : cookies) {
        System.out.println("    --> cookie " + cookie);
      }
    }


    this.cookies_ = cookies;
  }


  /**
   * Set Method for class field 'connection_'.
   *
   * @param connection_ is the value to set this class field to.
   *
   **/
  public void setConnection(HttpsURLConnection connection)
  {
    this.connection_ = connection;
  }  // setConnection Method


  /**
   * Get Method for class field 'connection_'.
   *
   * @return HttpsURLConnection - The value the class field 'connection_'.
   *
   **/
  public HttpsURLConnection getConnection()
  {
    return connection_;
  }  // getConnection Method


  /**
   * Sends the POST to the login url  parameters from the classVars.
   * It also caches the page response text into thew classVar postPageResponse_.<br>
   */
  public boolean doLogin()
  {
    return this.doLogin(loginUrl_, requestProps_, loginFormID_, usernameFormElementName_, passwordFormElementName_, username_, password_);
  }


  /**
   * Sends the POST to the login url with all required parameters.
   * It also caches the page response text into thew classVar postPageResponse_.<br>
   * <br> example requestProps<pre>
   *     HashMap <String, String> reqProps = new HashMap<String, String>();
   *     reqProps.put("Accept","text/html,application/xhtml+xml,application/xml");
   *     reqProps.put("Accept-Encoding  ","gzip, deflate, br");
   *     reqProps.put("Accept-Language  ","en-US,en;q=0.5");
   *     reqProps.put("Connection","keep-alive");
   *     reqProps.put("Content-Type","application/x-www-form-urlencoded");
   *     reqProps.put("Host","red.webarts.bc.ca");
   *     //reqProps.put("Referer","red.webarts.bc.ca");
   *     reqProps.put("Upgrade-Insecure-Requests","1");
   *</pre><br><B>NOTE:</B> you don't need to add the "User-Agent" or "Content-Length"
   *
   * @param loginUrl
   * @param requestProps
   * @param formID
   * @param usernameFormElementName
   * @param passwordFormElementName
   * @param username
   * @param password
   * @return boolean true if loggedIn
   **/
  public boolean doLogin( String loginUrl,
                          HashMap<String, String> requestProps,
                          String formID,
                          String usernameFormElementName,
                          String passwordFormElementName,
                          String username,
                          String password)
  {
    boolean retVal = false;
    try
    {
      baseUrl_ = loginUrl.substring(0, loginUrl.lastIndexOf('/') - 1);
      if (debugOut_) System.out.println("\n doLogin \n");
      if (debugOut_) System.out.println("loginUrl_:" + loginUrl);
      if (debugOut_) System.out.println("requestProps_:" + requestProps);
      if (debugOut_) System.out.println("loginFormID_:" + formID);
      if (debugOut_) System.out.println("usernameFormElementName_:" + usernameFormElementName);
      if (debugOut_) System.out.println("passwordFormElementName_:" + passwordFormElementName);
      if (debugOut_) System.out.println("username_,password_:" + username + " , " + password);
      String page = getPageContent(loginUrl);
      if (debugOut_) System.out.println("-------------------\nLoginPage:\n" + page + "\n-----------------\n");

      String postParams = getLoginFormParams(    page,
      formID,
      usernameFormElementName, passwordFormElementName,
      username, password);
      retVal = (sendPost(loginUrl, postParams, requestProps) == 200 ? true : false);
    }
    catch (Exception ex)
    {
      retVal = false;
    }
    alreadyLoggedIn_ = retVal;
    return retVal;
  }


  /** Scrape and return string between the scrapeStart and scrapeEnd from the url. **/
  public String doScrape( String scrapePageUrl,
                          String scrapeStart,
                          String scrapeEnd)
  {
    String retVal = "";
    try
    {
      String result = getPageContent(scrapePageUrl);
      retVal = result.substring(    result.indexOf(scrapeStart),
      result.indexOf(scrapeEnd));
    }
    catch (Exception ex)
    {
      retVal = "";
    }
    return retVal;
  }


  /** Scrape and return string between the scrapeStart and scrapeEnd from the url. **/
  public String doScrape( String scrapePageUrl,
                          String scrapeStart,
                          String scrapeEnd,
                          HashMap<String, String> reqProps,
                          boolean useCache)
  {
    String retVal = "";
    try
    {
      String result = pageContentCache_;
      if(!useCache || "".equals(pageContentCache_))
      {
        if (debugOut_) System.out.println(" doScrape : "+scrapePageUrl);
        result = getPageContent(scrapePageUrl, reqProps);
      }
      pageContentCache_ = result;
      int s = result.indexOf(scrapeStart);
      int e = result.indexOf(scrapeEnd);
      if (s==-1) s=0;
      if (e==-1) e=result.length();
      if (debugOut_) System.out.println("  Scrape Start="+scrapeStart + "   index="+s);
      if (debugOut_) System.out.println("  Scrape End="+scrapeEnd + "   index="+e);
      retVal = result.substring( s, e);
    }
    catch (Exception ex)
    {
      retVal = "";
    }
    return retVal;
  }


  /** Scrape (not using cache) and return response string from the url. **/
  public String doScrape( String scrapePageUrl,
                          String scrapeStart,
                          String scrapeEnd,
                          HashMap<String, String> reqProps)
  {
    return doScrape(  scrapePageUrl,
                      scrapeStart,
                      scrapeEnd,
                      reqProps,
                      false);
  }


  /** Scrape and return response string from the url. **/
  public String doScrape(String scrapePageUrl, HashMap<String, String> reqProps)
  {
    return doScrape(  scrapePageUrl,
                      getScrapeStart(),
                      getScrapeEnd(),
                      reqProps,
                      false);
  }


  /** scrapes and returns the data string using the default class fields for scrape start and end and requestprops. **/
  public String doScrape(String scrapePageUrl)
  {
    return doScrape(  scrapePageUrl, getScrapeStart(), getScrapeEnd(), getRequestProps(), false);
  }


  /** Scrape (optionally using the cache) and return response string from the url. **/
  public String doScrape(HashMap<String, String> reqProps, boolean useCache)
  {
    return doScrape(getScrapePageUrl(), getScrapeStart(), getScrapeEnd(), reqProps, useCache);
  }


  /** Scrape and return response string from the url. **/
  public String doScrape(HashMap<String, String> reqProps)
  {
    return doScrape(getScrapePageUrl(), getScrapeStart(), getScrapeEnd(), reqProps, false);
  }


  /** Scrape (optionally using the cache) and return response string from the url. **/
  public String doScrape(boolean useCache)
  {
    return doScrape(getScrapePageUrl(), getScrapeStart(), getScrapeEnd(), getRequestProps(), useCache);
  }


  /** Scrape and return response string from the url. **/
  public String doScrape()
  {
    return doScrape(getScrapePageUrl(), getScrapeStart(), getScrapeEnd(), getRequestProps(), false);
  }


  /** Makes the JSON string pretty with indenting. **/
  public static String prettyJson(String jsonStr)
  {
    String retVal = jsonStr;
    retVal = retVal.replace("[", " [\n");
    retVal = retVal.replace("]", "  ]" + SYSTEM_LINE_SEPERATOR);
    retVal = retVal.replace("]\n\"", "  ]\"" + SYSTEM_LINE_SEPERATOR);
    retVal = retVal.replace("{", "  {" + SYSTEM_LINE_SEPERATOR + "      ");
    retVal = retVal.replace("}", "}" + SYSTEM_LINE_SEPERATOR);
    retVal = retVal.replace(",", "," + SYSTEM_LINE_SEPERATOR + "      ");
    retVal = retVal.replace("}" + SYSTEM_LINE_SEPERATOR + "," + SYSTEM_LINE_SEPERATOR, "    }," + SYSTEM_LINE_SEPERATOR);
    retVal = retVal.replace("[\n", "  [ ");
    retVal = retVal.replace("        {", "    {");
    retVal = retVal.replace("[   {", "[\n   {");
    retVal = retVal.replace("\n}", "\n    }");
    return retVal;
  }


  /** As the name says. **/
  public JsonObject toJsonObject(String jsonStr)
  {
    JsonReader jsonReader = Json.createReader(new StringReader(jsonStr));
    JsonObject jsO = jsonReader.readObject();
    jsonReader.close();
    return jsO;
  }


  public JsonStructure toJsonStructure(String jsonStr)
  {
    JsonReader jsonReader = Json.createReader(new StringReader(jsonStr));
    JsonStructure jsO = jsonReader.read();
    jsonReader.close();
    return jsO;
  }


  /** Test method to do whatever tests I want. **/
  protected void test(String[] args)
  {
    test(args, getLoginFormID(), getRequestProps(),
    getLoginFormID(), getUsernameFormElementName(),
    getPasswordFormElementName(), getUsername(), getPassword());
  }


  /** . **/
  protected void test(String[] args,
                      String loginUrl,
                      HashMap<String, String> requestProps,
                      String formID,
                      String usernameFormElementName,
                      String passwordFormElementName,
                      String username,
                      String password)
  {
    String result = readFileToString("scrapedPageContent-" + dateStr_ + ".json");
    JsonStructure jsS = toJsonStructure(result);
    System.out.println("\n\n---------------------------\nFull Data Structure: \n---------------------------\n" + jsS.toString());
    System.out.println("ValueType=" + jsS.getValueType());
    JsonObject jsO = (JsonObject)jsS;
    JsonArray jsA = jsO.getJsonArray("Data");
    JsonObject dataHome = jsA.getJsonObject(0);
    String friendlyName = dataHome.getString("FriendlyName");
    System.out.println("    FriendlyName       : " + friendlyName);
    System.out.println("    MarketValue        : " + dataHome.getJsonNumber("MarketValue"));
    System.out.println("    TradeCash          : " + dataHome.getJsonNumber("TradeCash"));
    System.out.println("    BookValue          : " + dataHome.getJsonNumber("BookValue"));
    System.out.println("    ------------------   ------------------------");
    System.out.println("    UnrealizedGainLoss : " + dataHome.getJsonNumber("UnrealizedGainLoss"));
    JsonArray holdings = dataHome.getJsonArray("Holdings");

    // System.out.println("holdings ValueType="+holdings.getValueType());
    System.out.println("\n\n---------------------------\nHoldings:\n---------------------------\n" + prettyJson(holdings.toString()));
  }


  public static void main(String[] args)
  {
    String loginUrl = "https://red.webarts.bc.ca/owncloud/";
    String scrapePageUrl = "https://red.webarts.bc.ca/owncloud/index.php/apps/files/";
    String ocLoginFormElement = "login";  /* id of the form element */
    String ocUserLoginElement = "user";  /* the element name in the form to look for */
    String ocPasswordElement = "password";  /* the element name in the form to look for */
    String ocUserLogin = "tgutwin";
    String ocPassword = "";

    // String baseUrl="/";
    HashMap<String, String> reqProps = new HashMap<String, String>();
    reqProps.put("Accept", "text/html,application/xhtml+xml,application/xml");
    reqProps.put("Accept-Encoding       ", "gzip, deflate, br");
    reqProps.put("Accept-Language       ", "en-US,en;q=0.5");
    reqProps.put("Connection", "keep-alive");
    reqProps.put("Content-Type", "application/x-www-form-urlencoded");
    reqProps.put("Upgrade-Insecure-Requests", "1");
    reqProps.put("Host", "red.webarts.bc.ca");
    UrlScraper instance = new UrlScraper();
    if (args.length > 0 && args[0].toLowerCase().equals("-t"))
    {
      instance.test(args, loginUrl, reqProps, ocLoginFormElement, ocUserLoginElement, ocPasswordElement, ocUserLogin, ocPassword);
    }
    else
    {

      /*
       * @param loginUrl
       * @param requestProps
       * @param formID
       * @param usernameFormElementName
       * @param passwordFormElementName
       * @param username
       * @param password
       * @return boolean true if loggedIn
       */
      boolean success = false;
      success = instance.doLogin(loginUrl, reqProps, ocLoginFormElement, ocUserLoginElement, ocPasswordElement, ocUserLogin, ocPassword);

      // 4. success then go to the page you want.
      String result = "";
      String summaryTableHtml = "";
      if (success)
      {
        result = instance.doScrape(scrapePageUrl);
        writeStringToFile(result, "ocPageContent-" + instance.dateStr_ + ".html");
        System.out.println(result);
      }
    }
  }


  /**
   * Sends a POST request to the url, along with all the passed post parameters and request properties.
   *
   * @param url is the url to post this stuff to
   * @param postParams is a sequential string of params that get sent in this post
   * @param
   **/
  protected int sendPost(String url, String postParams, HashMap<String, String> reqProps) throws MalformedURLException, IOException, ProtocolException
  {
    if (debugOut_) System.out.println("\n sendPost to url=" + url);
    URL obj = new URL(url);
    connection_ = (HttpsURLConnection)obj.openConnection();

    // Act like a browser
    connection_.setUseCaches(false);
    connection_.setRequestMethod("POST");
    connection_.setRequestProperty("User-Agent", USER_AGENT);
    for (String key : reqProps.keySet())
    {
      if (!"User-Agent".equals(key) && !"Content-Length".equals(key) && !"Cookie".equals(key) && !"".equals(key.trim()) && !"".equals(((String)reqProps.get(key)).trim()))
      {
        connection_.setRequestProperty(key.trim(), ((String)reqProps.get(key)).trim());
      }


      if (debugOut_)
      {
        System.out.println("   setRequestProperty  " + key.trim() + " = " + ((String)reqProps.get(key)).trim());
      }
    }
    connection_.setRequestProperty("Content-Length", Integer.toString(postParams.length()));

    /*
     * for (String cookie : this.cookies_)
     * {
     * if(   !"".equals(cookie)
     * && !"".equals(((String)cookie.split(";", 1)[0]).trim()) )
     * {
     * connection_.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
     * if(debugOut_) System.out.println("   full  cookie = "+cookie);
     * if(debugOut_) System.out.println("   adding cookie = "+cookie.split(";", 1)[0]);
     * }
     * }
     */
    connection_.setDoOutput(true);
    connection_.setDoInput(true);

    // connection_.setInstanceFollowRedirects(false);
    cm_.setCookies(connection_);
    connection_.connect();

    // Send post request
    DataOutputStream wr = new DataOutputStream(connection_.getOutputStream());
    wr.writeBytes(postParams);
    wr.flush();
    wr.close();
    int responseCode = connection_.getResponseCode();
    cm_.storeCookies((java.net.URLConnection)connection_);
    System.out.println("\nSent 'POST' request to URL : " + url);
    System.out.println("Post parameters : " + postParams);
    System.out.println("Response Code : " + responseCode);

    // Read the response
    BufferedReader in = new BufferedReader(new InputStreamReader(connection_.getInputStream()));
    String inputLine;
    StringBuffer response = new StringBuffer();
    while ((inputLine = in.readLine()) != null)
    {
      response.append(inputLine);
    }
    in.close();
    postPageResponse_ = response.toString();
    if (debugOut_)  System.out.println("\n\n  -----------------\nPostPage Response:  ");
    if (debugOut_) System.out.println(postPageResponse_);

    writeStringToFile(postPageResponse_, "postPageResponse.html");
    return responseCode;
  }


  /** Connects/retrieves a URL; pulls its cookies and returns the resulting htnl as a string.
   **/
  protected String getPageContent(String url) throws MalformedURLException, IOException, ProtocolException
  {
    return getPageContent(url, null);
  }


  /**
   * Connects/retrieves a URL; adds its cookies and returns the resulting htnl as a string.
   *
   * @param url is the url to get
   * @param reqProps optional (can be null) map of properties to add as requestProperties
   **/
  protected String getPageContent(String url, HashMap<String, String> reqProps) throws MalformedURLException, IOException, ProtocolException
  {
    if (debugOut_)
    {
      System.out.println("\n GetPageContent from url= " + url);
    }
    URL obj = new URL(url);
    connection_ = (HttpsURLConnection)obj.openConnection();

    // default is GET
    connection_.setRequestMethod("GET");
    connection_.setUseCaches(false);

    // act like a browser
    connection_.setRequestProperty("User-Agent", USER_AGENT);
    connection_.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
    connection_.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
    if (reqProps != null)
    {
      for (String key : reqProps.keySet())
      {
        if (!"User-Agent".equals(key) &&
            !"Content-Length".equals(key) &&
            !"Cookie".equals(key) &&
            !"".equals(key.trim()) &&
            !"".equals(((String)reqProps.get(key)).trim()))
        {
          connection_.setRequestProperty(key.trim(), ((String)reqProps.get(key)).trim());
          if (debugOut_) System.out.println("   setRequestProperty  " + key.trim() + " = " + ((String)reqProps.get(key)).trim());
        }
      }
    }
    if (cookies_ != null)
    {
      for (String cookie : this.cookies_)
      {
        if (debugOut_) System.out.println("adding cookies to page request: " + cookie + " (" + cookie.split(";", 1)[0] + ")");
        connection_.addRequestProperty("Cookie", cookie.split(";", 1)[0]);  // add only the first in a group
      }
    }

    cm_.setCookies(connection_);
    connection_.connect();
    System.out.println("\nSending 'GET' request to URL : " + url);
    BufferedReader in = new BufferedReader(new InputStreamReader(connection_.getInputStream()));
    String inputLine;
    StringBuffer response = new StringBuffer();
    while ((inputLine = in.readLine()) != null)
    {
      response.append(inputLine);
    }
    in.close();
    int responseCode = connection_.getResponseCode();
    System.out.println("Response Code : " + responseCode);
    cm_.storeCookies((java.net.URLConnection)connection_);

    // Get the response cookies
    setCookies(connection_.getHeaderFields().get("Set-Cookie"));
    return response.toString();
  }


  /** Concatenates together the URL parameter string (ie. name=value&name2=val2&someOtherParamName=val3  )
   * for a specific login form in the passed URL/html string. It also subsititutes the username and password for
   * for the appropriate form elements. <br> This parameterString ends up getting sent to the POST request for a that form.
   *
   * @param html is the string representation of the URL that has the form to parse
   * @param formID the form id (or name) to parse parameters from
   * @param usernameFormElementName the form elementName used for the username field
   * @param passwordFormElementNamethe form elementName used for the password field
   * @param username the actual login username to use in the form
   * @param password the actual login password to use in the form
   **/
  public String getLoginFormParams(String html,
  String formID,
  String usernameFormElementName,
  String passwordFormElementName,
  String username,
  String password) throws UnsupportedEncodingException
  {
    System.out.println("Extracting form's data...");
    if (debugOut_)      System.out.println("\n ---------------------------------------- \n" + html + "\n ---------------------------------------- \n");

    Document doc = Jsoup.parse(html);
    if (debugOut_) System.out.println("\n doc is parsed?=" + (doc != null));
    if (debugOut_) System.out.println(" Looking for form id=" + formID);

    Element loginform = null;
    loginform = doc.getElementById(formID);
    if (loginform == null)
    {
      loginform = doc.getElementsByAttributeValue("name", formID).first();
    }
    else
    if (loginform == null)
    {
      loginform = doc.getElementsByAttributeValue("class", formID).first();
    }

    Elements inputElements = loginform.getElementsByTag("input");
    List<String> paramList = new ArrayList<String>();
    for (Element inputElement : inputElements) {
      String key = inputElement.attr("name");
      String value = inputElement.attr("value");
      if (debugOut_)
      {
        System.out.println("     form element " + key + " = " + value);
      }
      if (key.equals(usernameFormElementName))
      {
        value = username;
      }
      else
      if (key.equals(passwordFormElementName))
      {
        value = password;
      }
      else
      if (key.equals("timezone"))
      {
        value = "America/Los_Angeles";
      }
      else
      if (key.equals("timezone-offset"))
      {
        value = "-7";
      }
      paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
    }

    // build parameters list
    StringBuilder result = new StringBuilder();
    for (String param : paramList)
    {
      if (result.length() == 0)
      {
        result.append(param);
      }
      else
      {
        result.append("&" + param);
      }
    }
    return result.toString();
  }


  /**
   * Abstracts the writing of string to a file.
   *
   * @param s is the String to writeout
   * @param fileName is the file name of the file to write the String into
   * @return if success.. the full pathed filename is returned else null
   **/
  public static String writeStringToFile(String s, String fileName)
  {
    return writeStringToFile(s, fileName, false);
  }


  /**
   * Abstracts the writing of string to a (zip) file (Zip NOT IMPLEMENTED YET).
   *
   * @param s is the String to writeout
   * @param fileName is the file name of the file to write the String into
   * @param zipCompress boolean fall to compress with zip compression
   * @return if success.. the full pathed filename is returned else null
   **/
  public static String writeStringToFile(String s, String fileName, boolean zipCompress)
  {
    String retVal = fileName;
    try
    {
      // FileWriter was not closing the stream
      /*
       * FileWriter f = new FileWriter(fileName);
       * f.write(s);
       * f.flush();
       * f.close();
       * f = null;
       */
      FileOutputStream fos = new FileOutputStream(fileName);
      byte[] strBytes = s.getBytes();
      fos.write(strBytes);
      fos.flush();
      fos.close();
      fos = null;
      System.gc();  // this is required because a bug in Java won't realease
    }
    catch (IOException ioEx)
    {
      System.out.println("\nERROR Writing file: " + fileName);
      retVal = null;
    }
    return retVal;
  }


  /**
   * Abstracts the reading of a file and returns the contents as a String.
   *
   * @param fileName is the file naem to read into a String
   * @return the Text file contents as a String
   **/
  public static String readFileToString(String fileName)
  {
    String stringLine;
    BufferedReader in;
    StringBuffer stringOut = new StringBuffer();
    try
    {
      in = new BufferedReader(new FileReader(fileName));
      while ((stringLine = in.readLine()) != null) {
        stringOut.append(stringLine);
        stringOut.append(SYSTEM_LINE_SEPERATOR);
      }
    }
    catch (FileNotFoundException fnfEx)
    {
      System.out.println("Cannot find file: " + fileName);
    }
    catch (IOException ioEx)
    {
      System.out.println("Error Reading File to String: " + fileName);
    }
    return stringOut.toString();
  }
}