/*
 *  UrlGrab.java - A widget to grab URLs through a proxy.
 *
 *  $URL: svn://svn.webarts.bc.ca/open/trunk/projects/WebARTS/ca/bc/webarts/widgets/UrlGrab.java $
 *  $Revision: 1235 $
 *  $Date: 2018-03-02 22:30:24 -0800 (Fri, 02 Mar 2018) $
 *
 *
 *  Written by Tom Gutwin - WebARTS Design.
 *  Copyright (C) 2003-2016 WebARTS Design, North Vancouver Canada
 *  http://www.webarts.bc.ca
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
package ca.bc.webarts.widgets;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.IOException;

import java.lang.SecurityException;

import java.util.Properties;

import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.net.PasswordAuthentication;

/**
 * A very simplistic class to grab a URL and save the file to a local file.
 * It includes the option to specify a proxy as well as authentication on that
 * proxy.<br><br>It also includes a main method to test as well as use this
 * classes functionality from the commandline.
 *
 * <pre>
 *    Usage:
 *      java -DproxySet=true -DproxyHost=YourProxyServerHostname
 *           -DproxyPort=YourProxyServerPort
 *           -DproxyUser=YourProxyUsernameIFNEEDED
 *           -DproxyPassword=YourProxyUserPasswordIFNEEDED
 *           -DnonProxyHosts=CommaSeperatedlistOfNonProxiedSites
 *           ca.bc.webarts.widgets.UrlGrab [URL location 1] [URL location2]...
 *</pre>
 *
 * @author     Tom Gutwin P.Eng
 **/
public class UrlGrab extends Authenticator
{
  /** A holder for the NA Clients System File Separator. **/
  private static final String SYSTEM_FILE_SEPERATOR = File.separator;

  /**
   * A holder for the directory location to save downloads. DEFAULT is the
   * current working dir.
   **/
  private static final String DEFAULT_SAVE_LOCATION =
      System.getProperty("user.dir") + SYSTEM_FILE_SEPERATOR;

  public static final String DEFAULT_PROXY_SET = "true";
  public static final String DEFAULT_PROXY_HOST = "companyProxy";
  public static final String DEFAULT_PROXY_PORT = "80";
  public static final String DEFAULT_PROXY_NOPROXY = "";

  static private boolean areWeUsingProxy_ = true;
  static private boolean useProxyAuthentication_ = true;

  /** the class Proxy Username to authenticate with. **/
  static private String proxyUsername_ = "NedFlanders";

  /** the class Proxy Password to authenticate with. **/
  static private String proxyPassword_ = "someGoofyPasswordOfYourOwn";

  /** the Proxy hostname. **/
  static private String proxyHost_ = DEFAULT_PROXY_HOST;

  /** the Proxy port. **/
  static private String proxyPort_ = DEFAULT_PROXY_PORT;



  /**
   * The default empty constructor for this class.  It initsbut does no grabbing.
   **/
  public UrlGrab()
  {
  }


  /**
   * The main constructor for this class.  It inits and grabs the spec'd URL
   * into the current directory.
   *
   * @param urlStr the string representation of the URL to grab.
   *
   **/
  public UrlGrab(String urlStr)
  {
    new File(DEFAULT_SAVE_LOCATION).mkdir(); // ensures the dir exests
    URL urlToGet = null;
      try
      {
        this.initProxy();
        urlToGet = new URL(urlStr);
        grab(urlToGet, DEFAULT_SAVE_LOCATION+
          urlToGet.getFile().substring(urlToGet.getFile().lastIndexOf("/")+1 ));
      }
      catch (MalformedURLException badUrlEx)
      {
        System.out.println("Cannot grab " + urlStr + ".");
      }
  }


  /**
   * A constructor for this class that takes an array of URLs.  It inits and
   * grabs the spec'd URLs into the current directory.
   *
   * @param urlStr the string representation of the URL to grab.
   *
   **/
  public UrlGrab(String[] urlStr)
  {
    new File(DEFAULT_SAVE_LOCATION).mkdir(); // ensures the dir exests
    URL urlToGet = null;
    for (int i=0; i < urlStr.length; i++)
    {
      try
      {
        this.initProxy();
        urlToGet = new URL(urlStr[i]);
        String grabbedFilename = "";
        if (urlStr[i] !=null && !urlStr[i].equals(""))
        {
          grabbedFilename =
          urlToGet.getFile().substring(urlToGet.getFile().lastIndexOf("/")+1 );
          if (grabbedFilename.equals(""))
            grabbedFilename = "unamedGrabbedUrl";
        }
        grab(urlToGet, DEFAULT_SAVE_LOCATION + grabbedFilename);
      }
      catch (MalformedURLException badUrlEx)
      {
        System.out.println("Cannot grab " + urlStr[i] + ".");
      }
    }
  }


  /**
   * The main constructor for this class.  It inits and grabs the spec'd URL
   * into the spec'd filename.
   *
   * @param urlToGet the URL to grab.
   * @param grabbedFilename the filename of the local storage for the URL file.
   *
   **/
  public UrlGrab(URL urlToGet, String grabbedFilename)
  {
    if(urlToGet != null)
    {
      new File(DEFAULT_SAVE_LOCATION).mkdir(); // ensures the dir exests
      this.initProxy();
      grab(urlToGet, grabbedFilename);
    }
    else
    {
      System.out.println("Cannot grab a null URL.");
    }
  }


  /**
   *
   *  A simple/useless constructor for the sole purpose of providing the
   *  Authenticator impl to the net authentication mechanism.  Both the passed
   *  params can be null or "" - if they are the default class username and
   *  pass are used.
   *
   * @param  proxyUser is the usernamer to pass to the proxy
   * @param  proxyPass is the password to pass to the proxy
   */
  public UrlGrab(String proxyUser, String proxyPass)
  {
    if (proxyUser != null && !proxyUser.equals(""))
      proxyUsername_ = proxyUser;
    if (proxyPass != null && !proxyPass.equals(""))
      proxyPassword_ = proxyPass;
  }


  /**
   * The main entry for this app. It takes url strings on the commandline.
   *
   * @param args are the URLs to grab.
   **/
  public static void main(String [] args)
  {
    if (args.length > 0)
    {
      UrlGrab instance = new UrlGrab();
      for (int i=0; i< args.length; i++)
      {
        System.out.println("URL"+i+ " = " + args[i]);
        System.out.println(instance.grab(args[i]));
        System.out.println();
      }
    }
    else
    {
      System.out.println("You must specify at least 1 URL to grab.");
    }
  }


  /**
   *  Does the gruntwork to get the proxy properties set.
   **/

  public static void initProxy()
  {
    // Set up the proxy settings if needed
    Properties sysProps = System.getProperties();
    String tmpUseproxy  = sysProps.getProperty("proxySet");
    if (tmpUseproxy != null && !tmpUseproxy.equals(""))
      areWeUsingProxy_ = (tmpUseproxy.toLowerCase().equals("true")?true:false);

    if (areWeUsingProxy_)
    {
      System.out.println("Initializing the proxy settings");
      //check if the user had something defined in the Sys Props via cmdline.
      String tmpUser  = sysProps.getProperty("http.proxyUser");
      String tmpPass  = sysProps.getProperty("http.proxyPassword");
      String tmpHost  = sysProps.getProperty("http.proxyHost");
      String tmpPort  = sysProps.getProperty("http.proxyPort");

      if (tmpUser != null && !tmpUser.equals(""))
        proxyUsername_ = tmpUser;
      if (tmpPass != null && !tmpPass.equals(""))
        proxyPassword_ = tmpPass;
      if (tmpHost != null && !tmpHost.equals(""))
        proxyHost_ = tmpHost;
      if (tmpPort != null && !tmpPort.equals(""))
        proxyPort_ = tmpHost;

      if (useProxyAuthentication_)
      {
        System.out.println("Authenticating the proxy.");
        Authenticator.setDefault(new UrlGrab(proxyUsername_, proxyPassword_));
      }
      sysProps.put("proxySet", "true");
      sysProps.put("http.proxyHost", proxyHost_);
      sysProps.put("http.proxyPort", proxyPort_);
      sysProps.put("noProxy", "*.bctc.com,*.bchydro.com");
    }

  }


   /**
   *  Impl for Authenticator so an authorizing proxy will work.
   **/

  protected PasswordAuthentication getPasswordAuthentication()
  {
    return new PasswordAuthentication(
        proxyUsername_, proxyPassword_.toCharArray());
  }


  /** Simple URL to String using java 8. **/
  public String grab(String urlStr)
  {
    String retVal = "";
    try
    {
      URL url = new URL(urlStr);
      retVal = grab(url);
    }
    catch (MalformedURLException badUrlEx)
    {
      System.out.println("Cannot grab " + urlStr + " at this time:");
      System.out.println("Reason: Malformed URL Exception URL for " + urlStr);
    }
    return retVal;
  }


  /** Simple URL to String using java 8. **/
  public String grab(URL url)
  {
    String pageText = "empty";
    //java.net.URL url = new java.net.URL(localJolokiaStr_ +"/search/org.eclipse.jetty.webapp:type=webappcontext,*");
    try
    {
      java.net.URLConnection conn = url.openConnection();
      try (java.io.BufferedReader reader = new java.io.BufferedReader(new java.io.InputStreamReader(conn.getInputStream(), "UTF-8")))
      {
        pageText = reader.lines().collect(java.util.stream.Collectors.joining("\n"));
      }
    }
    catch (MalformedURLException badUrlEx)
    {
      System.out.println("Cannot grab " + url + " at this time:");
      System.out.println("Reason: Malformed URL Exception URL for " + url);
    }
    catch (IOException ioEx)
    {
      System.out.println("Cannot grab " + url + " at this time:");
      System.out.println("Reason: The remote download URL specified " +
                         url + " is not permitting a connection.");
    }

    return pageText;
  }


  public boolean grab(URL urlToGet, String grabbedFilename)
  {
    boolean retVal = true;
    if (!(grabbedFilename !=null && !grabbedFilename.equals("")))
      grabbedFilename = "unamedGrabbedUrl";
    //go get it
    try
    {
      URLConnection connection =
        (URLConnection) urlToGet.openConnection();
      System.out.println(connection.getContentType() + "> ("+
                         connection.getContentLength() + ")");

      // Store the file locally
      InputStream inUrl = null;
      if (urlToGet.getProtocol().equals("http"))
        inUrl = ((HttpURLConnection) connection).getInputStream();
      else
        inUrl = connection.getInputStream();
      FileOutputStream ostream = new FileOutputStream(grabbedFilename);

      // start transfering
      byte [] bytesRead= new byte[1024];
      int numBytes = inUrl.read(bytesRead);
      int status = 0;
      System.out.print("..");
      while (numBytes != -1)
      {
        // write what we read
        ostream.write(bytesRead, 0, numBytes);

        // update our status indicator
        status += numBytes;
        if (status % 1024 == 0)
          System.out.print(".");

        // go back for more
        numBytes = inUrl.read(bytesRead);
      }
      System.out.println(". Done ("+ status +" bytes grabbed)");
      ostream.close();
      retVal = true;
    }
    catch (NullPointerException nullEx)
    {
      System.out.println("Cannot grab " + grabbedFilename + " at this time:");
      System.out.println("Reason: Cannot get the content from " +
                         urlToGet.toString());
      retVal = false;
    }
    catch (MalformedURLException badUrlEx)
    {
      System.out.println("Cannot grab " + grabbedFilename + " at this time:");
      System.out.println("Reason: Cannot get the URL for " +
                         grabbedFilename);
      retVal = false;
    }
    catch (SecurityException secEx)
    {
      System.out.println("Cannot grab " + grabbedFilename + " at this time:");
      System.out.println("Reason: Cannot create the file due to security " +
                         "reasons.");
      retVal = false;
    }
    catch (FileNotFoundException fnfEx)
    {
      System.out.println("Cannot grab " + grabbedFilename + " at this time:");
      System.out.println("Reason: Cannot write the File.");
      retVal = false;
    }
    catch (IOException ioEx)
    {
      System.out.println("Cannot grab " + grabbedFilename + " at this time:");
      System.out.println("Reason: The remote download URL specified " +
                         urlToGet + " is not permitting a connection.");
      retVal = false;
    }
    return retVal;
  }

}