001package ca.bc.webarts.tools;
002
003import java.io.BufferedReader;
004import java.io.DataOutputStream;
005import java.io.FileOutputStream;
006import java.io.InputStreamReader;
007import java.io.IOException;
008import java.io.UnsupportedEncodingException;
009import java.net.CookieHandler;
010import java.net.CookieManager;
011import java.net.URL;
012import java.net.URLEncoder;
013import java.util.HashMap;
014import java.util.ArrayList;
015import java.util.Calendar;
016import java.util.List;
017
018import javax.net.ssl.HttpsURLConnection;
019
020import org.jsoup.Jsoup;
021import org.jsoup.nodes.Document;
022import org.jsoup.nodes.Element;
023import org.jsoup.select.Elements;
024
025
026/**
027  * A very basic tool using JSoup to login to a webPage, get authniticated and then load another page.
028**/
029public class HttpUrlConnectionScrape
030{
031  protected static final int SCRAPE_OC = 0;
032  protected static final int SCRAPE_CR = 1;
033  protected static final String USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0";
034
035  protected static int scrapeSite_ = SCRAPE_CR;
036
037  protected static Calendar rightNow_ = Calendar.getInstance();
038
039  protected String dateStr_ = String.valueOf(rightNow_.get(rightNow_.YEAR))+"-"+
040                   (rightNow_.get(rightNow_.MONTH)+1<10?"0"+String.valueOf(rightNow_.get(rightNow_.MONTH)+1):String.valueOf(rightNow_.get(rightNow_.MONTH)+1))+"-"+
041                   (rightNow_.get(rightNow_.DAY_OF_MONTH)<10?"0"+String.valueOf(rightNow_.get(rightNow_.DAY_OF_MONTH)):String.valueOf(rightNow_.get(rightNow_.DAY_OF_MONTH)));
042
043  protected boolean debugOut_=true;
044  protected List<String> cookies_;
045  protected HttpsURLConnection connection_;
046  protected String baseUrl_ = "/";
047
048  private String loginUrl_ = "";
049  /** Cache of the response text returned from the login page post. **/
050  protected String loginPageResponse_ = "";
051  private HashMap <String, String> requestProps_ = new HashMap<String, String>();;
052  private String loginFormID_ = "";
053  private String usernameFormElementName_ = "";
054  private String passwordFormElementName_ = "";
055  private String username_ = "";
056  private String password_ = "";
057  private String scrapePageUrl_ = "";
058  private String scrapeStart_ = "";
059  private String scrapeEnd_ = "";
060
061
062  /** default constructor does nothing. **/
063  public HttpUrlConnectionScrape( ){}
064
065
066  /** All In One constructor. **/
067  public HttpUrlConnectionScrape(  String loginUrl,
068                                   HashMap <String, String> requestProps,
069                                   String loginFormID,
070                                   String usernameFormElementName,
071                                   String passwordFormElementName,
072                                   String username,
073                                   String password,
074                                   String scrapePageUrl,
075                                   String scrapeStart,
076                                   String scrapeEnd)
077  {
078  }
079
080
081  /**
082    * Set Method for class field 'baseUrl_'.
083    *
084    * @param baseUrl_ is the value to set this class field to.
085    *
086    **/
087  public  void setBaseUrl(String baseUrl)
088  {
089    this.baseUrl_ = baseUrl;
090  }  // setBaseUrl Method
091
092
093  /**
094    * Get Method for class field 'baseUrl_'.
095    *
096    * @return String - The value the class field 'baseUrl_'.
097    *
098    **/
099  public String getBaseUrl()
100  {
101    return baseUrl_;
102  }  // getBaseUrl Method
103
104
105  /**
106    * Set Method for class field 'loginUrl_'.
107    *
108    * @param loginUrl is the value to set this class field to.
109    *
110    **/
111  public  void setLoginUrl(String loginUrl)
112  {
113    this.loginUrl_ = loginUrl;
114  }  // setLoginUrl Method
115
116
117  /**
118    * Get Method for class field 'loginUrl_'.
119    *
120    * @return String - The value the class field 'loginUrl_'.
121    *
122    **/
123  public String getLoginUrl()
124  {
125    return loginUrl_;
126  }  // getLoginUrl Method
127
128
129  /**
130    * Set Method for class field 'requestProps_'.
131    *
132    * @param requestProps is the value to set this class field to.
133    *
134    **/
135  public HashMap<String, String> setRequestProps(HashMap <String, String> requestProps)
136  {
137    this.requestProps_ = requestProps;
138  }  // setLoginUrl Method
139
140
141  /**
142    * Get Method for class field 'requestProps_'.
143    *
144    * @return HashMap<String, String> - The value the class field 'requestProps_'.
145    *
146    **/
147  public HashMap<String, String> getRequestProps()
148  {
149    return requestProps_;
150  }  // getRequestProps Method
151
152
153  /**
154    * Set Method for class field 'loginFormID_'.
155    *
156    * @param loginFormID is the value to set this class field to.
157    *
158    **/
159  public  void setLoginFormID_(String loginFormID)
160  {
161    this.loginFormID_ = loginFormID;
162  }  // setLoginFormID Method
163
164
165  /**
166    * Get Method for class field 'loginFormID_'.
167    *
168    * @return String - The value the class field 'loginFormID_'.
169    *
170    **/
171  public String getLoginFormID()
172  {
173    return loginFormID_;
174  }  // getLoginFormID Method
175
176
177  /**
178    * Set Method for class field 'usernameFormElementName_'.
179    *
180    * @param usernameFormElementName_ is the value to set this class field to.
181    *
182    **/
183  public  void setUsernameFormElementName(String usernameFormElementName)
184  {
185    this.usernameFormElementName_ = usernameFormElementName;
186  }  // setUsernameFormElementName_ Method
187
188
189  /**
190    * Get Method for class field 'usernameFormElementName_'.
191    *
192    * @return String - The value the class field 'usernameFormElementName_'.
193    *
194    **/
195  public String getUsernameFormElementName()
196  {
197    return usernameFormElementName_;
198  }  // getUsernameFormElementName Method
199
200
201  /**
202    * Set Method for class field 'passwordFormElementName_'.
203    *
204    * @param passwordFormElementName_ is the value to set this class field to.
205    *
206    **/
207  public  void setPasswordFormElementName(String passwordFormElementName)
208  {
209    this.passwordFormElementName_ = passwordFormElementName;
210  }  // setPasswordFormElementName Method
211
212
213  /**
214    * Get Method for class field 'passwordFormElementName_'.
215    *
216    * @return String - The value the class field 'passwordFormElementName_'.
217    *
218    **/
219  public String getPasswordFormElementName()
220  {
221    return passwordFormElementName_;
222  }  // getPasswordFormElementName Method
223
224
225  /**
226    * Set Method for class field 'username_'.
227    *
228    * @param username_ is the value to set this class field to.
229    *
230    **/
231  public  void setUsername(String username)
232  {
233    this.username_ = username;
234  }  // setUsername Method
235
236
237  /**
238    * Get Method for class field 'username_'.
239    *
240    * @return String - The value the class field 'username_'.
241    *
242    **/
243  public String getUsername()
244  {
245    return username_;
246  }  // getUsername Method
247
248
249  /**
250    * Set Method for class field 'password_'.
251    *
252    * @param password_ is the value to set this class field to.
253    *
254    **/
255  public  void setPassword(String password)
256  {
257    this.password_ = password;
258  }  // setPassword Method
259
260
261  /**
262    * Get Method for class field 'password_'.
263    *
264    * @return String - The value the class field 'password_'.
265    *
266    **/
267  public String getPassword()
268  {
269    return password_;
270  }  // getPassword Method
271
272
273  /**
274    * Set Method for class field 'scrapePageUrl_'.
275    *
276    * @param scrapePageUrl_ is the value to set this class field to.
277    *
278    **/
279  public  void setScrapePageUrl(String scrapePageUrl)
280  {
281    this.scrapePageUrl_ = scrapePageUrl;
282  }  // setScrapePageUrl_ Method
283
284
285  /**
286    * Get Method for class field 'scrapePageUrl_'.
287    *
288    * @return String - The value the class field 'scrapePageUrl_'.
289    *
290    **/
291  public String getScrapePageUrl()
292  {
293    return scrapePageUrl_;
294  }  // getScrapePageUrl Method
295
296
297  /**
298    * Set Method for class field 'scrapeStart_'.
299    *
300    * @param scrapeStart_ is the value to set this class field to.
301    *
302    **/
303  public  void setScrapeStart(String scrapeStart)
304  {
305    this.scrapeStart_ = scrapeStart;
306  }  // setScrapeStart Method
307
308
309  /**
310    * Get Method for class field 'scrapeStart_'.
311    *
312    * @return String - The value the class field 'scrapeStart_'.
313    *
314    **/
315  public String getScrapeStart()
316  {
317    return scrapeStart_;
318  }  // getScrapeStart Method
319
320
321  /**
322    * Set Method for class field 'scrapeEnd_'.
323    *
324    * @param scrapeEnd_ is the value to set this class field to.
325    *
326    **/
327  public  void setScrapeEnd(String scrapeEnd)
328  {
329    this.scrapeEnd_ = scrapeEnd_;
330  }  // setScrapeEnd Method
331
332
333  /**
334    * Get Method for class field 'scrapeEnd_'.
335    *
336    * @return String - The value the class field 'scrapeEnd_'.
337    *
338    **/
339  public String getScrapeEnd()
340  {
341    return scrapeEnd_;
342  }  // getScrapeEnd Method
343
344
345  public List<String> getCookies()
346  {
347    return cookies_;
348  }
349
350
351  public void setCookies(List<String> cookies)
352  {
353    this.cookies_ = cookies;
354  }
355
356
357  /**
358    * Set Method for class field 'connection_'.
359    *
360    * @param connection_ is the value to set this class field to.
361    *
362    **/
363  public  void setConnection(HttpsURLConnection connection)
364  {
365    this.connection_ = connection;
366  }  // setConnection Method
367
368
369  /**
370    * Get Method for class field 'connection_'.
371    *
372    * @return HttpsURLConnection - The value the class field 'connection_'.
373    *
374    **/
375  public HttpsURLConnection getConnection()
376  {
377    return connection_;
378  }  // getConnection Method
379
380
381  /**
382    * Sends the POST to the login url  parameters from the classVars.
383    * It also caches the page response text into thew classVar loginPageResponse_.<br>
384    */
385  public  boolean  doLogin()
386  {
387    return this.doLogin(loginUrl_,requestProps_,loginFormID_,usernameFormElementName_,passwordFormElementName_,username_,password_);
388  }
389
390
391  /**
392    * Sends the POST to the login url with all required parameters.
393    * It also caches the page response text into thew classVar loginPageResponse_.<br>
394    * <br> example requestProps<pre>
395    *     HashMap <String, String> reqProps = new HashMap<String, String>();
396    *     reqProps.put("Accept","text/html,application/xhtml+xml,application/xml");
397    *     reqProps.put("Accept-Encoding ","gzip, deflate, br");
398    *     reqProps.put("Accept-Language ","en-US,en;q=0.5");
399    *     reqProps.put("Connection","keep-alive");
400    *     reqProps.put("Content-Type","application/x-www-form-urlencoded");
401    *     reqProps.put("Host","red.webarts.bc.ca");
402    *     //reqProps.put("Referer","red.webarts.bc.ca");
403    *     reqProps.put("Upgrade-Insecure-Requests","1");
404    *</pre><br><B>NOTE:</B> you don't need to add the "User-Agent" or "Content-Length"
405    *
406    * @param loginUrl
407    * @param requestProps
408    * @param formID
409    * @param usernameFormElementName
410    * @param passwordFormElementName
411    * @param username
412    * @param password
413    * @return boolean true if loggedIn
414    **/
415  public  boolean  doLogin(String loginUrl,
416                           HashMap <String, String> requestProps,
417                           String formID,
418                           String usernameFormElementName,
419                           String passwordFormElementName,
420                           String username,
421                           String password)
422  {
423    boolean retVal = false;
424    baseUrl_ = loginUrl.substring(0,loginUrl.lastIndexOf("/")-1);
425    String page = getPageContent(loginUrl);
426    String postParams = getLoginFormParams(page,
427                                                    formID ,
428                                                    usernameFormElementName, passwordFormElementName,
429                                                    username, password);
430
431    retVal = (sendPost(loginUrl, postParams, requestProps)==200?true:false);
432
433    return retVal;
434  }
435
436
437  public  String  doScrape(String scrapePageUrl,
438                           String scrapeStart,
439                           String scrapeEnd)
440  {
441    String result = getPageContent(scrapePageUrl);
442    String retVal = result.substring(result.indexOf(scrapeStart),
443                                       result.indexOf(scrapeEnd));
444    return retVal;
445  }
446
447
448  public  String  doScrape(String scrapePageUrl)
449  {
450    return getPageContent(scrapePageUrl);
451  }
452
453
454  public static void main(String[] args) throws Exception
455  {
456    Calendar rightNow = Calendar.getInstance();
457    //System.out.println("StockDate: "+rightNow_.toString());
458
459    String dateStr = String.valueOf(rightNow_.get(rightNow_.YEAR))+"-"+
460                     (rightNow_.get(rightNow_.MONTH)+1<10?"0"+String.valueOf(rightNow_.get(rightNow_.MONTH)+1):String.valueOf(rightNow_.get(rightNow_.MONTH)+1))+"-"+
461                     (rightNow_.get(rightNow_.DAY_OF_MONTH)<10?"0"+String.valueOf(rightNow_.get(rightNow_.DAY_OF_MONTH)):String.valueOf(rightNow_.get(rightNow_.DAY_OF_MONTH)));
462
463    String loginUrl = "https://red.webarts.bc.ca/owncloud/";
464    String scrapePageUrl = "https://red.webarts.bc.ca/owncloud/index.php/apps/files/";
465    String crLoginUrl = "https://trading.credentialdirect.com/login.aspx";
466    String crScrapePageUrl = "https://trading.credentialdirect.com/AccountInquiry/Holdings";
467
468    String ocLoginFormElement = "login";  /* id of the form element */
469    String ocUserLoginElement = "user"; /* the element name in the form to look for */
470    String ocPasswordElement = "password";/* the element name in the form to look for */
471    String ocUserLogin = "tgutwin";
472    String ocPassword = "18BoogieWoogie";
473
474    String crLoginFormElement = "loginForm";  /* id of the form element */
475    String crUserLoginElement = "UserID";
476    String crPasswordElement = "Password";
477    String crUserLogin = "tgutwin@webarts.ca";
478    String crPassword = "mZx:3)P3v6+H]^;";
479
480    String baseUrl = "/";
481
482    HttpUrlConnectionScrape instance = new HttpUrlConnectionScrape();
483    // make sure cookies are turn on
484    CookieHandler.setDefault(new CookieManager());
485
486    // 1. Send a "GET" request, so that you can extract the form's data.
487    String page = "";
488    String postParams = null;
489
490    if(scrapeSite_==SCRAPE_OC)
491    {
492      baseUrl = loginUrl.substring(0,loginUrl.lastIndexOf("/")-1);
493      page = instance.getPageContent(loginUrl);
494      postParams = instance.getLoginFormParams( page,
495                                              ocLoginFormElement ,
496                                              ocUserLoginElement,ocPasswordElement,
497                                              ocUserLogin, ocPassword);
498    }
499    else if(scrapeSite_==SCRAPE_CR)
500    {
501      baseUrl = loginUrl.substring(0,crLoginUrl.lastIndexOf("/")-1);
502      page = instance.getPageContent(crLoginUrl);
503      postParams = instance.getLoginFormParams( page,
504                                              crLoginFormElement ,
505                                              crUserLoginElement, crPasswordElement,
506                                              crUserLogin, crPassword);
507    }
508
509    // setup the RequestProperties to asign to the request
510    HashMap <String, String> reqProps = new HashMap<String, String>();
511    reqProps.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
512    reqProps.put("Accept-Encoding       ","gzip, deflate, br");
513    reqProps.put("Accept-Language       ","en-US,en;q=0.5");
514    reqProps.put("Connection","keep-alive");
515    //reqProps.put("Content-Length","179");
516    reqProps.put("Content-Type","application/x-www-form-urlencoded");
517    //reqProps.put("Cookie","oc7c8e64d137=bse6lf621fk1n5mtfjsfmteu27; oc_sessionPassphrase=fE0HrV0ix%2BU6iCfMXgmdgf16ZHHiw%2BVFVzRcQL%2BUlCPyxRalZUp%2BsawpzCcR9LNDhM9iFsZKP4gjKPlHQxgcO6n8MXpG6vmfx2FFCzbfq2b%2F93jRhmZqYTBmHcwaM%2F4H; __utma=225680793.380225408.1478298658.1478298658.1478304047.2; astroVersion.style=astroVersion.professionalBlue; _ga=GA1.3.380225408.1478298658");
518    if(scrapeSite_==SCRAPE_OC)
519    {
520      reqProps.put("Host","red.webarts.bc.ca");
521      //reqProps.put("Referer","red.webarts.bc.ca");
522    }
523    else if(scrapeSite_==SCRAPE_CR)
524    {
525      reqProps.put("Host","trading.credentialdirect.com");
526      //reqProps.put("Referer","https://trading.credentialdirect.com/login.aspx");
527    }
528    reqProps.put("Upgrade-Insecure-Requests","1");
529    //reqProps.put("User-Agent" ,"Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0");
530
531    // 3. Construct above post's content and then send a POST request for authentication/login
532    if(scrapeSite_==SCRAPE_OC)  instance.sendPost(loginUrl, postParams, reqProps);
533    else if(scrapeSite_==SCRAPE_CR)  instance.sendPost(crLoginUrl, postParams, reqProps);
534
535    // 4. success then go the page you want.
536    String result = "";
537
538    if(scrapeSite_==SCRAPE_OC) result = instance.getPageContent(scrapePageUrl);
539    else if(scrapeSite_==SCRAPE_CR) result = instance.getPageContent(crScrapePageUrl);
540
541    writeStringToFile(result,"pageContent-"+dateStr+".html");
542    //System.out.println(result);
543
544    String summaryTableHtml = result.substring(result.indexOf("{\"Data\":"),
545                                               result.indexOf(",\"Total\":1,\"AggregateResults\":null"));
546    System.out.println(summaryTableHtml);
547    writeStringToFile(summaryTableHtml,"crSummaryData-"+dateStr+".json");
548  }
549
550
551  /**
552    * Sends a POST request to the url, along with all the passed post parameters and request properties.
553    *
554    * @param url is the url to post this stuff to
555    * @param postParams is a sequential string of params that get sent in this post
556    * @return  the responseCode
557    **/
558  private int sendPost(String url, String postParams, HashMap <String, String> reqProps) throws Exception
559  {
560    if(debugOut_) System.out.println("\n sendPost to url="+url);
561    URL obj = new URL(url);
562    connection_ = (HttpsURLConnection)obj.openConnection();
563    // Act like a browser
564    connection_.setUseCaches(false);
565    connection_.setRequestMethod("POST");
566    connection_.setRequestProperty("User-Agent", USER_AGENT);
567
568    for (String key : reqProps.keySet())
569    {
570      if(   !"User-Agent".equals(key)
571         && !"Content-Length".equals(key)
572         && !"Cookie".equals(key)
573         && !"".equals(key.trim())
574         && !"".equals(((String)reqProps.get(key)).trim()) )
575      connection_.setRequestProperty(key.trim(), ((String)reqProps.get(key)).trim());
576
577      if(debugOut_) System.out.println("   setRequestProperty  "+key.trim()+" = "+((String)reqProps.get(key)).trim());
578
579    }
580    connection_.setRequestProperty("Content-Length", Integer.toString(postParams.length()));
581    for (String cookie : this.cookies_)
582    {
583      if(   !"".equals(cookie)
584         && !"".equals(((String)cookie.split(";", 1)[0]).trim()) )
585      {
586        connection_.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
587        if(debugOut_) System.out.println("   full  cookie = "+cookie);
588        if(debugOut_) System.out.println("   adding cookie = "+cookie.split(";", 1)[0]);
589      }
590    }
591    connection_.setDoOutput(true);
592    connection_.setDoInput(true);
593
594    // Send post request
595    DataOutputStream wr = new DataOutputStream(connection_.getOutputStream());
596    wr.writeBytes(postParams);
597    wr.flush();
598    wr.close();
599    int responseCode = connection_.getResponseCode();
600    System.out.println("\nSent 'POST' request to URL : " + url);
601    System.out.println("Post parameters : " + postParams);
602    System.out.println("Response Code : " + responseCode);
603    BufferedReader in = new BufferedReader(new InputStreamReader(connection_.getInputStream()));
604    String inputLine;
605    StringBuffer response = new StringBuffer();
606    while ((inputLine = in.readLine()) != null)
607    {
608      response.append(inputLine);
609    }
610    in.close();
611    loginPageResponse_=response.toString();
612    if(debugOut_) System.out.println("\n\n  LoginForm Response:  ");
613    if(debugOut_) System.out.println(response.toString());
614
615    return responseCode;
616  }
617
618
619  /** Connects/retrieves a URL; pulls its coockkies and returns the resulting htnl as a string.
620    **/
621  private String getPageContent(String url) throws Exception
622  {
623    if(debugOut_) System.out.println("\n GetPageContent from url="+url);
624    URL obj = new URL(url);
625    connection_ = (HttpsURLConnection)obj.openConnection();
626    // default is GET
627    connection_.setRequestMethod("GET");
628    connection_.setUseCaches(false);
629    // act like a browser
630    connection_.setRequestProperty("User-Agent", USER_AGENT);
631    connection_.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
632    connection_.setRequestProperty("Accept-Language", "en-US,en;q=0.5");
633    if (cookies_ != null)
634    {
635      for (String cookie : this.cookies_) {
636        connection_.addRequestProperty("Cookie", cookie.split(";", 1)[0]);
637      }
638    }
639
640    int responseCode = connection_.getResponseCode();
641    System.out.println("\nSending 'GET' request to URL : " + url);
642    System.out.println("Response Code : " + responseCode);
643    BufferedReader in = new BufferedReader(new InputStreamReader(connection_.getInputStream()));
644    String inputLine;
645    StringBuffer response = new StringBuffer();
646    while ((inputLine = in.readLine()) != null) {
647      response.append(inputLine);
648    }
649    in.close();
650    // Get the response cookies
651    setCookies(connection_.getHeaderFields().get("Set-Cookie"));
652    return response.toString();
653  }
654
655
656  /** Concatenates together the URL parameter string (ie. name=value&name2=val2&someOtherParamName=val3  )
657    * for a specific login form in the passed URL/html string. It also subsititutes the username and password for
658    * for the appropriate form elements. <br> This parameterString ends up getting sent to the POST request for a that form.
659    *
660    * @param html is the string representation of the URL that has the form to parse
661    * @param formID the form id (or name) to parse parameters from
662    * @param usernameFormElementName the form elementName used for the username field
663    * @param passwordFormElementNamethe form elementName used for the password field
664    * @param username the actual login username to use in the form
665    * @param password the actual login password to use in the form
666    **/
667  public String getLoginFormParams(String html,
668                                   String formID,
669                                   String usernameFormElementName,
670                                   String passwordFormElementName,
671                                   String username,
672                                   String password)  throws UnsupportedEncodingException
673  {
674    System.out.println("Extracting form's data...");
675    if(debugOut_) System.out.println("\n ---------------------------------------- \n"+html+"\n ---------------------------------------- \n");
676    Document doc = Jsoup.parse(html);
677
678    if(debugOut_) System.out.println("\n doc is parsed?="+(doc!=null));
679    if(debugOut_) System.out.println(" Looking for form id="+formID);
680    Element loginform = null;
681    loginform = doc.getElementById(formID);
682    if(loginform==null) loginform = doc.getElementsByAttributeValue("name", formID).first();
683
684    Elements inputElements = loginform.getElementsByTag("input");
685    List<String> paramList = new ArrayList<String>();
686    for (Element inputElement : inputElements)
687    {
688
689      String key = inputElement.attr("name");
690      String value = inputElement.attr("value");
691      if(debugOut_) System.out.println("     form element "+key+ " = "+value);
692      if (key.equals(usernameFormElementName))
693      {
694        value = username;
695      }
696      else if (key.equals(passwordFormElementName))
697      {
698        value = password;
699      }
700      else if (key.equals("timezone"))
701      {
702        value = "America/Los_Angeles";
703      }
704      else if (key.equals("timezone-offset"))
705      {
706        value = "-7";
707      }
708
709      paramList.add(key + "=" + URLEncoder.encode(value, "UTF-8"));
710    }
711    // build parameters list
712    StringBuilder result = new StringBuilder();
713    for (String param : paramList)
714    {
715      if (result.length() == 0)
716      {
717        result.append(param);
718      }
719      else
720      {
721        result.append("&" + param);
722      }
723    }
724    return result.toString();
725  }
726
727
728   /**
729   * Abstracts the writing of string to a file.
730   *
731   * @param s is the String to writeout
732   * @param fileName is the file name of the file to write the String into
733   * @return if success.. the full pathed filename is returned else null
734   **/
735  public static String writeStringToFile(String s, String fileName)
736  {return  writeStringToFile( s,  fileName, false);}
737
738
739  /**
740   * Abstracts the writing of string to a (zip) file (Zip NOT IMPLEMENTED YET).
741   *
742   * @param s is the String to writeout
743   * @param fileName is the file name of the file to write the String into
744   * @param zipCompress boolean fall to compress with zip compression
745   * @return if success.. the full pathed filename is returned else null
746   **/
747  public static String writeStringToFile(String s, String fileName, boolean zipCompress)
748  {
749    String retVal = fileName;
750
751    try
752    {
753      // FileWriter was not closing the stream
754      /*
755      FileWriter f = new FileWriter(fileName);
756      f.write(s);
757      f.flush();
758      f.close();
759      f = null;
760      */
761      FileOutputStream fos = new FileOutputStream(fileName);
762      byte[] strBytes = s.getBytes();
763      fos.write(strBytes);
764      fos.flush();
765      fos.close();
766      fos = null;
767      System.gc(); // this is required because a bug in Java won't realease
768    }
769    catch (IOException ioEx)
770    {
771      System.out.println("\nERROR Writing file: "+fileName);
772      retVal = null;
773    }
774
775    return retVal;
776  }
777
778
779
780}
781