001package org.jsoup.examples; 002 003import org.jsoup.Jsoup; 004import org.jsoup.helper.StringUtil; 005import org.jsoup.helper.Validate; 006import org.jsoup.nodes.Document; 007import org.jsoup.nodes.Element; 008import org.jsoup.nodes.Node; 009import org.jsoup.nodes.TextNode; 010import org.jsoup.select.Elements; 011import org.jsoup.select.NodeTraversor; 012import org.jsoup.select.NodeVisitor; 013 014import java.io.IOException; 015 016/** 017 * HTML to plain-text. This example program demonstrates the use of jsoup to convert HTML input to lightly-formatted 018 * plain-text. That is divergent from the general goal of jsoup's .text() methods, which is to get clean data from a 019 * scrape. 020 * <p> 021 * Note that this is a fairly simplistic formatter -- for real world use you'll want to embrace and extend. 022 * </p> 023 * <p> 024 * To invoke from the command line, assuming you've downloaded the jsoup jar to your current directory:</p> 025 * <p><code>java -cp jsoup.jar org.jsoup.examples.HtmlToPlainText url [selector]</code></p> 026 * where <i>url</i> is the URL to fetch, and <i>selector</i> is an optional CSS selector. 027 * 028 * @author Jonathan Hedley, jonathan@hedley.net 029 */ 030public class HtmlToPlainText { 031 private static final String userAgent = "Mozilla/5.0 (jsoup)"; 032 private static final int timeout = 5 * 1000; 033 034 public static void main(String... args) throws IOException { 035 Validate.isTrue(args.length == 1 || args.length == 2, "usage: java -cp jsoup.jar org.jsoup.examples.HtmlToPlainText url [selector]"); 036 final String url = args[0]; 037 final String selector = args.length == 2 ? args[1] : null; 038 039 // fetch the specified URL and parse to a HTML DOM 040 Document doc = Jsoup.connect(url).userAgent(userAgent).timeout(timeout).get(); 041 042 HtmlToPlainText formatter = new HtmlToPlainText(); 043 044 if (selector != null) { 045 Elements elements = doc.select(selector); // get each element that matches the CSS selector 046 for (Element element : elements) { 047 String plainText = formatter.getPlainText(element); // format that element to plain text 048 System.out.println(plainText); 049 } 050 } else { // format the whole doc 051 String plainText = formatter.getPlainText(doc); 052 System.out.println(plainText); 053 } 054 } 055 056 /** 057 * Format an Element to plain-text 058 * @param element the root element to format 059 * @return formatted text 060 */ 061 public String getPlainText(Element element) { 062 FormattingVisitor formatter = new FormattingVisitor(); 063 NodeTraversor.traverse(formatter, element); // walk the DOM, and call .head() and .tail() for each node 064 065 return formatter.toString(); 066 } 067 068 // the formatting rules, implemented in a breadth-first DOM traverse 069 private class FormattingVisitor implements NodeVisitor { 070 private static final int maxWidth = 80; 071 private int width = 0; 072 private StringBuilder accum = new StringBuilder(); // holds the accumulated text 073 074 // hit when the node is first seen 075 public void head(Node node, int depth) { 076 String name = node.nodeName(); 077 if (node instanceof TextNode) 078 append(((TextNode) node).text()); // TextNodes carry all user-readable text in the DOM. 079 else if (name.equals("li")) 080 append("\n * "); 081 else if (name.equals("dt")) 082 append(" "); 083 else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5", "tr")) 084 append("\n"); 085 } 086 087 // hit when all of the node's children (if any) have been visited 088 public void tail(Node node, int depth) { 089 String name = node.nodeName(); 090 if (StringUtil.in(name, "br", "dd", "dt", "p", "h1", "h2", "h3", "h4", "h5")) 091 append("\n"); 092 else if (name.equals("a")) 093 append(String.format(" <%s>", node.absUrl("href"))); 094 } 095 096 // appends text to the string builder with a simple word wrap method 097 private void append(String text) { 098 if (text.startsWith("\n")) 099 width = 0; // reset counter if starts with a newline. only from formats above, not in natural text 100 if (text.equals(" ") && 101 (accum.length() == 0 || StringUtil.in(accum.substring(accum.length() - 1), " ", "\n"))) 102 return; // don't accumulate long runs of empty spaces 103 104 if (text.length() + width > maxWidth) { // won't fit, needs to wrap 105 String words[] = text.split("\\s+"); 106 for (int i = 0; i < words.length; i++) { 107 String word = words[i]; 108 boolean last = i == words.length - 1; 109 if (!last) // insert a space if not the last word 110 word = word + " "; 111 if (word.length() + width > maxWidth) { // wrap and reset counter 112 accum.append("\n").append(word); 113 width = word.length(); 114 } else { 115 accum.append(word); 116 width += word.length(); 117 } 118 } 119 } else { // fits as is, without need to wrap text 120 accum.append(text); 121 width += text.length(); 122 } 123 } 124 125 @Override 126 public String toString() { 127 return accum.toString(); 128 } 129 } 130}