001package org.jsoup.examples;
002
003import org.jsoup.Jsoup;
004import org.jsoup.helper.StringUtil;
005import org.jsoup.helper.Validate;
006import org.jsoup.nodes.Document;
007import org.jsoup.nodes.Element;
008import org.jsoup.nodes.Node;
009import org.jsoup.nodes.TextNode;
010import org.jsoup.select.Elements;
011import org.jsoup.select.NodeTraversor;
012import org.jsoup.select.NodeVisitor;
013
014import java.io.IOException;
015
016/**
017 * HTML to plain-text. This example program demonstrates the use of jsoup to convert HTML input to lightly-formatted
018 * plain-text. That is divergent from the general goal of jsoup's .text() methods, which is to get clean data from a
019 * scrape.
020 * <p>
021 * Note that this is a fairly simplistic formatter -- for real world use you'll want to embrace and extend.
022 * </p>
023 * <p>
024 * To invoke from the command line, assuming you've downloaded the jsoup jar to your current directory:</p>
025 * <p><code>java -cp jsoup.jar org.jsoup.examples.HtmlToPlainText url [selector]</code></p>
026 * where <i>url</i> is the URL to fetch, and <i>selector</i> is an optional CSS selector.
027 * 
028 * @author Jonathan Hedley, jonathan@hedley.net
029 */
030public class HtmlToPlainText {
031    private static final String userAgent = "Mozilla/5.0 (jsoup)";
032    private static final int timeout = 5 * 1000;
033
034    public static void main(String... args) throws IOException {
035        Validate.isTrue(args.length == 1 || args.length == 2, "usage: java -cp jsoup.jar org.jsoup.examples.HtmlToPlainText url [selector]");
036        final String url = args[0];
037        final String selector = args.length == 2 ? args[1] : null;
038
039        // fetch the specified URL and parse to a HTML DOM
040        Document doc = Jsoup.connect(url).userAgent(userAgent).timeout(timeout).get();
041
042        HtmlToPlainText formatter = new HtmlToPlainText();
043
044        if (selector != null) {
045            Elements elements = doc.select(selector); // get each element that matches the CSS selector
046            for (Element element : elements) {
047                String plainText = formatter.getPlainText(element); // format that element to plain text
048                System.out.println(plainText);
049            }
050        } else { // format the whole doc
051            String plainText = formatter.getPlainText(doc);
052            System.out.println(plainText);
053        }
054    }
055
056    /**
057     * Format an Element to plain-text
058     * @param element the root element to format
059     * @return formatted text
060     */
061    public String getPlainText(Element element) {
062        FormattingVisitor formatter = new FormattingVisitor();
063        NodeTraversor.traverse(formatter, element); // walk the DOM, and call .head() and .tail() for each node
064
065        return formatter.toString();
066    }
067
068    // the formatting rules, implemented in a breadth-first DOM traverse
069    private class FormattingVisitor implements NodeVisitor {
070        private static final int maxWidth = 80;
071        private int width = 0;
072        private StringBuilder accum = new StringBuilder(); // holds the accumulated text
073
074        // hit when the node is first seen
075        public void head(Node node, int depth) {
076            String name = node.nodeName();
077            if (node instanceof TextNode)
078                append(((TextNode) node).text()); // TextNodes carry all user-readable text in the DOM.
079            else if (name.equals("li"))
080                append("\n * ");
081            else if (name.equals("dt"))
082                append("  ");
083            else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5", "tr"))
084                append("\n");
085        }
086
087        // hit when all of the node's children (if any) have been visited
088        public void tail(Node node, int depth) {
089            String name = node.nodeName();
090            if (StringUtil.in(name, "br", "dd", "dt", "p", "h1", "h2", "h3", "h4", "h5"))
091                append("\n");
092            else if (name.equals("a"))
093                append(String.format(" <%s>", node.absUrl("href")));
094        }
095
096        // appends text to the string builder with a simple word wrap method
097        private void append(String text) {
098            if (text.startsWith("\n"))
099                width = 0; // reset counter if starts with a newline. only from formats above, not in natural text
100            if (text.equals(" ") &&
101                    (accum.length() == 0 || StringUtil.in(accum.substring(accum.length() - 1), " ", "\n")))
102                return; // don't accumulate long runs of empty spaces
103
104            if (text.length() + width > maxWidth) { // won't fit, needs to wrap
105                String words[] = text.split("\\s+");
106                for (int i = 0; i < words.length; i++) {
107                    String word = words[i];
108                    boolean last = i == words.length - 1;
109                    if (!last) // insert a space if not the last word
110                        word = word + " ";
111                    if (word.length() + width > maxWidth) { // wrap and reset counter
112                        accum.append("\n").append(word);
113                        width = word.length();
114                    } else {
115                        accum.append(word);
116                        width += word.length();
117                    }
118                }
119            } else { // fits as is, without need to wrap text
120                accum.append(text);
121                width += text.length();
122            }
123        }
124
125        @Override
126        public String toString() {
127            return accum.toString();
128        }
129    }
130}