[flaptor-util] r213 committed - Added xpath modifiers to deal with complex pages and broken html

[codesite...@google.com]

Revision: 213
Author: jhandl
Date: Tue Nov 9 10:46:47 2010
Log: Added xpath modifiers to deal with complex pages and broken html
http://code.google.com/p/flaptor-util/source/detail?r=213

Modified:
/trunk/src/com/flaptor/util/parser/HtmlParser.java
/trunk/src/com/flaptor/util/parser/ParseOutput.java
/trunk/src/com/flaptor/util/remote/WebServer.java

=======================================
--- /trunk/src/com/flaptor/util/parser/HtmlParser.java Mon Aug 31 11:54:51
2009
+++ /trunk/src/com/flaptor/util/parser/HtmlParser.java Tue Nov 9 10:46:47
2010
@@ -33,6 +33,7 @@
import org.dom4j.dom.DOMDocumentFactory;
import org.dom4j.io.DOMReader;
import org.dom4j.tree.DefaultAttribute;
+import org.dom4j.tree.DefaultElement;

import com.flaptor.util.Execute;
import com.flaptor.util.FileUtil;
@@ -43,6 +44,7 @@
import org.dom4j.Namespace;
import org.dom4j.QName;

+
/**
* This class implements a parser for html documents.
* @author Flaptor Development Team
@@ -161,7 +163,7 @@
replaceSeparatorTags(htmlDoc);

// extract the text from the html tags
- extractText(htmlDoc.getRootElement(), out, ParseOutput.CONTENT);
+ extractAllText(htmlDoc.getRootElement(), out, ParseOutput.CONTENT);

// extract special fields
extractFields(htmlDoc,out);
@@ -274,23 +276,46 @@
logger.debug("Exception occurred, ignoring links
in "+out.getUrl(), e);
}
}
+
+ private void extractText(Object obj, ParseOutput out, String field) {
+ logger.debug(" item of class " + obj.getClass().getName());
+ if (obj instanceof DefaultAttribute) {
+ DefaultAttribute attr = (DefaultAttribute) obj;
+ out.addFieldString(field,attr.getValue());
+ } else if (obj instanceof Element || obj instanceof
DefaultElement) {
+ Element el = (Element)obj;
+ extractAllText(el,out,field);
+ } else {
+ logger.debug("selected node of unknown type (" +
obj.getClass().getName() + ")");
+ }
+ }

private void extractFields(Document htmlDoc, ParseOutput out) {
+//System.out.println("XDOM: "+com.flaptor.util.DomUtil.domToString(htmlDoc));
for (String field: fieldDefinitions.keySet()) {
String xpath = fieldDefinitions.get(field);
List elements = htmlDoc.selectNodes(xpath);
- logger.debug("found " + elements.size() + " elements for " +
xpath);
- for ( Iterator iter = elements.iterator(); iter.hasNext();) {
- Object next = iter.next();
- if (next instanceof DefaultAttribute) {
- DefaultAttribute attr = (DefaultAttribute) next;
- out.addFieldString(field,attr.getValue());
- } else if ( next instanceof Element) {
- Element el = (Element)next;
- extractText(el,out,field);
- } else {
- logger.debug("xpath " + xpath + " selected some nodes
of unknown type (" + next.getClass().getName() + " )");
- }
+ if (field.endsWith(".count")) {
+ out.addFieldString(field.substring(0,field.length()-6),
String.valueOf(elements.size()));
+ } else if (field.matches(".*\\.[1-9][0-9]*$")) {
+ int p = field.lastIndexOf(".");
+ int n = 1, sel = Integer.parseInt(field.substring(p+1));
+ logger.debug("found " + elements.size() + " elements for "
+ xpath+", will select #"+sel);
+ Object obj = null;
+ for (Iterator iter = elements.iterator(); iter.hasNext()
&& n <= sel; n++) {
+ obj = iter.next();
+ logger.debug("... elem #"+n+": "+obj);
+ }
+ if (obj != null) {
+ logger.debug("... Selecting obj: "+obj+" for
field "+field.substring(0,p));
+ extractText(obj,out,field.substring(0,p));
+ }
+ } else {
+ logger.debug("found " + elements.size() + " elements for "
+ xpath);
+ for (Iterator iter = elements.iterator(); iter.hasNext();)
{
+ Object obj = iter.next();
+ extractText(obj,out,field);
+ }
}

}
@@ -348,14 +373,14 @@
* should be empty. After return, it contains the readable
text
* of the html and the outlinks.
*/
- protected void extractText(final Element e, final ParseOutput out,
final String fieldName) {
+ protected void extractAllText(final Element e, final ParseOutput out,
final String fieldName) {
//String nodeName = e.getName();
if (!(e.getNodeType() == Node.COMMENT_NODE)) {
int size = e.nodeCount();
for (int i = 0; i < size; i++) {
Node node = e.node(i);
if (node instanceof Element) {
- extractText((Element) node, out,fieldName);
+ extractAllText((Element)node, out, fieldName);
} else if (node instanceof Text) {
String t = node.getText();
out.addFieldString(fieldName,t);
@@ -373,18 +398,30 @@
}

public static void main(String[] arg) throws Exception {
- HtmlParser parser = new HtmlParser();
//parser.test(arg[0],arg[1]);
+ if (arg.length < 3) {
+ System.out.println("HtmlParser <file_name> <var_name>
<xpath>");
+ System.exit(0);
+ }
String str = FileUtil.readFile(new File(arg[0]));
+ String var = arg[1];
+ String xpath = arg[2];
+ Map<String,String> fields = new HashMap<String,String>();
+ fields.put(var,xpath);
+ HtmlParser parser = new HtmlParser(null,null,fields);
String url = "http://url.com";
if (arg.length > 1) { url = arg[1]; }
ParseOutput out = parser.parse(url, str.getBytes("UTF-8"));
System.out.println("-------------------------------------------");
- System.out.println("TITLE: "+out.getTitle());
+/*
for (Pair<String,String> link : out.getLinks()) {
System.out.println("LINK: "+link.first()+"
("+link.last()+")");
}
- System.out.println("CONTENT: "+out.getText());
+*/
+ int p = var.indexOf(".");
+ if (p >= 0) var = var.substring(0,p);
+ System.out.println(var+": "+out.getField(var));
+// System.out.println("CONTENT: "+out.getText());
}

}
=======================================
--- /trunk/src/com/flaptor/util/parser/ParseOutput.java Tue Nov 10 15:03:38
2009
+++ /trunk/src/com/flaptor/util/parser/ParseOutput.java Tue Nov 9 10:46:47
2010
@@ -6,34 +6,18 @@
package com.flaptor.util.parser;

-import java.io.File;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.logging.Level;
-import java.util.regex.Pattern;

import org.apache.log4j.Logger;
-import org.cyberneko.html.parsers.DOMParser;
-import org.dom4j.Attribute;
-import org.dom4j.Document;
-import org.dom4j.Element;
-import org.dom4j.Node;
-import org.dom4j.Text;
-import org.dom4j.dom.DOMDocumentFactory;
-import org.dom4j.io.DOMReader;
-import org.dom4j.tree.DefaultAttribute;

import com.flaptor.util.Execute;
-import com.flaptor.util.FileUtil;
import com.flaptor.util.Pair;

-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ArrayBlockingQueue;

/**
* The result of the parser is stored in an object of this class.
@@ -175,6 +159,8 @@
base = base.replace(" ", "%20");
try {
// Re-encode the query part, to handle partially encoded urls.
+ query = query.replaceAll("%([^0-9])","%25$1");
+ query = query.replaceAll("%$","%25");
query =
java.net.URLEncoder.encode(java.net.URLDecoder.decode(query,"UTF-8"),"UTF-8");
query = query.replace("%3D","=").replace("%26","&");
} catch (java.io.UnsupportedEncodingException e) {
=======================================
--- /trunk/src/com/flaptor/util/remote/WebServer.java Fri Aug 15 13:11:20
2008
+++ /trunk/src/com/flaptor/util/remote/WebServer.java Tue Nov 9 10:46:47
2010
@@ -58,22 +58,34 @@
*/
public static void main(final String[] args) {
if (args.length != 2) {
- System.err.println("usage: WebServer [handler class] [port]");
+ System.err.println("\n Usage: WebServer <handler class> <port>\n
or: WebServer <webapp dir> <port>\n");
System.exit(-1);
}

Execute.configureLog4j();

- Object handler = null;
- try {
- handler = Class.forName(args[0]).newInstance();
- } catch (Exception e) {
- System.err.println(e);
+ try {
+
+ WebServer server = new WebServer(Integer.parseInt(args[1]));
+ try {
+ Object handler = Class.forName(args[0]).newInstance();
+ server.addHandler("/", (AbstractHandler)handler);
+ } catch (ClassNotFoundException e1) {
+ URL webappPath =
Thread.currentThread().getContextClassLoader().getResource(args[0]);
+ if (null == webappPath) {
+ System.err.println("Can't find webapp directory
["+args[0]+"] in the classpath.");
+ System.exit(0);
+ }
+ String path = webappPath.getPath();
+ server.addWebAppHandler("/", path);
+ }
+
+ server.start();
+
+ } catch (Exception e2) {
+ logger.error("Instantiating WebServer: ",e2);
System.exit(-1);
}
- WebServer server = new WebServer(Integer.parseInt(args[1]));
- server.addHandler("/", (AbstractHandler)handler);
- server.start();
}

/**
@@ -197,6 +209,8 @@
throw new RuntimeException (e);
}
}
+
+

}