Index: src/main/java/crawlercommons/sitemaps/SiteMapParser.java
===================================================================
--- src/main/java/crawlercommons/sitemaps/SiteMapParser.java	(revision 0)
+++ src/main/java/crawlercommons/sitemaps/SiteMapParser.java	(revision 0)
@@ -0,0 +1,555 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Date;
+import java.util.zip.GZIPInputStream;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import crawlercommons.sitemaps.SiteMap.SitemapType;
+
+public class SiteMapParser {
+    public static final Log LOG = LogFactory.getLog(SiteMapParser.class);
+
+    /** According to the specs, 50K URLs per Sitemap is the max */
+    private static final int MAX_URLS = 50000;
+
+    /** Sitemap docs must be limited to 10MB (10,485,760 bytes) */
+    private static int MAX_BYTES_ALLOWED = 10485760;
+
+    public SiteMapParser() {
+    }
+
+    /**
+     * Returned a processed copy of an unprocessed sitemap object, i.e. transfer the value of 
+     * getLastModified.
+     **/  
+    public SiteMap parseSiteMap(String contentType, byte[] content, SiteMap sitemap) throws UnknownFormatException, IOException {
+        AbstractSiteMap asmp = parseSiteMap(contentType, content, sitemap.getUrl());
+        SiteMap smap2 = (SiteMap)asmp;
+        smap2.setLastModified(sitemap.getLastModified());
+        return smap2;
+    }
+    
+    /**
+     * Returns a SiteMap or SiteMapIndex given a content type, byte content and
+     * the URL of a sitemap
+     **/
+    public AbstractSiteMap parseSiteMap(String contentType, byte[] content, URL url) throws UnknownFormatException, IOException {
+
+        // Use extension or MIME type to determine how we should try
+        // to process the response
+        if (url.getPath().endsWith(".xml") || contentType.contains("text/xml") || contentType.contains("application/xml") || contentType.contains("application/x-xml")
+                        || contentType.contains("application/atom+xml") || contentType.contains("application/rss+xml")) {
+
+            // Try parsing the XML which could be in a number of formats
+            return processXml(url, new String(content));
+        } else if (contentType.contains("text/plain")) {
+            // plain text
+            return (AbstractSiteMap) processText(new String(content), url.toString());
+        } else if (url.getPath().endsWith(".gz") || contentType.contains("application/gzip") || contentType.contains("application/x-gzip") || contentType.contains("application/x-gunzip")
+                        || contentType.contains("application/gzipped") || contentType.contains("application/gzip-compressed") || contentType.contains("application/x-compress")
+                        || contentType.contains("gzip/document") || contentType.contains("application/octet-stream")) {
+            return processGzip(url, content);
+        }
+        throw new UnknownFormatException("Unknown format " + contentType + " at " + url);
+    }
+
+    /**
+     * Parse the given XML content.
+     * 
+     * @param sitemapUrl
+     * @param xmlContent
+     * @return
+     * @throws UnknownFormatException
+     */
+    private AbstractSiteMap processXml(URL sitemapUrl, String xmlContent) throws UnknownFormatException {
+
+        InputSource is = new InputSource();
+        is.setCharacterStream(new StringReader(xmlContent));
+        return processXml(sitemapUrl, is);
+    }
+
+    /**
+     * Process a text-based Sitemap. Text sitemaps only list URLs but no
+     * priorities, last mods, etc.
+     * 
+     * @param content
+     * @throws IOException
+     */
+    private SiteMap processText(String content, String sitemapUrl) throws IOException {
+
+        LOG.debug("Processing textual Sitemap");
+
+        SiteMap textSiteMap = new SiteMap(sitemapUrl);
+        textSiteMap.setType(SitemapType.TEXT);
+
+        BufferedReader reader = new BufferedReader(new StringReader(content));
+
+        String line;
+
+        int i = 1;
+        while ((line = reader.readLine()) != null) {
+            if (line.length() > 0 && i <= MAX_URLS) {
+                try {
+                    URL url = new URL(line);
+                    if (urlIsLegal(textSiteMap.getBaseUrl(), url.toString())) {
+                        LOG.debug("  " + i + ". " + url);
+                        i++;
+                        SiteMapURL surl = new SiteMapURL(url);
+                        textSiteMap.addSiteMapUrl(surl);
+                    }
+                } catch (MalformedURLException e) {
+                    LOG.debug("Bad URL [" + line + "].");
+                }
+            }
+        }
+        textSiteMap.setProcessed(true);
+        return textSiteMap;
+    }
+
+    /**
+     * Decompress the gzipped content and process the resulting XML Sitemap.
+     * 
+     * @param url
+     *            - URL of the gzipped content
+     * @param response
+     *            - Gzipped content
+     * @throws MalformedURLException
+     * @throws IOException
+     * @throws UnknownFormatException
+     */
+    private AbstractSiteMap processGzip(URL url, byte[] response) throws MalformedURLException, IOException, UnknownFormatException {
+
+        LOG.debug("Processing gzip");
+
+        AbstractSiteMap smi;
+
+        InputStream is = new ByteArrayInputStream(response);
+
+        // Remove .gz ending
+        String xmlUrl = url.toString().replaceFirst("\\.gz$", "");
+
+        LOG.debug("XML url = " + xmlUrl);
+
+        InputStream decompressed = new GZIPInputStream(is);
+        InputSource in = new InputSource(decompressed);
+        in.setSystemId(xmlUrl);
+        smi = processXml(url, in);
+        decompressed.close();
+        return smi;
+    }
+
+    /**
+     * Parse the given XML content.
+     * 
+     * @param sitemapUrl
+     * @param is
+     * @throws UnknownFormatException
+     */
+    private AbstractSiteMap processXml(URL sitemapUrl, InputSource is) throws UnknownFormatException {
+
+        Document doc = null;
+
+        try {
+            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+            doc = dbf.newDocumentBuilder().parse(is);
+        } catch (Exception e) {
+            throw new UnknownFormatException("Error parsing XML for " + sitemapUrl);
+        }
+
+        // See if this is a sitemap index
+        NodeList nodeList = doc.getElementsByTagName("sitemapindex");
+        if (nodeList.getLength() > 0) {
+            nodeList = doc.getElementsByTagName("sitemap");
+            return parseSitemapIndex(sitemapUrl, nodeList);
+        } else if (doc.getElementsByTagName("urlset").getLength() > 0) {
+            // This is a regular Sitemap
+            return parseXmlSitemap(sitemapUrl, doc);
+        } else if (doc.getElementsByTagName("link").getLength() > 0) {
+            // Could be RSS or Atom
+            return parseSyndicationFormat(sitemapUrl, doc);
+        }
+        throw new UnknownFormatException("Unknown XML format for " + sitemapUrl);
+    }
+
+    /**
+     * Parse XML that contains a valid Sitemap. Example of a Sitemap: <?xml
+     * version="1.0" encoding="UTF-8"?> <urlset
+     * xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <url>
+     * <loc>http://www.example.com/</loc> <lastmod>2005-01-01</lastmod>
+     * <changefreq>monthly</changefreq> <priority>0.8</priority> </url> <url>
+     * <loc
+     * >http://www.example.com/catalog?item=12&amp;desc=vacation_hawaii</loc>
+     * <changefreq>weekly</changefreq> </url> </urlset>
+     * 
+     * @param doc
+     */
+    private SiteMap parseXmlSitemap(URL sitemapUrl, Document doc) {
+
+        SiteMap sitemap = new SiteMap(sitemapUrl);
+        sitemap.setType(SitemapType.XML);
+
+        NodeList list = doc.getElementsByTagName("url");
+
+        // Loop through the <url>s
+        for (int i = 0; i < list.getLength(); i++) {
+
+            Node n = list.item(i);
+
+            if (n.getNodeType() == Node.ELEMENT_NODE) {
+                Element elem = (Element) n;
+
+                String loc = getElementValue(elem, "loc");
+
+                URL url = null;
+                try {
+                    url = new URL(loc);
+                    String lastMod = getElementValue(elem, "lastmod");
+                    String changeFreq = getElementValue(elem, "changefreq");
+                    String priority = getElementValue(elem, "priority");
+
+                    if (urlIsLegal(sitemap.getBaseUrl(), url.toString())) {
+                        SiteMapURL sUrl = new SiteMapURL(url.toString(), lastMod, changeFreq, priority);
+                        sitemap.addSiteMapUrl(sUrl);
+                        LOG.debug("  " + (i + 1) + ". " + sUrl);
+                    }
+                } catch (MalformedURLException e) {
+                    // e.printStackTrace();
+
+                    // Can't create an entry with a bad URL
+                    LOG.debug("Bad url: [" + loc + "]");
+                }
+            }
+        }
+        sitemap.setProcessed(true);
+        return sitemap;
+    }
+
+    /**
+     * Parse XML that contains a Sitemap Index. Example Sitemap Index:
+     * 
+     * <?xml version="1.0" encoding="UTF-8"?> <sitemapindex
+     * xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <sitemap>
+     * <loc>http://www.example.com/sitemap1.xml.gz</loc>
+     * <lastmod>2004-10-01T18:23:17+00:00</lastmod> </sitemap> <sitemap>
+     * <loc>http://www.example.com/sitemap2.xml.gz</loc>
+     * <lastmod>2005-01-01</lastmod> </sitemap> </sitemapindex>
+     * 
+     * @param url
+     *            - URL of Sitemap Index
+     * @param nodeList
+     */
+    private SiteMapIndex parseSitemapIndex(URL url, NodeList nodeList) {
+
+        LOG.debug("Parsing Sitemap Index");
+
+        SiteMapIndex sitemapIndex = new SiteMapIndex(url);
+
+        // Loop through the <sitemap>s
+        for (int i = 0; i < nodeList.getLength() && i < MAX_URLS; i++) {
+
+            Node firstNode = nodeList.item(i);
+
+            URL sitemapUrl = null;
+            Date lastModified = null;
+
+            if (firstNode.getNodeType() == Node.ELEMENT_NODE) {
+                Element elem = (Element) firstNode;
+                String loc = getElementValue(elem, "loc");
+
+                try {
+                    sitemapUrl = new URL(loc);
+                    String lastmod = getElementValue(elem, "lastmod");
+                    lastModified = SiteMap.convertToDate(lastmod);
+
+                    // Right now we are not worried about sitemapUrls that point
+                    // to different websites.
+
+                    SiteMap s = new SiteMap(sitemapUrl, lastModified);
+                    sitemapIndex.addSitemap(s);
+                    LOG.debug("  " + (i + 1) + ". " + s);
+                } catch (MalformedURLException e) {
+                    // e.printStackTrace();
+
+                    // Don't create an entry for a bad URL
+                    LOG.debug("Bad url: [" + loc + "]");
+                }
+            }
+        }
+        return sitemapIndex;
+    }
+
+    /**
+     * Parse the XML document, looking for "feed" element to determine if it's
+     * an Atom doc and "rss" to determine if it's an RSS doc.
+     * 
+     * @param sitemapUrl
+     * @param doc
+     *            - XML document to parse
+     * @throws UnknownFormatException
+     *             if XML does not appear to be Arom or RSS
+     */
+    private SiteMap parseSyndicationFormat(URL sitemapUrl, Document doc) throws UnknownFormatException {
+
+        SiteMap sitemap = new SiteMap(sitemapUrl);
+
+        // See if this is an Atom feed by looking for "feed" element
+        NodeList list = doc.getElementsByTagName("feed");
+        if (list.getLength() > 0) {
+            parseAtom(sitemap, (Element) list.item(0), doc);
+            sitemap.setProcessed(true);
+            return sitemap;
+        } else {
+            // See if RSS feed by looking for "rss" element
+            list = doc.getElementsByTagName("rss");
+            if (list.getLength() > 0) {
+                parseRSS(sitemap, doc);
+                sitemap.setProcessed(true);
+                return sitemap;
+            } else {
+                throw new UnknownFormatException("Unknown syndication format at " + sitemapUrl);
+            }
+        }
+    }
+
+    /**
+     * Parse the XML document which is assumed to be in Atom format. Atom 1.0
+     * example:
+     * 
+     * <?xml version="1.0" encoding="utf-8"?> <feed
+     * xmlns="http://www.w3.org/2005/Atom">
+     * 
+     * <title>Example Feed</title> <subtitle>A subtitle.</subtitle> <link
+     * href="http://example.org/feed/" rel="self"/> <link
+     * href="http://example.org/"/> <modified>2003-12-13T18:30:02Z</modified>
+     * <author> <name>John Doe</name> <email>johndoe@example.com</email>
+     * </author> <id>urn:uuid:60a76c80-d399-11d9-b91C-0003939e0af6</id>
+     * 
+     * <entry> <title>Atom-Powered Robots Run Amok</title> <link
+     * href="http://example.org/2003/12/13/atom03"/>
+     * <id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
+     * <updated>2003-12-13T18:30:02Z</updated> <summary>Some text.</summary>
+     * </entry>
+     * 
+     * </feed>
+     * 
+     * @param elem
+     * @param doc
+     */
+    private void parseAtom(SiteMap sitemap, Element elem, Document doc) {
+
+        // Grab items from <feed><entry><link href="URL" /></entry></feed>
+        // Use lastmod date from <feed><modified>DATE</modified></feed>
+
+        LOG.debug("Parsing Atom XML");
+
+        sitemap.setType(SitemapType.ATOM);
+
+        String lastMod = getElementValue(elem, "modified");
+        LOG.debug("lastMod=" + lastMod);
+
+        NodeList list = doc.getElementsByTagName("entry");
+
+        // Loop through the <entry>s
+        for (int i = 0; i < list.getLength() && i < MAX_URLS; i++) {
+
+            Node n = list.item(i);
+
+            if (n.getNodeType() == Node.ELEMENT_NODE) {
+                elem = (Element) n;
+
+                String href = getElementAttributeValue(elem, "link", "href");
+                LOG.debug("href=" + href);
+
+                URL url = null;
+                try {
+                    url = new URL(href);
+
+                    if (urlIsLegal(sitemap.getBaseUrl(), url.toString())) {
+                        SiteMapURL sUrl = new SiteMapURL(url.toString(), lastMod, null, null);
+                        sitemap.addSiteMapUrl(sUrl);
+                        LOG.debug("  " + (i + 1) + ". " + sUrl);
+                    }
+                } catch (MalformedURLException e) {
+                    // Can't create an entry with a bad URL
+                    LOG.debug("Bad url: [" + href + "]");
+                }
+
+            }
+        }
+    }
+
+    /**
+     * Parse XML document which is assumed to be in RSS format. RSS 2.0 example:
+     * 
+     * <?xml version="1.0"?> <rss version="2.0"> <channel> <title>Lift Off
+     * News</title> <link>http://liftoff.msfc.nasa.gov/</link>
+     * <description>Liftoff to Space Exploration.</description>
+     * <language>en-us</language> <pubDate>Tue, 10 Jun 2003 04:00:00
+     * GMT</pubDate> <lastBuildDate>Tue, 10 Jun 2003 09:41:01
+     * GMT</lastBuildDate> <docs>http://blogs.law.harvard.edu/tech/rss</docs>
+     * <generator>Weblog Editor 2.0</generator>
+     * <managingEditor>editor@example.com</managingEditor>
+     * <webMaster>webmaster@example.com</webMaster> <ttl>5</ttl>
+     * 
+     * <item> <title>Star City</title>
+     * <link>http://liftoff.msfc.nasa.gov/news/2003/news-starcity.asp</link>
+     * <description>How do Americans get ready to work with Russians aboard the
+     * International Space Station? They take a crash course in culture,
+     * language and protocol at Russia's Star City.</description> <pubDate>Tue,
+     * 03 Jun 2003 09:39:21 GMT</pubDate>
+     * <guid>http://liftoff.msfc.nasa.gov/2003/06/03.html#item573</guid> </item>
+     * 
+     * <item> <title>Space Exploration</title>
+     * <link>http://liftoff.msfc.nasa.gov/</link> <description>Sky watchers in
+     * Europe, Asia, and parts of Alaska and Canada will experience a partial
+     * eclipse of the Sun on Saturday, May 31.</description> <pubDate>Fri, 30
+     * May 2003 11:06:42 GMT</pubDate>
+     * <guid>http://liftoff.msfc.nasa.gov/2003/05/30.html#item572</guid> </item>
+     * 
+     * </channel> </rss>
+     * 
+     * @param sitemap
+     * @param doc
+     */
+    private void parseRSS(SiteMap sitemap, Document doc) {
+
+        // Grab items from <item><link>URL</link></item>
+        // and last modified date from <pubDate>DATE</pubDate>
+
+        LOG.debug("Parsing RSS doc");
+        sitemap.setType(SitemapType.RSS);
+        NodeList list = doc.getElementsByTagName("channel");
+        Element elem = (Element) list.item(0);
+
+        // Treat publication date as last mod (Tue, 10 Jun 2003 04:00:00 GMT)
+        String lastMod = getElementValue(elem, "pubDate");
+
+        LOG.debug("lastMod=" + lastMod);
+
+        list = doc.getElementsByTagName("item");
+
+        // Loop through the <item>s
+        for (int i = 0; i < list.getLength() && i < MAX_URLS; i++) {
+
+            Node n = list.item(i);
+
+            if (n.getNodeType() == Node.ELEMENT_NODE) {
+                elem = (Element) n;
+
+                String link = getElementValue(elem, "link");
+                LOG.debug("link=" + link);
+
+                try {
+                    URL url = new URL(link);
+
+                    if (urlIsLegal(sitemap.getBaseUrl(), url.toString())) {
+                        SiteMapURL sUrl = new SiteMapURL(url.toString(), lastMod, null, null);
+                        sitemap.addSiteMapUrl(sUrl);
+                        LOG.debug("  " + (i + 1) + ". " + sUrl);
+                    }
+                } catch (MalformedURLException e) {
+                    // Can't create an entry with a bad URL
+                    LOG.debug("Bad url: [" + link + "]");
+                }
+            }
+        }
+    }
+
+    /**
+     * Get the element's textual content.
+     * 
+     * @param elem
+     * @param elementName
+     * @return
+     */
+    private String getElementValue(Element elem, String elementName) {
+
+        NodeList list = elem.getElementsByTagName(elementName);
+        Element e = (Element) list.item(0);
+        if (e != null) {
+            NodeList children = e.getChildNodes();
+            if (children.item(0) != null) {
+                return ((Node) children.item(0)).getNodeValue().trim();
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * Get the element's attribute value.
+     * 
+     * @param elem
+     * @param elementName
+     * @param attributeName
+     * @return
+     */
+    private String getElementAttributeValue(Element elem, String elementName, String attributeName) {
+
+        NodeList list = elem.getElementsByTagName(elementName);
+        Element e = (Element) list.item(0);
+        if (e != null) {
+            return e.getAttribute(attributeName);
+        }
+
+        return null;
+    }
+
+    /**
+     * See if testUrl is under sitemapUrl. Only URLs under sitemapUrl are legal.
+     * Both URLs are first converted to lowercase before the comparison is made
+     * (this could be an issue on web servers that are case sensitive).
+     * 
+     * @param sitemapUrl
+     * @param testUrl
+     * @return true if testUrl is under sitemapUrl, false otherwise
+     */
+    private boolean urlIsLegal(String sitemapBaseUrl, String testUrl) {
+
+        boolean ret = false;
+
+        // Don't try a comparison if the URL is too short to match
+        if (sitemapBaseUrl != null && sitemapBaseUrl.length() <= testUrl.length()) {
+            String u = testUrl.substring(0, sitemapBaseUrl.length()).toLowerCase();
+            ret = sitemapBaseUrl.equals(u);
+        }
+
+        LOG.debug("urlIsLegal: " + sitemapBaseUrl + " <= " + testUrl + " ? " + ret);
+
+        return ret;
+    }
+
+}
Index: src/main/java/crawlercommons/sitemaps/SiteMap.java
===================================================================
--- src/main/java/crawlercommons/sitemaps/SiteMap.java	(revision 0)
+++ src/main/java/crawlercommons/sitemaps/SiteMap.java	(revision 0)
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Hashtable;
+
+public class SiteMap extends AbstractSiteMap {
+
+    /** Various Sitemap types */
+    public enum SitemapType {
+        XML, ATOM, RSS, TEXT
+    };
+
+    /** W3C date the Sitemap was last modified */
+    private Date lastModified;
+
+    /** Indicates if we have tried to process this Sitemap or not */
+    private boolean processed;
+
+    /** This Sitemap's type */
+    private SitemapType type;
+
+    /**
+     * The base URL for the Sitemap is where the Sitemap was found If found at
+     * http://foo.org/abc/sitemap.xml then baseUrl is http://foo.org/abc/
+     * Sitemaps can only contain URLs that are under the base URL.
+     */
+    private String baseUrl;
+
+    /** URLs found in this Sitemap */
+    private Hashtable<String, SiteMapURL> urlList;
+
+    /**
+     * lastModified uses the W3C date format
+     * (http://www.w3.org/TR/NOTE-datetime)
+     */
+    private static DateFormat dateFormats[] = { new SimpleDateFormat("yyyy-MM-dd"), new SimpleDateFormat("yyyy-MM-dd'T'HH:mm+hh:00"), new SimpleDateFormat("yyyy-MM-dd'T'HH:mm-hh:00"),
+                    new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+hh:00"), new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss-hh:00"),
+
+                    // Accept RSS dates
+                    new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz") };
+
+    public static DateFormat fullDateFormat = dateFormats[1];
+
+    public SiteMap() {
+        urlList = new Hashtable<String, SiteMapURL>();
+        lastModified = null;
+        setProcessed(false);
+    }
+
+    public SiteMap(URL url) {
+        this();
+        setUrl(url);
+    }
+
+    public SiteMap(String url) {
+        this();
+        setUrl(url);
+    }
+
+    public SiteMap(URL url, Date lastModified) {
+        this(url);
+        setLastModified(lastModified);
+    }
+
+    public SiteMap(String url, String lastModified) {
+        this(url);
+        setLastModified(lastModified);
+    }
+
+    /**
+     * @return the Collection of SitemapUrls in this Sitemap.
+     */
+    public Collection<SiteMapURL> getSiteMapUrls() {
+        return urlList.values();
+    }
+
+    /**
+     * @param url
+     *            - the URL of the Sitemap
+     */
+    private void setUrl(URL url) {
+        this.url = url;
+        setBaseUrl(url);
+    }
+
+    /**
+     * @param url
+     *            - the URL of the Sitemap
+     */
+    private void setUrl(String url) {
+        try {
+            this.url = new URL(url);
+
+            setBaseUrl(this.url);
+        } catch (MalformedURLException e) {
+            this.url = null;
+        }
+    }
+
+    /**
+     * @param lastModified
+     *            - the lastModified to set
+     */
+    public void setLastModified(Date lastModified) {
+        this.lastModified = lastModified;
+    }
+
+    /**
+     * @param lastModified
+     *            - the lastModified to set
+     */
+    public void setLastModified(String lastModified) {
+        this.lastModified = SiteMap.convertToDate(lastModified);
+    }
+
+    /**
+     * @return the lastModified date of the Sitemap
+     */
+    public Date getLastModified() {
+        return lastModified;
+    }
+
+    public String toString() {
+        String s = "url=\"" + url + "\",lastMod=";
+        s += (lastModified == null) ? "null" : SiteMap.fullDateFormat.format(lastModified);
+        s += ",type=" + type + ",processed=" + processed + ",urlListSize=" + urlList.size();
+        return s;
+    }
+
+    /**
+     * Convert the given date (given in an acceptable DateFormat), null if the
+     * date is not in the correct format.
+     * 
+     * @param date
+     *            - the date to be parsed
+     * @return the Date equivalent
+     */
+    public static Date convertToDate(String date) {
+
+        if (date != null) {
+            for (DateFormat df : dateFormats) {
+                try {
+                    return df.parse(date);
+                } catch (ParseException e) {
+                    // do nothing
+                }
+            }
+        }
+
+        // Not successful parsing any dates
+        return null;
+    }
+
+    /**
+     * @param processed
+     *            - indicate if the Sitemap has been processed.
+     */
+    void setProcessed(boolean processed) {
+        this.processed = processed;
+    }
+
+    /**
+     * @return true if the Sitemap has been processed i.e it contains 
+     * at least one SiteMapURL
+     */
+    public boolean isProcessed() {
+        return processed;
+    }
+
+    /**
+     * This is private because only once we know the Sitemap's URL can we
+     * determine the base URL.
+     * 
+     * @param sitemapUrl
+     */
+    private void setBaseUrl(URL sitemapUrl) {
+        baseUrl = sitemapUrl.toString().toLowerCase();
+
+        // baseUrl = "http://foo.org/abc/sitemap.xml";
+
+        // Remove everything back to last slash.
+        // So http://foo.org/abc/sitemap.xml becomes http://foo.org/abc/
+        baseUrl = baseUrl.replaceFirst("/[^/]*$", "/");
+    }
+
+    /**
+     * @return the baseUrl for this Sitemap.
+     */
+    public String getBaseUrl() {
+        return baseUrl;
+    }
+
+    /**
+     * @param url
+     *            The SitemapUrl to be added to the Sitemap.
+     */
+    public void addSiteMapUrl(SiteMapURL url) {
+        urlList.put(url.getUrl().toString(), url);
+    }
+
+    /**
+     * @param type
+     *            the Sitemap type to set
+     */
+    void setType(SitemapType type) {
+        this.type = type;
+    }
+
+    /**
+     * @return the Sitemap type
+     */
+    public SitemapType getType() {
+        return type;
+    }
+
+    public boolean isIndex() {
+        return false;
+    }
+
+}
Index: src/main/java/crawlercommons/sitemaps/SiteMapURL.java
===================================================================
--- src/main/java/crawlercommons/sitemaps/SiteMapURL.java	(revision 0)
+++ src/main/java/crawlercommons/sitemaps/SiteMapURL.java	(revision 0)
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+// JDK imports
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Date;
+
+/**
+ * The SitemapUrl class represents a URL found in a Sitemap.
+ * 
+ * @author fmccown
+ */
+public class SiteMapURL {
+
+    /** Allowed change frequencies */
+    public enum ChangeFrequency {
+        ALWAYS, HOURLY, DAILY, WEEKLY, MONTHLY, YEARLY, NEVER
+    };
+
+    /** URL found in Sitemap (required) */
+    private URL url;
+
+    /** When URL was last modified (optional) */
+    private Date lastModified;
+
+    /** How often the URL changes (optional) */
+    private ChangeFrequency changeFreq;
+
+    /** Value between [0.0 - 1.0] (optional) */
+    private double priority;
+
+    public SiteMapURL(String url) {
+        setUrl(url);
+    }
+
+    public SiteMapURL(URL url) {
+        setUrl(url);
+    }
+
+    public SiteMapURL(String url, String lastModified, String changeFreq, String priority) {
+
+        setUrl(url);
+        setLastModified(lastModified);
+        setChangeFrequency(changeFreq);
+        setPriority(priority);
+    }
+
+    public SiteMapURL(URL url, Date lastModified, ChangeFrequency changeFreq, double priority) {
+
+        setUrl(url);
+        setLastModified(lastModified);
+        setChangeFrequency(changeFreq);
+        setPriority(priority);
+    }
+
+    /**
+     * Return the URL.
+     * 
+     * @return URL
+     */
+    public URL getUrl() {
+        return url;
+    }
+
+    /**
+     * Set the URL.
+     * 
+     * @param url
+     */
+    public void setUrl(URL url) {
+        this.url = url;
+    }
+
+    /**
+     * Set the URL.
+     * 
+     * @param url
+     */
+    public void setUrl(String url) {
+        try {
+            this.url = new URL(url);
+        } catch (MalformedURLException e) {
+            // e.printStackTrace();
+            System.out.println("Bad url: [" + url + "]");
+            this.url = null;
+        }
+    }
+
+    /**
+     * Return when this URL was last modified.
+     * 
+     * @return last modified date
+     */
+    public Date getLastModified() {
+        return lastModified;
+    }
+
+    /**
+     * Set when this URL was last modified.
+     * 
+     * @param lastModified
+     */
+    public void setLastModified(String lastModified) {
+        this.lastModified = SiteMap.convertToDate(lastModified);
+    }
+
+    /**
+     * Set when this URL was last modified.
+     * 
+     * @param lastModified
+     */
+    public void setLastModified(Date lastModified) {
+        this.lastModified = lastModified;
+    }
+
+    /**
+     * Return this URL's priority (a value between [0.0 - 1.0]).
+     * 
+     * @return URL's priority (a value between [0.0 - 1.0])
+     */
+    public double getPriority() {
+        return priority;
+    }
+
+    /**
+     * Set the URL's priority to a value between [0.0 - 1.0] (0.0 is used if the
+     * given priority is out of range).
+     * 
+     * @param priority
+     */
+    public void setPriority(double priority) {
+
+        // Ensure proper value
+        if (priority < 0.0 || priority > 1.0) {
+            this.priority = 0.0;
+        } else {
+            this.priority = priority;
+        }
+    }
+
+    /**
+     * Set the URL's priority to a value between [0.0 - 1.0] (0.0 is used if the
+     * given priority is out of range).
+     * 
+     * @param priority
+     */
+    public void setPriority(String priority) {
+
+        if (priority != null && priority.length() > 0) {
+            try {
+                setPriority(Double.parseDouble(priority));
+            } catch (NumberFormatException e) {
+                setPriority(0.0);
+            }
+        } else {
+            setPriority(0.0);
+        }
+    }
+
+    /**
+     * Return the URL's change frequency
+     * 
+     * @return the URL's change frequency
+     */
+    public ChangeFrequency getChangeFrequency() {
+        return changeFreq;
+    }
+
+    /**
+     * Set the URL's change frequency
+     * 
+     * @param changeFreq
+     */
+    public void setChangeFrequency(ChangeFrequency changeFreq) {
+        this.changeFreq = changeFreq;
+    }
+
+    /**
+     * Set the URL's change frequency
+     * 
+     * @param changeFreq
+     */
+    public void setChangeFrequency(String changeFreq) {
+
+        if (changeFreq != null) {
+            changeFreq = changeFreq.toUpperCase();
+
+            if (changeFreq.contains("ALWAYS")) {
+                this.changeFreq = ChangeFrequency.ALWAYS;
+            } else if (changeFreq.contains("HOURLY")) {
+                this.changeFreq = ChangeFrequency.HOURLY;
+            } else if (changeFreq.contains("DAILY")) {
+                this.changeFreq = ChangeFrequency.DAILY;
+            } else if (changeFreq.contains("WEEKLY")) {
+                this.changeFreq = ChangeFrequency.WEEKLY;
+            } else if (changeFreq.contains("MONTHLY")) {
+                this.changeFreq = ChangeFrequency.MONTHLY;
+            } else if (changeFreq.contains("YEARLY")) {
+                this.changeFreq = ChangeFrequency.YEARLY;
+            } else if (changeFreq.contains("NEVER")) {
+                this.changeFreq = ChangeFrequency.NEVER;
+            } else {
+                this.changeFreq = null;
+            }
+        }
+    }
+
+    public String toString() {
+        String s = "url=\"" + url + "\",";
+        s += "lastMod=";
+        s += (lastModified == null) ? "null" : SiteMap.fullDateFormat.format(lastModified);
+        s += ",changeFreq=" + changeFreq;
+        s += ",priority=" + priority;
+        return s;
+    }
+
+}
Index: src/main/java/crawlercommons/sitemaps/AbstractSiteMap.java
===================================================================
--- src/main/java/crawlercommons/sitemaps/AbstractSiteMap.java	(revision 0)
+++ src/main/java/crawlercommons/sitemaps/AbstractSiteMap.java	(revision 0)
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+import java.net.URL;
+
+/** SiteMap or SiteMapIndex**/
+public abstract class AbstractSiteMap {
+
+    URL url;
+    
+    public abstract boolean isIndex();
+    
+    /**
+     * @return the URL of the Sitemap
+     */
+    public URL getUrl() {
+        return url;
+    }
+    
+}
Index: src/main/java/crawlercommons/sitemaps/UnknownFormatException.java
===================================================================
--- src/main/java/crawlercommons/sitemaps/UnknownFormatException.java	(revision 0)
+++ src/main/java/crawlercommons/sitemaps/UnknownFormatException.java	(revision 0)
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+@SuppressWarnings("serial")
+public class UnknownFormatException extends Exception {
+
+    private String error;
+
+    // ----------------------------------------------
+    // Default constructor - initializes instance variable to unknown
+
+    public UnknownFormatException() {
+        super();
+        error = "unknown";
+    }
+
+    // -----------------------------------------------
+    // Constructor receives some kind of message that is saved in an instance
+    // variable.
+
+    public UnknownFormatException(String err) {
+        super(err);
+        error = err;
+    }
+
+    // ------------------------------------------------
+    // public method, callable by exception catcher. It returns the error
+    // message.
+
+    public String getError() {
+        return error;
+    }
+}
Index: src/main/java/crawlercommons/sitemaps/SiteMapIndex.java
===================================================================
--- src/main/java/crawlercommons/sitemaps/SiteMapIndex.java	(revision 0)
+++ src/main/java/crawlercommons/sitemaps/SiteMapIndex.java	(revision 0)
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawlercommons.sitemaps;
+
+import java.net.URL;
+import java.util.Collection;
+import java.util.Hashtable;
+import java.util.Map.Entry;
+
+public class SiteMapIndex extends AbstractSiteMap {
+
+    /** URLs found in this Sitemap Index */
+    private Hashtable<String, SiteMap> sitemaps;
+
+    public SiteMapIndex() {
+        sitemaps = new Hashtable<String, SiteMap>();
+    }
+
+    public SiteMapIndex(URL url) {
+        this();
+        this.url = url;
+    }
+
+    /**
+     * @return a Collection of Sitemaps in this Sitemap Index.
+     */
+    public Collection<SiteMap> getSitemaps() {
+        return sitemaps.values();
+    }
+
+    /**
+     * Add this Sitemap to the list of Sitemaps,
+     * 
+     * @param sitemap
+     *            - Sitemap to be added to the list of Sitemaps
+     */
+    void addSitemap(SiteMap sitemap) {
+        sitemaps.put(sitemap.getUrl().toString(), sitemap);
+    }
+
+    /**
+     * Returns the Sitemap that has the given URL. Returns null if the URL
+     * cannot be found.
+     * 
+     * @param url
+     *            - The Sitemap's URL
+     * @return SiteMap corresponding to the URL or null
+     */
+    public SiteMap getSitemap(URL url) {
+        return sitemaps.get(url.toString());
+    }
+
+    /**
+     * @return true if there are Sitemaps in this index that have not been
+     *         processed yet, false otherwise.
+     */
+    public boolean hasUnprocessedSitemap() {
+
+        // Find an unprocessed Sitemap
+        for (Entry<String, SiteMap> sitemap : sitemaps.entrySet()) {
+            SiteMap s = sitemap.getValue();
+            if (!s.isProcessed()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * @return an unprocessed Sitemap or null if no unprocessed Sitemaps could
+     *         be found.
+     */
+    public SiteMap nextUnprocessedSitemap() {
+        for (Entry<String, SiteMap> sitemap : sitemaps.entrySet()) {
+            SiteMap s = sitemap.getValue();
+            if (!s.isProcessed()) {
+                return s;
+            }
+        }
+
+        return null;
+    }
+
+    public String toString() {
+        return "url=\"" + url + "\",sitemapListSize=" + sitemaps.size();
+    }
+
+    public boolean isIndex() {
+        return true;
+    }
+
+}
Index: src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java
===================================================================
--- src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java	(revision 0)
+++ src/test/java/crawlercommons/sitemaps/SiteMapParserTest.java	(revision 0)
@@ -0,0 +1,83 @@
+package crawlercommons.sitemaps;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Date;
+import java.util.Iterator;
+
+import junit.framework.TestCase;
+
+import org.junit.After;
+import org.junit.Before;
+
+public class SiteMapParserTest extends TestCase {
+
+    @Before
+    public void setUp() throws Exception {
+    }
+
+    @After
+    public void tearDown() throws Exception {
+    }
+
+    public void testSitemapIndex() throws UnknownFormatException, IOException {
+        SiteMapParser parser = new SiteMapParser();
+        String contentType = "text/xml";
+
+        String scontent = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">" + "<sitemap>"
+                        + "  <loc>http://www.example.com/sitemap1.xml.gz</loc>" + " <lastmod>2004-10-01T18:23:17+00:00</lastmod>" + " </sitemap>" + "<sitemap>"
+                        + "    <loc>http://www.example.com/sitemap2.xml.gz</loc>" + "   <lastmod>2005-01-01</lastmod>" + " </sitemap>" + " </sitemapindex>";
+
+        byte[] content = scontent.getBytes();
+        URL url = new URL("http://www.example.com/sitemapindex.xml");
+        AbstractSiteMap asm = parser.parseSiteMap(contentType, content, url);
+        assertEquals(true, asm.isIndex());
+        assertEquals(true, asm instanceof SiteMapIndex);
+        SiteMapIndex smi = (SiteMapIndex) asm;
+        assertEquals(2, smi.getSitemaps().size());
+        SiteMap currentSiteMap = smi.getSitemap(new URL("http://www.example.com/sitemap1.xml.gz"));
+        assertNotNull(currentSiteMap);
+        assertEquals("http://www.example.com/sitemap1.xml.gz", currentSiteMap.getUrl().toString());
+        assertEquals(SiteMap.convertToDate("2004-10-01T18:23:17+00:00"), currentSiteMap.getLastModified());
+
+        currentSiteMap = smi.getSitemap(new URL("http://www.example.com/sitemap2.xml.gz"));
+        assertNotNull(currentSiteMap);
+        assertEquals("http://www.example.com/sitemap2.xml.gz", currentSiteMap.getUrl().toString());
+        assertEquals(SiteMap.convertToDate("2005-01-01"), currentSiteMap.getLastModified());
+    }
+
+    public void testSitemapXML() throws UnknownFormatException, IOException {
+        SiteMapParser parser = new SiteMapParser();
+        String contentType = "text/xml";
+
+        String scontent = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">" + "  <url>" + "<loc>http://www.example.com/</loc>"
+                        + "<lastmod>2005-01-01</lastmod>" + "<changefreq>monthly</changefreq>" + "<priority>0.8</priority>" + "</url>" + "<url>"
+                        + "<loc>http://www.example.com/catalog?item=12&amp;desc=vacation_hawaii</loc>" + "<changefreq>weekly</changefreq>" + "</url>" + "<url>"
+                        + "<loc>http://www.example.com/catalog?item=73&amp;desc=vacation_new_zealand</loc>" + "<lastmod>2004-12-23</lastmod>" + "<changefreq>weekly</changefreq>" + "</url>" + "<url>"
+                        + "<loc>http://www.example.com/catalog?item=74&amp;desc=vacation_newfoundland</loc>" + "<lastmod>2004-12-23T18:00:15+00:00</lastmod>" + "<priority>0.3</priority>" + "</url>"
+                        + "<url>" + "<loc>http://www.example.com/catalog?item=83&amp;desc=vacation_usa</loc>" + "<lastmod>2004-11-23</lastmod>" + "</url>" + "</urlset>";
+
+        byte[] content = scontent.getBytes();
+        URL url = new URL("http://www.example.com/sitemap.xml");
+        AbstractSiteMap asm = parser.parseSiteMap(contentType, content, url);
+        assertEquals(false, asm.isIndex());
+        assertEquals(true, asm instanceof SiteMap);
+        SiteMap sm = (SiteMap) asm;
+        assertEquals(5, sm.getSiteMapUrls().size());
+    }
+
+    public void testSitemapTXT() throws UnknownFormatException, IOException {
+        SiteMapParser parser = new SiteMapParser();
+        String contentType = "text/plain";
+
+        String scontent = "http://www.example.com/catalog?item=1\nhttp://www.example.com/catalog?item=11";
+
+        byte[] content = scontent.getBytes();
+        URL url = new URL("http://www.example.com/sitemap.txt");
+        AbstractSiteMap asm = parser.parseSiteMap(contentType, content, url);
+        assertEquals(false, asm.isIndex());
+        assertEquals(true, asm instanceof SiteMap);
+        SiteMap sm = (SiteMap) asm;
+        assertEquals(2, sm.getSiteMapUrls().size());
+    }
+}
