split modules

webmagic-core/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <groupId>us.codecraft</groupId>
+    <version>0.0.1-SNAPSHOT</version>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>webmagic-core</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.2.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.7</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>13.0.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>1.2.17</version>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-collections</groupId>
+            <artifactId>commons-collections</artifactId>
+            <version>3.2.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>net.sourceforge.htmlcleaner</groupId>
+            <artifactId>htmlcleaner</artifactId>
+            <version>2.4</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>1.3.2</version>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+                <configuration>
+                    <encoding>UTF-8</encoding>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-release-plugin</artifactId>
+                <version>2.0-beta-7</version>
+            </plugin>
+        </plugins>
+    </build>
+
+
+</project>
\ No newline at end of file

src/main/java/us/codecraft/spider/Page.java → webmagic-core/src/main/java/us/codecraft/webmagic/Page.java

-package us.codecraft.spider;
+package us.codecraft.webmagic;
 
 import org.apache.commons.lang3.StringUtils;
-import us.codecraft.spider.selector.Selectable;
-import us.codecraft.spider.utils.UrlUtils;
+import us.codecraft.webmagic.selector.Selectable;
+import us.codecraft.webmagic.utils.UrlUtils;
 
 import java.util.ArrayList;
 import java.util.List;

src/main/java/us/codecraft/spider/Request.java → webmagic-core/src/main/java/us/codecraft/webmagic/Request.java

-package us.codecraft.spider;
-
-import us.codecraft.spider.Site;
-
-import java.util.List;
+package us.codecraft.webmagic;
 
 /**
  * User: cairne

src/main/java/us/codecraft/spider/Site.java → webmagic-core/src/main/java/us/codecraft/webmagic/Site.java

-package us.codecraft.spider;
+package us.codecraft.webmagic;
 
 import java.util.HashSet;
 import java.util.Set;

src/main/java/us/codecraft/spider/Spider.java → webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

-package us.codecraft.spider;
+package us.codecraft.webmagic;
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Logger;
-import us.codecraft.spider.downloader.Downloader;
-import us.codecraft.spider.downloader.HttpClientDownloader;
-import us.codecraft.spider.pipeline.ConsolePipeline;
-import us.codecraft.spider.pipeline.Pipeline;
-import us.codecraft.spider.processor.PageProcessor;
-import us.codecraft.spider.schedular.QueueSchedular;
-import us.codecraft.spider.schedular.Schedular;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import us.codecraft.webmagic.downloader.Downloader;
+import us.codecraft.webmagic.downloader.HttpClientDownloader;
+import us.codecraft.webmagic.pipeline.ConsolePipeline;
+import us.codecraft.webmagic.pipeline.Pipeline;
+import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.schedular.QueueSchedular;
+import us.codecraft.webmagic.schedular.Schedular;
 
 /**
  * User: cairne

src/main/java/us/codecraft/spider/downloader/Downloader.java → webmagic-core/src/main/java/us/codecraft/webmagic/downloader/Downloader.java

-package us.codecraft.spider.downloader;
+package us.codecraft.webmagic.downloader;
 
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Request;
-import us.codecraft.spider.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
 
 /**
  * User: cairne

src/main/java/us/codecraft/spider/downloader/HttpClientDownloader.java → webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

-package us.codecraft.spider.downloader;
+package us.codecraft.webmagic.downloader;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.log4j.Logger;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Request;
-import us.codecraft.spider.Site;
-import us.codecraft.spider.selector.Html;
-import us.codecraft.spider.selector.PlainText;
-import us.codecraft.spider.utils.UrlUtils;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.selector.PlainText;
+import us.codecraft.webmagic.utils.UrlUtils;
 
 
 /**

src/main/java/us/codecraft/spider/downloader/HttpClientPool.java → webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java

-package us.codecraft.spider.downloader;
+package us.codecraft.webmagic.downloader;
 
 import org.apache.http.HttpVersion;
 import org.apache.http.client.HttpClient;
@@ -10,7 +10,7 @@ import org.apache.http.conn.scheme.SchemeRegistry;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.impl.conn.PoolingClientConnectionManager;
 import org.apache.http.params.*;
-import us.codecraft.spider.Site;
+import us.codecraft.webmagic.Site;
 
 /**
  * User: cairne

src/main/java/us/codecraft/spider/pipeline/ConsolePipeline.java → webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ConsolePipeline.java

-package us.codecraft.spider.pipeline;
+package us.codecraft.webmagic.pipeline;
 
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Site;
-import us.codecraft.spider.selector.Selectable;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.selector.Selectable;
 
 import java.util.Map;
 

src/main/java/us/codecraft/spider/pipeline/FilePipeline.java → webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java

-package us.codecraft.spider.pipeline;
+package us.codecraft.webmagic.pipeline;
 
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Site;
-import us.codecraft.spider.selector.Selectable;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.selector.Selectable;
 
 import java.io.File;
 import java.io.FileWriter;
@@ -19,7 +19,7 @@ import java.util.Map;
  */
 public class FilePipeline implements Pipeline {
 
-    private String path = "/data/temp/spider/";
+    private String path = "/data/temp/webmagic/";
 
     public FilePipeline(){
 

src/main/java/us/codecraft/spider/pipeline/Pipeline.java → webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/Pipeline.java

-package us.codecraft.spider.pipeline;
+package us.codecraft.webmagic.pipeline;
 
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
 
 /**
  * User: cairne

src/main/java/us/codecraft/spider/processor/PageProcessor.java → webmagic-core/src/main/java/us/codecraft/webmagic/processor/PageProcessor.java

-package us.codecraft.spider.processor;
+package us.codecraft.webmagic.processor;
 
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
 
 /**
  * User: cairne

src/main/java/us/codecraft/spider/processor/SimplePageProcessor.java → webmagic-core/src/main/java/us/codecraft/webmagic/processor/SimplePageProcessor.java

-package us.codecraft.spider.processor;
+package us.codecraft.webmagic.processor;
 
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Site;
-import us.codecraft.spider.utils.UrlUtils;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.utils.UrlUtils;
 
 import java.util.List;
 

src/main/java/us/codecraft/spider/schedular/FileCacheQueueSchedular.java → webmagic-core/src/main/java/us/codecraft/webmagic/schedular/FileCacheQueueSchedular.java

-package us.codecraft.spider.schedular;
+package us.codecraft.webmagic.schedular;
 
 import org.apache.commons.lang3.math.NumberUtils;
 import org.apache.log4j.Logger;
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Request;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Request;
 
 import java.io.*;
 import java.util.LinkedHashSet;

src/main/java/us/codecraft/spider/schedular/QueueSchedular.java → webmagic-core/src/main/java/us/codecraft/webmagic/schedular/QueueSchedular.java

-package us.codecraft.spider.schedular;
+package us.codecraft.webmagic.schedular;
 
 import org.apache.log4j.Logger;
-import us.codecraft.spider.Request;
-import us.codecraft.spider.Site;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
 
 import java.util.HashSet;
 import java.util.Set;

src/main/java/us/codecraft/spider/schedular/Schedular.java → webmagic-core/src/main/java/us/codecraft/webmagic/schedular/Schedular.java

-package us.codecraft.spider.schedular;
+package us.codecraft.webmagic.schedular;
 
-import us.codecraft.spider.Request;
-import us.codecraft.spider.Site;
+import us.codecraft.webmagic.Request;
+import us.codecraft.webmagic.Site;
 
 /**
  * User: cairne

src/main/java/us/codecraft/spider/selector/Html.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/Html.java

-package us.codecraft.spider.selector;
-
-import org.apache.commons.collections.CollectionUtils;
+package us.codecraft.webmagic.selector;
 
 import java.util.ArrayList;
 import java.util.List;

src/main/java/us/codecraft/spider/selector/PlainText.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/PlainText.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import org.apache.commons.collections.CollectionUtils;
 

src/main/java/us/codecraft/spider/selector/RegexResult.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/RegexResult.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 /**
  * User: cairne

src/main/java/us/codecraft/spider/selector/RegexSelector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/RegexSelector.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import org.apache.commons.lang3.StringUtils;
 

src/main/java/us/codecraft/spider/selector/ReplaceSelector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/ReplaceSelector.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import java.util.List;
 import java.util.regex.Matcher;

src/main/java/us/codecraft/spider/selector/Selectable.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selectable.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import java.util.List;
 

src/main/java/us/codecraft/spider/selector/Selector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selector.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import java.util.List;
 

src/main/java/us/codecraft/spider/selector/SelectorFactory.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/SelectorFactory.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import org.apache.commons.lang3.StringUtils;
 

src/main/java/us/codecraft/spider/selector/SmartContentSelector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/SmartContentSelector.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import org.apache.log4j.Logger;
 import org.htmlcleaner.HtmlCleaner;

src/main/java/us/codecraft/spider/selector/XpathSelector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/XpathSelector.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import org.htmlcleaner.*;
 

src/main/java/us/codecraft/spider/utils/UrlUtils.java → webmagic-core/src/main/java/us/codecraft/webmagic/utils/UrlUtils.java

-package us.codecraft.spider.utils;
+package us.codecraft.webmagic.utils;
 
 import org.apache.commons.lang3.StringUtils;
 

src/test/java/us/codecraft/spider/HtmlTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/HtmlTest.java

-package us.codecraft.spider;
+package us.codecraft.webmagic;
 
 import org.junit.Assert;
 import org.junit.Test;
-import us.codecraft.spider.selector.Html;
+import us.codecraft.webmagic.selector.Html;
 
 /**
  * User: cairne

src/test/java/us/codecraft/spider/SpiderTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/SpiderTest.java

-package us.codecraft.spider;
+package us.codecraft.webmagic;
 
 import org.junit.Ignore;
 import org.junit.Test;
-import us.codecraft.spider.pipeline.FilePipeline;
-import us.codecraft.spider.processor.SimplePageProcessor;
-import us.codecraft.spider.samples.HuxiuProcessor;
-import us.codecraft.spider.schedular.FileCacheQueueSchedular;
+import us.codecraft.webmagic.pipeline.FilePipeline;
+import us.codecraft.webmagic.processor.SimplePageProcessor;
+import us.codecraft.webmagic.samples.HuxiuProcessor;
+import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
 
 /**
  * User: cairne
@@ -24,12 +24,12 @@ public class SpiderTest {
     @Test
     public void testGlobalSpider(){
 //        PageProcessor pageProcessor = new MeicanProcessor();
-//        Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor.getSite(),"/data/temp/spider/cache/")).
+//        Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
 //                processor(pageProcessor).run();
         SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html");
         pageProcessor2.getSite().setEncoding("GBK");
         System.out.println(pageProcessor2.getSite().getEncoding());
-        Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor2.getSite(),"/data/temp/spider/cache/")).
+        Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor2.getSite(),"/data/temp/webmagic/cache/")).
                 processor(pageProcessor2).run();
 
 

src/test/java/us/codecraft/spider/samples/DiandianBlogProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/DiandianBlogProcessor.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/DianpingBlogProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/DianpingBlogProcessor.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/DiaoyuwengProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/DiaoyuwengProcessor.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Site;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/F58PageProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/F58PageProcesser.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/HuxiuProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/HuxiuProcessor.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/KaichibaProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/KaichibaProcessor.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Site;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 /**
  * User: cairne

src/test/java/us/codecraft/spider/samples/MeicanProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/MeicanProcessor.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Site;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/NjuBBSProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/NjuBBSProcessor.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/OschinaBlogPageProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/OschinaBlogPageProcesser.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/OschinaPageProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/OschinaPageProcesser.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/QzoneBlogProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/QzoneBlogProcessor.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Page;
-import us.codecraft.spider.Site;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/samples/SinaBlogProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/SinaBlogProcesser.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 /**
  * User: cairne

src/test/java/us/codecraft/spider/samples/TianyaPageProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/TianyaPageProcesser.java

-package us.codecraft.spider.samples;
+package us.codecraft.webmagic.samples;
 
-import us.codecraft.spider.Site;
-import us.codecraft.spider.Page;
-import us.codecraft.spider.processor.PageProcessor;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
 
 import java.util.List;
 

src/test/java/us/codecraft/spider/selector/HtmlCleanerTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/selector/HtmlCleanerTest.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import org.htmlcleaner.CleanerProperties;
 import org.htmlcleaner.HtmlCleaner;
@@ -6,7 +6,6 @@ import org.htmlcleaner.TagNode;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.net.MalformedURLException;
 import java.net.URL;
 
 /**

src/test/java/us/codecraft/spider/selector/RegexSelectorTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/selector/RegexSelectorTest.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import junit.framework.Assert;
 import org.junit.Test;

src/test/java/us/codecraft/spider/selector/SmartConentSelectorTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/selector/SmartConentSelectorTest.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
-import org.htmlcleaner.CleanerProperties;
-import org.htmlcleaner.HtmlCleaner;
-import org.htmlcleaner.TagNode;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.net.URL;
 
 /**
  * User: cairne

src/test/java/us/codecraft/spider/selector/XpathSelectorTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/selector/XpathSelectorTest.java

-package us.codecraft.spider.selector;
+package us.codecraft.webmagic.selector;
 
 import org.junit.Assert;
 import org.junit.Test;

src/test/java/us/codecraft/spider/utils/UrlUtilsTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/utils/UrlUtilsTest.java

-package us.codecraft.spider.utils;
+package us.codecraft.webmagic.utils;
 
 import org.junit.Assert;
 import org.junit.Test;

webmagic-core/target/classes/log4j.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+    <appender name="stdout" class="org.apache.log4j.ConsoleAppender">
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
+        </layout>
+    </appender>
+
+    <logger name="org.springframework" additivity="false">
+        <level value="warn" />
+        <appender-ref ref="stdout" />
+    </logger>
+
+    <logger name="net.sf.ehcache" additivity="false">
+        <level value="warn" />
+        <appender-ref ref="stdout" />
+    </logger>
+
+    <root>
+        <level value="info" />
+        <appender-ref ref="stdout" />
+    </root>
+
+</log4j:configuration>

webmagic-core/target/test-classes/log4j.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+    <appender name="stdout" class="org.apache.log4j.ConsoleAppender">
+        <layout class="org.apache.log4j.PatternLayout">
+            <param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
+        </layout>
+    </appender>
+
+    <logger name="org.springframework" additivity="false">
+        <level value="warn" />
+        <appender-ref ref="stdout" />
+    </logger>
+
+    <logger name="org.apache" additivity="false">
+        <level value="warn" />
+        <appender-ref ref="stdout" />
+    </logger>
+
+    <logger name="net.sf.ehcache" additivity="false">
+        <level value="warn" />
+        <appender-ref ref="stdout" />
+    </logger>
+
+    <root>
+        <level value="debug" />
+        <appender-ref ref="stdout" />
+    </root>
+
+</log4j:configuration>

webmagic-plugin/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <groupId>us.codecraft</groupId>
+    <version>0.0.1-SNAPSHOT</version>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>webmagic-plugin</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-core</artifactId>
+            <version>0.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.7</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+                <configuration>
+                    <encoding>UTF-8</encoding>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-release-plugin</artifactId>
+                <version>2.0-beta-7</version>
+            </plugin>
+        </plugins>
+    </build>
+
+
+</project>
\ No newline at end of file

webmagic-samples/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <groupId>us.codecraft</groupId>
+    <version>0.0.1-SNAPSHOT</version>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>webmagic-samples</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-core</artifactId>
+            <version>0.0.1-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.7</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+                <configuration>
+                    <encoding>UTF-8</encoding>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-release-plugin</artifactId>
+                <version>2.0-beta-7</version>
+            </plugin>
+        </plugins>
+    </build>
+
+
+</project>
\ No newline at end of file

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/DiandianBlogProcessor.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午8:08
+ */
+public class DiandianBlogProcessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        //http://progressdaily.diandian.com/post/2013-01-24/40046867275
+        List<String> requests = page.getHtml().rs("<a[^<>]*href=[\"']{1}(http://17dujingdian\\.com/post/[^#]*?)[\"']{1}").toStrings();
+        page.addTargetRequests(requests);
+        page.putField("title",page.getHtml().x("//div[@id='content']//h2/a"));
+        page.putField("content",page.getHtml().sc());
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("www.diandian.com").setStartUrl("http://17dujingdian.com/").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/DianpingBlogProcessor.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午8:08
+ */
+public class DianpingBlogProcessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        //http://progressdaily.diandian.com/post/2013-01-24/40046867275
+        List<String> requests = page.getHtml().rs("<a[^<>]*href=[\"']{1}(/shop/.*?)[\"']{1}").toStrings();
+        page.addTargetRequests(requests);
+        requests = page.getHtml().rs("<a[^<>]*href=[\"']{1}(/search/category/.*?)[\"']{1}").toStrings();
+        page.addTargetRequests(requests);
+        if (page.getUrl().toString().contains("shop")){
+            page.putField("title", page.getHtml().x("//h1[@class='shop-title']"));
+            page.putField("content", page.getHtml().sc());
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("www.dianping.com").setStartUrl("http://www.dianping.com/").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/DiaoyuwengProcessor.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午8:08
+ */
+public class DiaoyuwengProcessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        //http://progressdaily.diandian.com/post/2013-01-24/40046867275
+        List<String> requests = page.getHtml().rs("<a[^<>]*href=[\"']{1}(/shop/.*?)[\"']{1}").toStrings();
+        page.addTargetRequests(requests);
+        requests = page.getHtml().rs("<a[^<>]*href=[\"']{1}(/search/category/.*?)[\"']{1}").toStrings();
+        page.addTargetRequests(requests);
+        if (page.getUrl().toString().contains("shop")){
+            page.putField("title", page.getHtml().x("//h1[@class='shop-title']"));
+            page.putField("content", page.getHtml().sc());
+        }
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("www.dianping.com").setStartUrl("http://www.dianping.com/").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/F58PageProcesser.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午1:48
+ */
+public class F58PageProcesser implements PageProcessor {
+
+    @Override
+    public void process(Page page) {
+        List<String> strings = page.getHtml().rs("<a[^<>]*href=[\"']{1}(/yewu/.*?)[\"']{1}").toStrings();
+        page.addTargetRequests(strings);
+        page.putField("title",page.getHtml().r("<title>(.*)</title>"));
+        page.putField("body",page.getHtml().x("//dd[@class='w133']"));
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("sh.58.com").setStartUrl("http://sh.58.com/");  //To change body of implemented methods use File | Settings | File Templates.
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/HuxiuProcessor.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午8:08
+ */
+public class HuxiuProcessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        //http://progressdaily.diandian.com/post/2013-01-24/40046867275
+        List<String> requests = page.getHtml().rs("<a[^<>\"']*href=[\"']{1}([/]{0,1}article[^<>#\"']*?)[\"']{1}").toStrings();
+        page.addTargetRequests(requests);
+        page.putField("title",page.getHtml().x("//div[@class='neirong']//h1[@class='ph xs5']"));
+        page.putField("content",page.getHtml().sc());
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("www.huxiu.com").setStartUrl("http://www.huxiu.com/").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/KaichibaProcessor.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+/**
+ * User: cairne
+ * Date: 13-5-20
+ * Time: 下午5:31
+ */
+public class KaichibaProcessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        //http://progressdaily.diandian.com/post/2013-01-24/40046867275
+        int i = Integer.valueOf(page.getUrl().r("shop/(\\d+)").toString()) + 1;
+        page.addTargetRequests("http://kaichiba.com/shop/"+i);
+        page.putField("title",page.getHtml().x("//Title"));
+        page.putField("items", page.getHtml().xs("//li[@class=\"foodTitle\"]").rp("^\\s+", "").rp("\\s+$", "").rp("<span>.*?</span>", ""));
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("kaichiba.com").setStartUrl("http://kaichiba.com/shop/41725781").setEncoding("utf-8").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/MeicanProcessor.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-5-20
+ * Time: 下午5:31
+ */
+public class MeicanProcessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        //http://progressdaily.diandian.com/post/2013-01-24/40046867275
+        List<String> requests = page.getHtml().xs("//a[@class=\"area_link flat_btn\"]/@href").toStrings();
+        if (requests.size() > 2) {
+            requests = requests.subList(0, 2);
+        }
+        page.addTargetRequests(requests);
+        page.addTargetRequests(page.getHtml().as().rs("(.*/restaurant/[^#]+)").toStrings());
+        page.putField("items", page.getHtml().xs("//ul[@class=\"dishes menu_dishes\"]/li/span[@class=\"name\"]"));
+        page.putField("prices", page.getHtml().xs("//ul[@class=\"dishes menu_dishes\"]/li/span[@class=\"price_outer\"]/span[@class=\"price\"]"));
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("meican.com").setStartUrl("http://www.meican.com/shanghai/districts").setEncoding("utf-8").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/NjuBBSProcessor.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午8:08
+ */
+public class NjuBBSProcessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        List<String> requests = page.getHtml().rs("<a[^<>]*href=(bbstcon\\?board=Pictures&file=[^>]*)").toStrings();
+        page.addTargetRequests(requests);
+        page.putField("title",page.getHtml().x("//div[@id='content']//h2/a"));
+        page.putField("content",page.getHtml().sc());
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("bbs.nju.edu.cn").setStartUrl("http://bbs.nju.edu.cn/board?board=Pictures").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/OschinaBlogPageProcesser.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午1:48
+ */
+public class OschinaBlogPageProcesser implements PageProcessor {
+
+    @Override
+    public void process(Page page) {
+        List<String> strings = page.getHtml().as().r("(http://my\\.oschina\\.net)").toStrings();
+        page.addTargetRequests(strings);
+        page.putField("title", page.getHtml().xs("//div[@class='BlogEntity']/div[@class='BlogTitle']/h1"));
+        page.putField("content", page.getHtml().sc());
+        page.putField("author", page.getUrl().r("my\\.oschina\\.net/(\\w+)/blog/\\d+"));
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("my.oschina.net").setStartUrl("http://www.oschina.net/").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/OschinaPageProcesser.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午1:48
+ */
+public class OschinaPageProcesser implements PageProcessor {
+
+    @Override
+    public void process(Page page) {
+        List<String> strings = page.getHtml().rs("<a[^<>]*href=[\"']{1}(http://www\\.oschina\\.net/question/[\\w]+)[\"']{1}").toStrings();
+        page.addTargetRequests(strings);
+        page.putField("title", page.getHtml().x("//div[@class='QTitle']/h1/a"));
+        page.putField("content", page.getHtml().xs("//div[@class='Question']//div[@class='Content']/div[@class='detail']"));
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("www.oschina.net").setStartUrl("http://www.oschina.net/").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/QzoneBlogProcessor.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午8:08
+ */
+public class QzoneBlogProcessor implements PageProcessor {
+    @Override
+    public void process(Page page) {
+        //http://progressdaily.diandian.com/post/2013-01-24/40046867275
+
+        //http://b1.cnc.qzone.qq.com/cgi-bin/blognew/get_abs?hostUin=233017404&uin=233017404&blogType=0&statYear=2013&source=0&statYear=2013&g_tk=291639571&g_tk=291639571&reqInfo=7&pos=0&num=15&source=0&rand=0.46480297949165106
+        // &cateName=&cateHex=&statYear=2013&reqInfo=7&pos=0&num=15&sortType=0&source=0&rand=0.46480297949165106&g_tk=291639571&verbose=1&ref=qzone
+        List<String> requests = page.getHtml().rs("<a[^<>]*href=[\"']{1}(http://17dujingdian\\.com/post/[^#]*?)[\"']{1}").toStrings();
+        page.addTargetRequests(requests);
+        page.putField("title",page.getHtml().x("//div[@id='content']//h2/a"));
+        page.putField("content",page.getHtml().sc());
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("www.diandian.com").setStartUrl("http://17dujingdian.com/").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/SinaBlogProcesser.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午1:48
+ */
+public class SinaBlogProcesser implements PageProcessor {
+
+    @Override
+    public void process(Page page) {
+        page.addTargetRequests(page.getHtml().rs("<a[^<>]*href=[\"']{1}(http://blog\\.sina\\.com\\.cn/s/blog_.*?)[\"']{1}").toStrings());
+        page.putField("title", page.getHtml().x("//div[@class='articalTitle']/h2"));
+        page.putField("body",page.getHtml().sc());
+        //x("//dd[@class='w133']")
+        page.putField("date",page.getHtml().x("//div[@id='articlebody']//span[@class='time SG_txtc']").r("\\((.*)\\)"));
+        page.putField("tags",page.getHtml().xs("//td[@class='blog_tag']/h3/a"));
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("blog.sina.com.cn").setStartUrl("http://blog.sina.com.cn/").
+                setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
+    }
+}

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/TianyaPageProcesser.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import java.util.List;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 下午1:48
+ */
+public class TianyaPageProcesser implements PageProcessor {
+
+    @Override
+    public void process(Page page) {
+        List<String> strings = page.getHtml().rs("<a[^<>]*href=[\"']{1}(/post-free.*?\\.shtml)[\"']{1}").toStrings();
+        page.addTargetRequests(strings);
+        page.putField("title", page.getHtml().x("//div[@id='post_head']//span[@class='s_title']//b"));
+        page.putField("body",page.getHtml().sc());
+    }
+
+    @Override
+    public Site getSite() {
+        return Site.me().setDomain("http://bbs.tianya.cn/").setStartUrl("http://bbs.tianya.cn/");  //To change body of implemented methods use File | Settings | File Templates.
+    }
+}

webmagic-samples/src/test/java/us/codecraft/webmagic/HtmlTest.java

+package us.codecraft.webmagic;
+
+import org.junit.Assert;
+import org.junit.Test;
+import us.codecraft.webmagic.selector.Html;
+
+/**
+ * User: cairne
+ * Date: 13-4-21
+ * Time: 上午8:42
+ */
+public class HtmlTest {
+
+    @Test
+    public void testRegexSelector() {
+        Html selectable = new Html("aaaaaaab");
+        Assert.assertEquals("abbabbab", (selectable.r("(.*)").rp("aa(a)", "$1bb").toString()));
+
+    }
+}

webmagic-samples/src/test/java/us/codecraft/webmagic/SpiderTest.java

+package us.codecraft.webmagic;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import us.codecraft.webmagic.pipeline.FilePipeline;
+import us.codecraft.webmagic.processor.SimplePageProcessor;
+import us.codecraft.webmagic.samples.HuxiuProcessor;
+import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
+
+/**
+ * User: cairne
+ * Date: 13-4-20
+ * Time: 下午7:46
+ */
+public class SpiderTest {
+
+
+    @Test
+    public void testSpider() throws InterruptedException {
+        Spider me = Spider.me().pipeline(new FilePipeline()).processor(new HuxiuProcessor());
+        me.run();
+    }
+
+    @Test
+    public void testGlobalSpider(){
+//        PageProcessor pageProcessor = new MeicanProcessor();
+//        Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
+//                processor(pageProcessor).run();
+        SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html");
+        pageProcessor2.getSite().setEncoding("GBK");
+        System.out.println(pageProcessor2.getSite().getEncoding());
+        Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor2.getSite(),"/data/temp/webmagic/cache/")).
+                processor(pageProcessor2).run();
+
+
+    }
+
+    @Test
+    public void test(){
+        System.out.println(System.getProperty("java.io.tmpdir"));
+    }
+
+
+    @Ignore
+    @Test
+    public void languageSchema() {
+
+
+        /**
+         *
+         * _hrefs = rs("<a[^<>]*href=[\"']{1}(/yewu/.*?)[\"']{1}")
+         * title = r(""<title>(.*)</title>"")
+         * body = x("//dd[@class='w133']")
+         *
+         * site.domain = "sh.58.com"
+         * site.ua=""
+         * site.cookie="aa:bb"
+         *
+         */
+
+        /**
+         *
+         *
+         * if (page == r('') && refer(1) == 1) {
+         *
+         *      type = _refer(1)
+         *      content = _text.t().c()
+         *      title = x("asd@asd").r("",1)
+         *      body[r(_currentUrl).g(1)] = body[r(_currentUrl).g(1)] + (x("").r("",1,2).c())
+         *
+         *      body=body[r(_currentUrl).g(1)]
+         *      tags[%] = (tags[%] + xs('')) . r('')
+         *
+         *      _targetUrls.add('' + x('').r(''))
+         *      _sourceUrls.add()
+         *      _header.put("","");
+         *      _cookie.add("asdsadasdsa");
+         *
+         *
+         * }
+         *
+         * _cookie.add(_cookie[''])
+         *
+         * if (page == r('') && refer(1) == 1)
+         *  (
+         *      _targetUrl = '' + x('') & r('')
+         *      _sourceUrl = ''
+         *  )
+         *
+         */
+
+        /**
+         * <condition></>
+         * <selector>
+         *     <fields>
+         *
+         *     <type>
+         *         <selector></selector>
+         *         <selector></selector>
+         *     </type>
+         *         </>
+         *     </>
+         */
+
+        /**
+         *
+         * if (model.url('') && model.refer(1) == 1)
+         *  (
+         *
+         *      model.set(type, model.refer(1))
+         *      content = t(_html) > c()
+         *      title = x(_html, 'asd@asd') > r('',1)
+         *      body[r(_currentUrl).g(1)] = body[r(_currentUrl).g(1)] + (x('') > r('',1,2) > c()) | x('')
+         *      tags[%] = tags + xs('') > r('')
+         *      model.setTargetUrl();
+         *
+         *      _targetUrl = '' + x('') & r('')
+         *      _sourceUrl = ''
+         * )
+         *
+         * _cookie.add(_cookie[''])
+         *
+         * if (page == r('') && refer(1) == 1)
+         *  (
+         *      _targetUrl = '' + x('') & r('')
+         *      _sourceUrl = ''
+         *  )
+         *
+         */
+    }
+}