add worker as container

webmagic-avalon/webmagic-worker/pom.xml

@@ -16,6 +16,18 @@
             <artifactId>webmagic-avalon-common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.aspectj</groupId>
+            <artifactId>aspectjrt</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

webmagic-avalon/webmagic-worker/src/main/java/us/codecraft/webmagic/worker/Worker.java

+package us.codecraft.webmagic.worker;
+
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.utils.ThreadUtils;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * @author code4crafter@gmail.com
+ */
+public class Worker {
+
+    public static final int DEFAULT_POOL_SIZE = 10;
+
+    private int poolSize;
+
+    private ExecutorService executorService;
+
+    private Map<String,Spider> spiderMap;
+
+    public Worker(int poolSize) {
+        this.poolSize = poolSize;
+        this.executorService = initExecutorService();
+        this.spiderMap = new ConcurrentHashMap<String, Spider>();
+    }
+
+    public Worker() {
+        this(DEFAULT_POOL_SIZE);
+    }
+
+    protected ExecutorService initExecutorService() {
+        return ThreadUtils.newFixedThreadPool(poolSize);
+    }
+
+    public void addSpider(Spider spider) {
+        spider.setExecutorService(executorService);
+        spiderMap.put(spider.getUUID(), spider);
+    }
+
+    public Spider getSpider(String uuid){
+        return spiderMap.get(uuid);
+    }
+
+}

webmagic-avalon/webmagic-worker/src/test/java/us/codecraft/webmagic/worker/WorkerTest.java

+package us.codecraft.webmagic.worker;
+
+import org.junit.Test;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.*;
+
+/**
+ * @author code4crafter@gmail.com
+ */
+public class WorkerTest {
+
+    @Test
+    public void testWorkerAsSpiderContains() throws Exception {
+        PageProcessor pageProcessor = mock(PageProcessor.class);
+        Site site = mock(Site.class);
+        when(pageProcessor.getSite()).thenReturn(site);
+        when(site.getDomain()).thenReturn("codecraft.us");
+        Worker worker = new Worker();
+        Spider spider = Spider.create(pageProcessor);
+        worker.addSpider(spider);
+        assertThat(worker.getSpider("codecraft.us")).isEqualTo(spider);
+    }
+}

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -643,6 +643,11 @@ public class Spider implements Runnable, Task {
         return uuid;
     }
 
+    public Spider setExecutorService(ExecutorService executorService) {
+        this.executorService = executorService;
+        return this;
+    }
+
     @Override
     public Site getSite() {
         return site;