add worker

pom.xml

@@ -12,6 +12,8 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <spring-version>3.1.1.RELEASE</spring-version>
+
     </properties>
     <artifactId>webmagic-parent</artifactId>
     <name>webmagic-parent</name>
@@ -49,6 +51,8 @@
         <module>webmagic-core</module>
         <module>webmagic-extension/</module>
         <module>webmagic-scripts/</module>
+        <module>webmagic-panel</module>
+        <module>webmagic-worker</module>
     </modules>
 
     <dependencyManagement>

webmagic-panel/README.md

+Worker:
+
+任务执行者，提供Http接口，监控运行状态，终止和开始job
+
+队列：
+
+仍然使用redis
+
+Panel:
+
+提供Web管理后台，管理
+
+
+
+1. 新建任务
+	1. 通过脚本
+	2. 配置
+	3. 分配机器
+2. 已有任务
+3. 任务查看
\ No newline at end of file

webmagic-panel/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>webmagic-parent</artifactId>
+        <groupId>us.codecraft</groupId>
+        <version>0.4.3-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>us.codecraft</groupId>
+    <artifactId>webmagic-panel</artifactId>
+    <version>0.4.3-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-scripts</artifactId>
+            <version>0.4.3-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+
+
+</project>
\ No newline at end of file

webmagic-worker/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>webmagic-parent</artifactId>
+        <groupId>us.codecraft</groupId>
+        <version>0.4.3-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>us.codecraft</groupId>
+    <artifactId>webmagic-worker</artifactId>
+    <version>0.4.3-SNAPSHOT</version>
+
+    <dependencies>
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-scripts</artifactId>
+            <version>0.4.3-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>express.java</artifactId>
+            <version>0.1.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-core</artifactId>
+            <version>${spring-version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-asm</artifactId>
+            <version>${spring-version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+            <version>${spring-version}</version>
+        </dependency>
+    </dependencies>
+
+
+</project>
\ No newline at end of file

webmagic-worker/src/main/java/us/codecraft/webmagic/worker/Bootstrap.java

+package us.codecraft.webmagic.worker;
+
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import us.codecraft.express.WebServer;
+
+/**
+ * @author code4crafter@gmail.com
+ */
+public class Bootstrap {
+
+    public static void main(String[] args) throws Exception {
+        ClassPathXmlApplicationContext classPathXmlApplicationContext = new ClassPathXmlApplicationContext(
+                new String[]{"classpath*:/spring/applicationContext-*.xml"}
+        );
+        WebServer webServer = classPathXmlApplicationContext.getBean(WebServer.class);
+        webServer.port(11111).start();
+    }
+
+}

webmagic-worker/src/main/java/us/codecraft/webmagic/worker/SpiderManager.java

+package us.codecraft.webmagic.worker;
+
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import us.codecraft.express.WebServer;
+import us.codecraft.express.controller.AjaxController;
+import us.codecraft.express.controller.ParamMap;
+import us.codecraft.express.controller.ResultMap;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.scripts.Language;
+import us.codecraft.webmagic.scripts.ScriptProcessor;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * @author code4crafter@gmail.com
+ */
+@Component
+public class SpiderManager implements InitializingBean {
+
+	@Autowired
+	private WebServer webServer;
+
+	private Map<String, Spider> spiderMap = new ConcurrentHashMap<String, Spider>();
+
+	public Spider newSpider(ParamMap params) {
+		Spider spider = Spider
+				.create(new ScriptProcessor(Language.JavaScript, params.get("script"), params.getInt("thread")))
+				.thread(params.getInt("thread")).addUrl(params.get("url"));
+		spider.start();
+		return spider;
+	}
+
+	@Override
+	public void afterPropertiesSet() throws Exception {
+		AjaxController newController = new AjaxController() {
+			@Override
+			public Object ajax(ParamMap params) {
+				try {
+					Spider spider = newSpider(params);
+					spiderMap.put(params.get("uuid"), spider);
+					return ResultMap.create().put("code", 200).put("msg", "success");
+				} catch (Exception e) {
+					// If you provide worker to user, DO NOT return
+					// e.getMessage()!
+					return ResultMap.create().put("code", 500).put("msg", e.getMessage());
+				}
+			}
+		};
+		webServer.post("/new/${uuid}", newController);
+		webServer.get("/new/${uuid}", newController);
+		webServer.get("/status/${uuid}", new AjaxController() {
+			@Override
+			public Object ajax(ParamMap params) {
+				Spider spider = spiderMap.get(params.get("uuid"));
+                ResultMap put = ResultMap.create().put("pageCount", spider.getPageCount())
+                        .put("status", spider.getStatus().name()).put("thread", spider.getThreadAlive());
+                return put;
+            }
+		});
+	}
+}

webmagic-worker/src/main/resources/spring/applicationContext-webmagic-worker.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
+        xmlns:util="http://www.springframework.org/schema/util"
+        xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+                http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
+                http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+        <context:annotation-config />
+
+        <context:component-scan base-package="us.codecraft.webmagic.worker" />
+        
+        <beans>
+        	<bean name="server" class="us.codecraft.express.connector.jetty.JettyWebServer">
+        	</bean>
+        </beans>
+
+
+</beans>