extension point and sample for JMX support #98

webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitor.java

@@ -21,31 +21,39 @@ import java.util.concurrent.atomic.AtomicInteger;
  * @author code4crafer@gmail.com
  * @since 0.5.0
  */
-public class SpiderMonitor implements SpiderMonitorMBean {
+public class SpiderMonitor {
 
-    private List<SpiderStatus> spiderStatuses = new ArrayList<SpiderStatus>();
+    private List<SpiderStatusMBean> spiderStatuses = new ArrayList<SpiderStatusMBean>();
 
-    @Override
-    public List<SpiderStatus> getSpiders() {
+    public List<SpiderStatusMBean> getSpiders() {
         return spiderStatuses;
     }
 
-    @Override
-    public SpiderStatus getSpider() {
+    public SpiderStatusMBean getSpider() {
         return spiderStatuses.get(0);
     }
 
-    public void register(Spider spider) {
-        MonitorSpiderListener monitorSpiderListener = new MonitorSpiderListener();
-        if (spider.getSpiderListeners() == null) {
-            List<SpiderListener> spiderListeners = new ArrayList<SpiderListener>();
-            spiderListeners.add(monitorSpiderListener);
-            spider.setSpiderListeners(spiderListeners);
-        } else {
-            spider.getSpiderListeners().add(monitorSpiderListener);
+    public SpiderMonitor register(Spider... spiders) {
+        for (Spider spider : spiders) {
+            MonitorSpiderListener monitorSpiderListener = new MonitorSpiderListener();
+            if (spider.getSpiderListeners() == null) {
+                List<SpiderListener> spiderListeners = new ArrayList<SpiderListener>();
+                spiderListeners.add(monitorSpiderListener);
+                spider.setSpiderListeners(spiderListeners);
+            } else {
+                spider.getSpiderListeners().add(monitorSpiderListener);
+            }
+            spiderStatuses.add(getSpiderStatusMBean(spider, monitorSpiderListener));
         }
-        spiderStatuses.add(new SpiderStatus(spider, monitorSpiderListener));
+        return this;
+    }
+
+    protected SpiderStatusMBean getSpiderStatusMBean(Spider spider, MonitorSpiderListener monitorSpiderListener) {
+        return new SpiderStatus(spider, monitorSpiderListener);
+    }
 
+    public static SpiderMonitor create(){
+        return new SpiderMonitor();
     }
 
     public class MonitorSpiderListener implements SpiderListener {
@@ -81,41 +89,44 @@ public class SpiderMonitor implements SpiderMonitorMBean {
     }
 
 
-    public static void main(String[] args) throws MalformedObjectNameException,
-            NullPointerException, InstanceAlreadyExistsException,
-            MBeanRegistrationException, NotCompliantMBeanException, IOException {
-
-        int rmiPort = 1099;
-        SpiderMonitor spiderMonitor = new SpiderMonitor();
-        String jmxServerName = "TestJMXServer";
-
-        Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor()).addUrl("http://my.oschina.net/flashsword/blog").thread(2);
-
-        spiderMonitor.register(oschinaSpider);
-
-        Spider githubSpider = Spider.create(new GithubRepoPageProcessor()).addUrl("https://github.com/code4craft");
-
-        spiderMonitor.register(githubSpider);
+    public void jmxStart() throws IOException, JMException {
+        jmxStart(14721);
+    }
 
+    public void jmxStart(int rmiPort) throws IOException, JMException {
+        String jmxServerName = "WebMagic";
         // jdkfolder/bin/rmiregistry.exe 9999
         Registry registry = LocateRegistry.createRegistry(rmiPort);
 
         MBeanServer mbs = MBeanServerFactory.createMBeanServer(jmxServerName);
         //MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
 
-        ObjectName objName = new ObjectName(jmxServerName + ":name=" + "HelloWorld");
-        mbs.registerMBean(spiderMonitor, objName);
+        ObjectName objName;
 
         JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://localhost:" + rmiPort + "/" + jmxServerName);
         System.out.println("JMXServiceURL: " + url.toString());
         JMXConnectorServer jmxConnServer = JMXConnectorServerFactory.newJMXConnectorServer(url, null, mbs);
         jmxConnServer.start();
 
-        for (SpiderStatus spiderStatuse : spiderMonitor.spiderStatuses) {
-            objName = new ObjectName(jmxServerName + ":name=" + spiderStatuse.getName());
-            mbs.registerMBean(spiderStatuse, objName);
+        for (SpiderStatusMBean spiderStatus : spiderStatuses) {
+            objName = new ObjectName(jmxServerName + ":name=" + spiderStatus.getName());
+            mbs.registerMBean(spiderStatus, objName);
         }
+    }
+
 
+    public static void main(String[] args) throws JMException,
+            NullPointerException,
+            IOException {
+
+        Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
+                .addUrl("http://my.oschina.net/flashsword/blog").thread(2);
+        Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
+                .addUrl("https://github.com/code4craft");
+
+        SpiderMonitor spiderMonitor = new SpiderMonitor();
+        spiderMonitor.register(oschinaSpider, githubSpider);
+        spiderMonitor.jmxStart();
 
     }
 

webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatus.java

 package us.codecraft.webmagic.monitor;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import us.codecraft.webmagic.Spider;
 
 import java.util.List;
@@ -8,11 +10,13 @@ import java.util.List;
  * @author code4crafer@gmail.com
  * @since 0.5.0
  */
-public class SpiderStatus implements SpiderStatusMBean{
+public class SpiderStatus implements SpiderStatusMBean {
 
-    private final Spider spider;
+    protected final Spider spider;
 
-    private final SpiderMonitor.MonitorSpiderListener monitorSpiderListener;
+    protected Logger logger = LoggerFactory.getLogger(getClass());
+
+    protected final SpiderMonitor.MonitorSpiderListener monitorSpiderListener;
 
     public SpiderStatus(Spider spider, SpiderMonitor.MonitorSpiderListener monitorSpiderListener) {
         this.spider = spider;
@@ -23,24 +27,46 @@ public class SpiderStatus implements SpiderStatusMBean{
         return spider.getUUID();
     }
 
-    public int getLeftPages() {
+    public int getLeftPageCount() {
         if (spider.getScheduler() instanceof MonitorableScheduler) {
             return ((MonitorableScheduler) spider.getScheduler()).getLeftRequestsCount(spider);
         }
+        logger.warn("Get leftPageCount fail, try to use a Scheduler implement MonitorableScheduler for monitor count!");
         return -1;
     }
 
-    public int getTotalPages() {
+    public int getTotalPageCount() {
         if (spider.getScheduler() instanceof MonitorableScheduler) {
             return ((MonitorableScheduler) spider.getScheduler()).getTotalRequestsCount(spider);
         }
+        logger.warn("Get totalPageCount fail, try to use a Scheduler implement MonitorableScheduler for monitor count!");
         return -1;
     }
 
+    @Override
+    public int getSuccessPageCount() {
+        return monitorSpiderListener.getSuccessCount().get();
+    }
+
+    @Override
+    public int getErrorPageCount() {
+        return monitorSpiderListener.getErrorCount().get();
+    }
+
     public List<String> getErrorPages() {
         return monitorSpiderListener.getErrorUrls();
     }
 
+    @Override
+    public String getStatus() {
+        return spider.getStatus().name();
+    }
+
+    @Override
+    public int getThread() {
+        return spider.getThreadAlive();
+    }
+
     public void start() {
         spider.start();
     }

webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatusMBean.java

@@ -10,13 +10,21 @@ public interface SpiderStatusMBean {
 
     public String getName();
 
-    public int getLeftPages();
+    public String getStatus();
+
+    public int getThread();
+
+    public int getTotalPageCount();
+
+    public int getLeftPageCount();
+
+    public int getSuccessPageCount();
+
+    public int getErrorPageCount();
 
-    public int getTotalPages();
     public List<String> getErrorPages();
 
     public void start();
 
     public void stop();
-
 }

webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatus.java

+package us.codecraft.webmagic.monitor;
+
+import us.codecraft.webmagic.Spider;
+
+/**
+ * @author code4crafer@gmail.com
+ */
+public class CustomSpiderStatus extends SpiderStatus implements CustomSpiderStatusMBean {
+
+    public CustomSpiderStatus(Spider spider, SpiderMonitor.MonitorSpiderListener monitorSpiderListener) {
+        super(spider, monitorSpiderListener);
+    }
+
+
+    @Override
+    public String getSchedulerName() {
+        return spider.getScheduler().getClass().getName();
+    }
+}

webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitorMBean.java → webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatusMBean.java

 package us.codecraft.webmagic.monitor;
 
-import java.util.List;
-
 /**
  * @author code4crafer@gmail.com
  */
-public interface SpiderMonitorMBean {
-
-    public List<SpiderStatus> getSpiders();
+public interface CustomSpiderStatusMBean extends SpiderStatusMBean {
 
-    public SpiderStatus getSpider();
+    public String getSchedulerName();
 
 }

webmagic-core/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java

+package us.codecraft.webmagic.monitor;
+
+import org.junit.Test;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;
+import us.codecraft.webmagic.processor.example.OschinaBlogPageProcessor;
+
+/**
+ * @author code4crafer@gmail.com
+ * @since 0.5.0
+ */
+public class SpiderMonitorTest {
+
+    @Test
+    public void testInherit() throws Exception {
+        SpiderMonitor spiderMonitor = new SpiderMonitor(){
+            @Override
+            protected SpiderStatusMBean getSpiderStatusMBean(Spider spider, MonitorSpiderListener monitorSpiderListener) {
+                return new CustomSpiderStatus(spider, monitorSpiderListener);
+            }
+        };
+
+        Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
+                .addUrl("http://my.oschina.net/flashsword/blog").thread(2);
+        Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
+                .addUrl("https://github.com/code4craft");
+
+        spiderMonitor.register(oschinaSpider, githubSpider);
+        spiderMonitor.jmxStart();
+
+    }
+}