Commit 3a666fce authored by yihua.huang's avatar yihua.huang

add sample of 36kr #110

parent b1258f4f
package us.codecraft.webmagic.model.samples; package us.codecraft.webmagic.model.samples;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task; import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.model.OOSpider; import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.pipeline.PageModelPipeline; import us.codecraft.webmagic.pipeline.PageModelPipeline;
import us.codecraft.webmagic.model.annotation.ExtractBy; import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.model.annotation.ExtractByUrl; import us.codecraft.webmagic.model.annotation.ExtractByUrl;
import us.codecraft.webmagic.model.annotation.HelpUrl; import us.codecraft.webmagic.model.annotation.HelpUrl;
import us.codecraft.webmagic.model.annotation.TargetUrl; import us.codecraft.webmagic.model.annotation.TargetUrl;
import javax.management.JMException;
import java.io.IOException;
/** /**
* @author code4crafter@gmail.com <br> * @author code4crafter@gmail.com <br>
*/ */
...@@ -25,14 +30,17 @@ public class Kr36NewsModel { ...@@ -25,14 +30,17 @@ public class Kr36NewsModel {
@ExtractByUrl @ExtractByUrl
private String url; private String url;
public static void main(String[] args) { public static void main(String[] args) throws IOException, JMException {
//Just for benchmark //Just for benchmark
OOSpider.create(Site.me().addStartUrl("http://www.36kr.com/").setSleepTime(0), new PageModelPipeline() { Spider thread = OOSpider.create(Site.me().addStartUrl("http://www.36kr.com/").setSleepTime(0), new PageModelPipeline() {
@Override @Override
public void process(Object o, Task task) { public void process(Object o, Task task) {
} }
},Kr36NewsModel.class).thread(20).run(); }, Kr36NewsModel.class).thread(20);
thread.run();
SpiderMonitor spiderMonitor = SpiderMonitor.create();
spiderMonitor.register(thread).jmxStart();
} }
public String getTitle() { public String getTitle() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment