Commit 8564d51e authored by yihua.huang's avatar yihua.huang

remove site.addStartUrl in samples

parent c13110c4
......@@ -21,7 +21,7 @@ public class DianpingFtlDataScanner implements AfterExtractor {
private List<String> data;
public static void main(String[] args) {
OOSpider.create(Site.me().addStartUrl("http://w.alpha.dp/").setSleepTime(0), DianpingFtlDataScanner.class)
OOSpider.create(Site.me().setSleepTime(0), DianpingFtlDataScanner.class)
.thread(5).run();
}
......
......@@ -41,9 +41,10 @@ public class GithubRepo implements HasKey {
private String url;
public static void main(String[] args) {
OOSpider.create(Site.me().addStartUrl("https://github.com/explore").setSleepTime(0).setRetryTimes(3),
OOSpider.create(Site.me().setSleepTime(0).setRetryTimes(3),
new JsonFilePageModelPipeline(), GithubRepo.class)
.scheduler(new FileCacheQueueScheduler("/data/webmagic/cache/")).thread(15).run();
.addUrl("https://github.com/explore")
.setScheduler(new FileCacheQueueScheduler("/data/webmagic/cache/")).thread(15).run();
}
@Override
......
......@@ -28,7 +28,7 @@ public class IteyeBlog implements Blog{
}
public static void main(String[] args) {
OOSpider.create(Site.me().addStartUrl("http://flashsword20.iteye.com/blog"), IteyeBlog.class).run();
OOSpider.create(Site.me(), IteyeBlog.class).addUrl("http://flashsword20.iteye.com/blog").run();
}
public String getTitle() {
......
......@@ -32,12 +32,12 @@ public class Kr36NewsModel {
public static void main(String[] args) throws IOException, JMException {
//Just for benchmark
Spider thread = OOSpider.create(Site.me().addStartUrl("http://www.36kr.com/").setSleepTime(0), new PageModelPipeline() {
Spider thread = OOSpider.create(Site.me().setSleepTime(0), new PageModelPipeline() {
@Override
public void process(Object o, Task task) {
}
}, Kr36NewsModel.class).thread(20);
}, Kr36NewsModel.class).thread(20).addUrl("http://www.36kr.com/");
thread.start();
SpiderMonitor spiderMonitor = SpiderMonitor.instance();
spiderMonitor.register(thread);
......
......@@ -22,7 +22,7 @@ public class OschinaAnswer implements AfterExtractor{
private String content;
public static void main(String[] args) {
OOSpider.create(Site.me().addStartUrl("http://www.oschina.net/question/567527_120597"), OschinaAnswer.class).run();
OOSpider.create(Site.me(), OschinaAnswer.class).addUrl("http://www.oschina.net/question/567527_120597").run();
}
@Override
......
......@@ -26,7 +26,7 @@ public class OschinaBlog{
public static void main(String[] args) {
OOSpider.create(Site.me()
.setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36").addStartUrl("http://my.oschina.net/flashsword/blog")
.setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36")
.setSleepTime(0)
.setRetryTimes(3)
,new PageModelPipeline() {
......@@ -34,7 +34,7 @@ public class OschinaBlog{
public void process(Object o, Task task) {
}
}, OschinaBlog.class).thread(10).run();
}, OschinaBlog.class).thread(10).addUrl("http://my.oschina.net/flashsword/blog").run();
}
public String getTitle() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment