Commit f9b15795 authored by yihua.huang's avatar yihua.huang

Merge branch 'master' of github.com:code4craft/webmagic

parents 22c394e6 3efa7741
......@@ -23,8 +23,9 @@ public abstract class LocalDuplicatedRemovedScheduler implements Scheduler {
@Override
public void push(Request request, Task task) {
logger.debug("push to queue " + request.getUrl());
logger.trace("get a candidate url " + request.getUrl());
if (request.getExtra(Request.CYCLE_TRIED_TIMES) != null || urls.add(request.getUrl())) {
logger.debug("push to queue " + request.getUrl());
pushWhenNoDuplicate(request, task);
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment