Merge pull request #157 from zhugw/patch-1

Update FileCacheQueueScheduler.java

Merge pull request #157 from zhugw/patch-1
Update FileCacheQueueScheduler.java
689e89a9 · Yihua Huang · 147401ce · 1db940a0 · 689e89a9
Commit 689e89a9 authored Sep 11, 2014 by Yihua Huang
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 0 deletions

FileCacheQueueScheduler.java ...codecraft/webmagic/scheduler/FileCacheQueueScheduler.java +2 -0

No files found.
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
@@ -145,6 +145,8 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
        if (!inited.get()) {
            init(task);
        }
+        if(urls.contains(request.getUrl())) //已存在此URL 表示已抓取过 跳过
+            return;
        queue.add(request);
        fileUrlWriter.println(request.getUrl());
    }