Merge pull request #188 from EdwardsBean/retry_time

add retry sleep time

webmagic-core/src/main/java/us/codecraft/webmagic/Site.java

@@ -39,6 +39,8 @@ public class Site {
 
     private int cycleRetryTimes = 0;
 
+    private int retrySleepTime = 1000;
+
     private int timeOut = 5000;
 
     private static final Set<Integer> DEFAULT_STATUS_CODE_SET = new HashSet<Integer>();
@@ -359,6 +361,20 @@ public class Site {
         return useGzip;
     }
 
+    public int getRetrySleepTime() {
+        return retrySleepTime;
+    }
+
+    /**
+     * Set retry sleep times when download fail, 1000 by default. <br>
+     *
+     * @param retrySleepTime
+     */
+    public Site setRetrySleepTime(int retrySleepTime) {
+        this.retrySleepTime = retrySleepTime;
+        return this;
+    }
+
     /**
      * Whether use gzip. <br>
      * Default is true, you can set it to false to disable gzip.

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -407,14 +407,14 @@ public class Spider implements Runnable, Task {
     protected void processRequest(Request request) {
         Page page = downloader.download(request, this);
         if (page == null) {
-            sleep(site.getSleepTime());
+            sleep(site.getRetrySleepTime());
             onError(request);
             return;
         }
         // for cycle retry
         if (page.isNeedCycleRetry()) {
             extractAndAddRequests(page, true);
-            sleep(site.getSleepTime());
+            sleep(site.getRetrySleepTime());
             return;
         }
         pageProcessor.process(page);