update pom

webmagic-scripts/pom.xml

@@ -31,6 +31,11 @@
             <artifactId>webmagic-core</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>us.codecraft</groupId>
+            <artifactId>webmagic-extension</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 
     <build>

webmagic-scripts/src/main/java/us/codecraft/webmagic/scripts/ScriptConsole.java

@@ -92,8 +92,8 @@ public class ScriptConsole {
         ScriptProcessor pageProcessor = ScriptProcessorBuilder.custom()
                 .language(params.getLanguage()).scriptFromFile(params.getScriptFileName()).thread(params.getThread()).build();
         pageProcessor.getSite().setSleepTime(params.getSleepTime());
-        pageProcessor.getSite().setAcceptStatCode(Sets.<Integer>newHashSet(200, 404, 500));
-        pageProcessor.getSite().setUserAgent("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5");
+        pageProcessor.getSite().setRetryTimes(3);
+        pageProcessor.getSite().setAcceptStatCode(Sets.<Integer>newHashSet(200, 404,403, 500,502));
         Spider spider = Spider.create(pageProcessor).thread(params.getThread());
         spider.clearPipeline().addPipeline(new Pipeline() {
             @Override