修改代理

pom.xml

@@ -6,7 +6,7 @@
         <version>7</version>
     </parent>
     <groupId>us.codecraft.duiba</groupId>
-    <version>0.8.8-SNAPSHOT</version>
+    <version>0.9.0-SNAPSHOT</version>
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>
     <properties>

webmagic-core/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.8-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -306,29 +306,33 @@ public class Spider implements Runnable, Task {
         logger.info("Spider {} started!",getUUID());
         while (!Thread.currentThread().isInterrupted() && stat.get() == STAT_RUNNING) {
         //while (!Thread.interrupted() && stat.get() == STAT_RUNNING) {
-            final Request request = scheduler.poll(this);
-            if (request == null) {
-                if (threadPool.getThreadAlive() == 0 && exitWhenComplete) {
-                    break;
-                }
-                // wait until new url added
-                waitNewUrl();
-            } else {
-                threadPool.execute(new Runnable() {
-                    @Override
-                    public void run() {
-                        try {
-                            processRequest(request);
-                            onSuccess(request);
-                        } catch (Exception e) {
-                            onError(request);
-                            logger.error("process request " + request + " error", e);
-                        } finally {
-                            pageCount.incrementAndGet();
-                            signalNewUrl();
-                        }
+            try {
+                final Request request = scheduler.poll(this);
+                if (request == null) {
+                    if (threadPool.getThreadAlive() == 0 && exitWhenComplete) {
+                        break;
                     }
-                });
+                    // wait until new url added
+                    waitNewUrl();
+                } else {
+                    threadPool.execute(new Runnable() {
+                        @Override
+                        public void run() {
+                            try {
+                                processRequest(request);
+                                onSuccess(request);
+                            } catch (Exception e) {
+                                onError(request);
+                                logger.error("process request " + request + " error", e);
+                            } finally {
+                                pageCount.incrementAndGet();
+                                signalNewUrl();
+                            }
+                        }
+                    });
+                }
+            } catch (Exception e) {
+                logger.error(e.getMessage(), e);
             }
         }
         stat.set(STAT_STOPPED);

webmagic-extension/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.8-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

webmagic-extension/src/main/java/us/codecraft/webmagic/proxy/Data5UProxyProvider.java

@@ -22,7 +22,7 @@ public class Data5UProxyProvider implements ProxyProvider{
     private Logger logger = LoggerFactory.getLogger(getClass());
 
     private static Cache<String, Proxy> proxyCache = CacheBuilder.newBuilder()
-            .expireAfterWrite(30, TimeUnit.SECONDS)  //20秒过期，Data5u提供的代理有1分钟过期，2分钟过期，5分钟过期
+            .expireAfterWrite(5, TimeUnit.SECONDS)  //5秒过期，Data5u提供的代理有1分钟过期，2分钟过期，5分钟过期
             .maximumSize(10).build();
 
     private String data5UApiUrl;

webmagic-samples/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.8-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

webmagic-saxon/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.8-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

webmagic-scripts/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.8-SNAPSHOT</version>
+        <version>0.9.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>