修改获取代理的方式

webmagic-core/src/main/java/us/codecraft/webmagic/utils/ProxyUtils.java

@@ -20,11 +20,15 @@ public class ProxyUtils {
 	private static final Logger logger = LoggerFactory.getLogger(ProxyUtils.class);
 
 	public static boolean validateProxy(Proxy p) {
+		if (p == null) {
+			return false;
+		}
+
 		Socket socket = null;
 		try {
 			socket = new Socket();
 			InetSocketAddress endpointSocketAddr = new InetSocketAddress(p.getHost(), p.getPort());
-			socket.connect(endpointSocketAddr, 100);
+			socket.connect(endpointSocketAddr, 400);
 			return true;
 		} catch (IOException e) {
 			logger.warn("FAILRE - CAN not connect!  remote: " + p);
@@ -38,7 +42,6 @@ public class ProxyUtils {
 				}
 			}
 		}
-
 	}
 
 }

webmagic-extension/src/main/java/us/codecraft/webmagic/proxy/Data5UProxyProvider.java

@@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.utils.HttpClientUtils;
+import us.codecraft.webmagic.utils.ProxyUtils;
 
 import java.util.concurrent.TimeUnit;
 
@@ -21,7 +22,7 @@ public class Data5UProxyProvider implements ProxyProvider{
     private Logger logger = LoggerFactory.getLogger(getClass());
 
     private static Cache<String, Proxy> proxyCache = CacheBuilder.newBuilder()
-            .expireAfterWrite(50, TimeUnit.SECONDS)  //20秒过期，Data5u提供的代理有1分钟过期，2分钟过期，5分钟过期
+            .expireAfterWrite(30, TimeUnit.SECONDS)  //20秒过期，Data5u提供的代理有1分钟过期，2分钟过期，5分钟过期
             .maximumSize(10).build();
 
     private String data5UApiUrl;
@@ -48,12 +49,29 @@ public class Data5UProxyProvider implements ProxyProvider{
 
     //返回结果 221.32.22.54:33084
     private Proxy getProxyFromData5u() throws Exception {
-        String result = HttpClientUtils.doGet(data5UApiUrl);
-        if (StringUtils.contains(result,"false")) {
-            throw new Exception("data5U orderId is not correct");
-        }
-        String[] res = StringUtils.split(result, "\n");
-        String[] ipAndPort = StringUtils.split(res[0], ":");
-        return new Proxy(ipAndPort[0], Integer.valueOf(ipAndPort[1]));
+        Proxy proxy;
+
+        do {
+
+            String result = HttpClientUtils.doGet(data5UApiUrl);
+            if (StringUtils.contains(result,"false")) {
+                throw new Exception("data5U orderId is not correct");
+            }
+            String[] res = StringUtils.split(result, "\n");
+            String[] ipAndPort = StringUtils.split(res[0], ":");
+            proxy = new Proxy(ipAndPort[0], Integer.valueOf(ipAndPort[1]));
+
+        } while (!ProxyUtils.validateProxy(proxy));
+        return proxy;
     }
+
+//    public static void main(String[] args) {
+//         Data5UProxyProvider proxyProvider = new Data5UProxyProvider("b323ab32e6ca1be6b803291943d47b62");
+//        try {
+//            System.out.println(proxyProvider.getProxyFromData5u());
+//        } catch (Exception e) {
+//            e.printStackTrace();
+//        }
+//
+//    }
 }