添加data5U动态代理

webmagic-core/src/main/java/us/codecraft/webmagic/utils/HttpClientUtils.java

 package us.codecraft.webmagic.utils;
 
 import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -25,4 +32,12 @@ public abstract class HttpClientUtils {
         }
         return results;
     }
+
+    public static String doGet(String url) throws IOException {
+        CloseableHttpClient httpClient= HttpClients.createDefault();
+        HttpGet httpget = new HttpGet(url);
+        CloseableHttpResponse response = httpClient.execute(httpget);
+        HttpEntity httpEntity= response.getEntity();
+        return EntityUtils.toString(httpEntity);
+    }
 }

webmagic-core/src/main/resources/log4j.xml

@@ -9,7 +9,7 @@
     </appender>
 
     <logger name="org.apache" additivity="false">
-        <level value="warn" />
+        <level value="error" />
         <appender-ref ref="stdout" />
     </logger>
 

webmagic-extension/src/main/java/us/codecraft/webmagic/proxy/Data5UProxyProvider.java

+package us.codecraft.webmagic.proxy;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Task;
+import us.codecraft.webmagic.utils.HttpClientUtils;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * data5U的动态代理
+ * 官方网址：http://www.data5u.com/
+ * 使用时，建议选择1分钟有效期，并且不带ttl参数
+ */
+public class Data5UProxyProvider implements ProxyProvider{
+
+    private Logger logger = LoggerFactory.getLogger(getClass());
+
+    private static Cache<String, Proxy> proxyCache = CacheBuilder.newBuilder()
+            .expireAfterWrite(50, TimeUnit.SECONDS)  //20秒过期，Data5u提供的代理有1分钟过期，2分钟过期，5分钟过期
+            .maximumSize(10).build();
+
+    private String data5UApiUrl;
+
+    public Data5UProxyProvider(String orderId){
+        this.data5UApiUrl = "http://api.ip.data5u.com/dynamic/get.html?order=" + orderId;
+    }
+
+    @Override
+    public void returnProxy(Proxy proxy, Page page, Task task) {
+        //do nothing
+    }
+
+    @Override
+    public Proxy getProxy(Task task) {
+        final String key = "proxy";
+        try {
+            return proxyCache.get(key, () -> getProxyFromData5u());
+        } catch (Exception e) {
+            logger.warn("get proxy from data5U fail {}", e);
+            return null;
+        }
+    }
+
+    //返回结果 221.32.22.54:33084
+    private Proxy getProxyFromData5u() throws IOException {
+        String result = HttpClientUtils.doGet(data5UApiUrl);
+        String[] res = StringUtils.split(result, "\n");
+        String[] ipAndPort = StringUtils.split(res[0], ":");
+        return new Proxy(ipAndPort[0], Integer.valueOf(ipAndPort[1]));
+    }
+}

webmagic-extension/src/main/resources/log4j.xml

@@ -9,7 +9,7 @@
     </appender>
 
     <logger name="org.apache" additivity="false">
-        <level value="warn" />
+        <level value="error" />
         <appender-ref ref="stdout" />
     </logger>