fix bg

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -88,6 +88,10 @@ public class HttpClientDownloader extends AbstractDownloader {
         } catch (IOException e) {
             logger.warn("download page {} error", request.getUrl(), e);
             onError(request);
+            //移除代理
+            if (proxyProvider != null && proxy != null) {
+                proxyProvider.deleteProxy(proxy);
+            }
             return page;
         } finally {
             if (httpResponse != null) {

webmagic-core/src/main/java/us/codecraft/webmagic/proxy/ProxyProvider.java

@@ -26,4 +26,10 @@ public interface ProxyProvider {
      */
     Proxy getProxy(Task task);
 
+    /**
+     * 代理不可用时，移除代理
+     * @param proxy
+     */
+    void deleteProxy(Proxy proxy);
+    
 }

webmagic-core/src/main/java/us/codecraft/webmagic/proxy/SimpleProxyProvider.java

@@ -64,4 +64,9 @@ public class SimpleProxyProvider implements ProxyProvider {
         }
         return p % size;
     }
+
+    @Override
+    public void deleteProxy(Proxy proxy) {
+        proxies.remove(proxy);
+    }
 }

webmagic-extension/src/main/java/us/codecraft/webmagic/proxy/Data5UProxyProvider.java

@@ -65,7 +65,12 @@ public class Data5UProxyProvider implements ProxyProvider{
         return proxy;
     }
 
-//    public static void main(String[] args) {
+    @Override
+    public void deleteProxy(Proxy proxy) {
+        proxyCache.cleanUp();
+    }
+
+    //    public static void main(String[] args) {
 //         Data5UProxyProvider proxyProvider = new Data5UProxyProvider("b323ab32e6ca1be6b803291943d47b62");
 //        try {
 //            System.out.println(proxyProvider.getProxyFromData5u());