change return proxy from spider to httpclientdownloader #128

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -2,7 +2,6 @@ package us.codecraft.webmagic;
 
 import com.google.common.collect.Lists;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.http.HttpHost;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import us.codecraft.webmagic.downloader.Downloader;
@@ -325,10 +324,6 @@ public class Spider implements Runnable, Task {
                             onError(requestFinal);
                             logger.error("process request " + requestFinal + " error", e);
                         } finally {
-							if (site.getHttpProxyPool().isEnable()) {
-								site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal
-										.getExtra(Request.STATUS_CODE));
-							}
                             pageCount.incrementAndGet();
                             signalNewUrl();
                         }

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -117,6 +117,10 @@ public class HttpClientDownloader extends AbstractDownloader {
             } catch (IOException e) {
                 logger.warn("close response fail", e);
             }
+            if (site.getHttpProxyPool().isEnable()) {
+                site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request
+                        .getExtra(Request.STATUS_CODE));
+            }
         }
     }