Merge branch 'kapsterio-fix'

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -325,10 +325,6 @@ public class Spider implements Runnable, Task {
                             onError(requestFinal);
                             logger.error("process request " + requestFinal + " error", e);
                         } finally {
-                            if (site.getHttpProxyPool()!=null && site.getHttpProxyPool().isEnable()) {
-                                site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal
-                                        .getExtra(Request.STATUS_CODE));
-                            }
                             pageCount.incrementAndGet();
                             signalNewUrl();
                         }
@@ -408,9 +404,7 @@ public class Spider implements Runnable, Task {
     protected void processRequest(Request request) {
         Page page = downloader.download(request, this);
         if (page == null) {
-            sleep(site.getRetrySleepTime());
-            onError(request);
-            return;
+            throw new RuntimeException("unaccpetable response status");
         }
         // for cycle retry
         if (page.isNeedCycleRetry()) {

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -119,6 +119,10 @@ public class HttpClientDownloader extends AbstractDownloader {
             return null;
         } finally {
         	request.putExtra(Request.STATUS_CODE, statusCode);
+            if (site.getHttpProxyPool()!=null && site.getHttpProxyPool().isEnable()) {
+                site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request
+                        .getExtra(Request.STATUS_CODE));
+            }
             try {
                 if (httpResponse != null) {
                     //ensure the connection is released back to pool