add cookie supoort

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -3,6 +3,7 @@ package us.codecraft.webmagic.downloader;
 import com.google.common.collect.Sets;
 import org.apache.http.HttpResponse;
 import org.apache.http.annotation.ThreadSafe;
+import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.RequestBuilder;
@@ -102,8 +103,9 @@ public class HttpClientDownloader implements Downloader {
         }
         RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
                 .setConnectionRequestTimeout(site.getTimeOut())
-                .setConnectTimeout(site.getTimeOut());
-        if (site.getHttpProxy()!=null){
+                .setConnectTimeout(site.getTimeOut())
+                .setCookieSpec(CookieSpecs.BEST_MATCH);
+        if (site.getHttpProxy() != null) {
             requestConfigBuilder.setProxy(site.getHttpProxy());
         }
         requestBuilder.setConfig(requestConfigBuilder.build());

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java

@@ -81,10 +81,11 @@ public class HttpClientPool {
         if (site!=null){
             httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(),true));
         }
+        generateCookie(httpClientBuilder,site);
         return httpClientBuilder.build();
     }
 
-    private void generateCookie(DefaultHttpClient httpClient, Site site) {
+    private void generateCookie(HttpClientBuilder httpClientBuilder, Site site) {
         CookieStore cookieStore = new BasicCookieStore();
         if (site.getCookies() != null) {
             for (Map.Entry<String, String> cookieEntry : site.getCookies().entrySet()) {
@@ -93,7 +94,7 @@ public class HttpClientPool {
                 cookieStore.addCookie(cookie);
             }
         }
-        httpClient.setCookieStore(cookieStore);
+        httpClientBuilder.setDefaultCookieStore(cookieStore);
     }
 
 }