#627 set charset to request

webmagic-core/src/main/java/us/codecraft/webmagic/Request.java

@@ -51,6 +51,8 @@ public class Request implements Serializable {
      */
     private boolean binaryContent = false;
 
+    private String charset;
+
     public Request() {
     }
 
@@ -176,6 +178,14 @@ public class Request implements Serializable {
         this.binaryContent = binaryContent;
     }
 
+    public String getCharset() {
+        return charset;
+    }
+
+    public void setCharset(String charset) {
+        this.charset = charset;
+    }
+
     @Override
     public String toString() {
         return "Request{" +

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -83,7 +83,7 @@ public class HttpClientDownloader extends AbstractDownloader {
         Page page = Page.fail();
         try {
             httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
-            page = handleResponse(request, task.getSite().getCharset(), httpResponse, task);
+            page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
             onSuccess(request);
             logger.info("downloading page success {}", request.getUrl());
             return page;

webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java

@@ -289,4 +289,37 @@ public class HttpClientDownloaderTest {
         });
     }
 
+    @Test
+    public void test_download_set_charset() throws Exception {
+        HttpServer server = httpServer(13423);
+        server.response(header("Content-Type","text/html; charset=utf-8")).response("hello world!");
+        Runner.running(server, new Runnable() {
+            @Override
+            public void run() throws Exception {
+                final HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
+                Request request = new Request();
+                request.setUrl("http://127.0.0.1:13423/");
+                Page page = httpClientDownloader.download(request, Site.me().toTask());
+                assertThat(page.getCharset()).isEqualTo("utf-8");
+            }
+        });
+    }
+
+    @Test
+    public void test_download_set_request_charset() throws Exception {
+        HttpServer server = httpServer(13423);
+        server.response("hello world!");
+        Runner.running(server, new Runnable() {
+            @Override
+            public void run() throws Exception {
+                final HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
+                Request request = new Request();
+                request.setCharset("utf-8");
+                request.setUrl("http://127.0.0.1:13423/");
+                Page page = httpClientDownloader.download(request, Site.me().setCharset("gbk").toTask());
+                assertThat(page.getCharset()).isEqualTo("utf-8");
+            }
+        });
+    }
+
 }