todo bugfix

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -482,7 +482,7 @@ public class Spider implements Runnable, Task {
     /**
      * Exit when complete. <br/>
      * True: exit when all url of the site is downloaded. <br/>
-     * False: not exit until call stop manually.<br/>
+     * False: not exit until call stop() manually.<br/>
      *
      * @param exitWhenComplete
      * @return

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -90,11 +90,15 @@ public class HttpClientDownloader implements Downloader {
         HttpClient httpClient = getHttpClientPool().getClient(site);
         try {
             HttpGet httpGet = new HttpGet(request.getUrl());
+
             if (headers!=null){
                 for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
                     httpGet.addHeader(headerEntry.getKey(),headerEntry.getValue());
                 }
             }
+            if (!httpGet.containsHeader("Accept-Encoding")) {
+                httpGet.addHeader("Accept-Encoding", "gzip");
+            }
             HttpResponse httpResponse = null;
             int tried = 0;
             boolean retry;
@@ -168,6 +172,7 @@ public class HttpClientDownloader implements Downloader {
             HeaderElement[] codecs = ceheader.getElements();
             for (HeaderElement codec : codecs) {
                 if (codec.getName().equalsIgnoreCase("gzip")) {
+                    //todo bugfix
                     httpResponse.setEntity(
                             new GzipDecompressingEntity(httpResponse.getEntity()));
                 }