Merge pull request #233 from x1ny/master

修正FileCacheQueueScheduler导致程序不能正常结束和未关闭流

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -374,6 +374,7 @@ public class Spider implements Runnable, Task {
     public void close() {
         destroyEach(downloader);
         destroyEach(pageProcessor);
+        destroyEach(scheduler);
         for (Pipeline pipeline : pipelines) {
             destroyEach(pipeline);
         }

webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java

@@ -12,17 +12,19 @@ import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
+
 /**
  * Store urls and cursor in files so that a Spider can resume the status when shutdown.<br>
  *
  * @author code4crafter@gmail.com <br>
  * @since 0.2.0
  */
-public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {
+public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler,Closeable {
 
     private String filePath = System.getProperty("java.io.tmpdir");
 
@@ -44,6 +46,8 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
 
     private Set<String> urls;
     
+    private ScheduledExecutorService flushThreadPool;
+
     public FileCacheQueueScheduler(String filePath) {
         if (!filePath.endsWith("/") && !filePath.endsWith("\\")) {
             filePath += "/";
@@ -94,7 +98,8 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
     }
 
     private void initFlushThread() {
-        Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
+    	flushThreadPool = Executors.newScheduledThreadPool(1);
+    	flushThreadPool.scheduleAtFixedRate(new Runnable() {
             @Override
             public void run() {
                 flush();
@@ -162,6 +167,12 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
         }
     }
     
+    public void close() throws IOException {
+		flushThreadPool.shutdown();	
+		fileUrlWriter.close();
+		fileCursorWriter.close();
+	}
+
     private String getFileName(String filename) {
         return filePath + task.getUUID() + filename;
     }