add timeout for wait/notify #111

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -20,6 +20,7 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.Condition;
@@ -104,6 +105,8 @@ public class Spider implements Runnable, Task {
 
     private Date startTime;
 
+    private int emptySleepTime = 30000;
+
     /**
      * create a spider with pageProcessor.
      *
@@ -524,7 +527,7 @@ public class Spider implements Runnable, Task {
             if (threadPool.getThreadAlive() == 0 && exitWhenComplete) {
                 return;
             }
-            newUrlCondition.await();
+            newUrlCondition.await(emptySleepTime, TimeUnit.MILLISECONDS);
         } catch (InterruptedException e) {
             logger.warn("waitNewUrl - interrupted, error {}", e);
         } finally {
@@ -716,4 +719,13 @@ public class Spider implements Runnable, Task {
     public Scheduler getScheduler() {
         return scheduler;
     }
+
+    /**
+     * Set wait time when no url is polled.<br></br>
+     *
+     * @param emptySleepTime In MILLISECONDS.
+     */
+    public void setEmptySleepTime(int emptySleepTime) {
+        this.emptySleepTime = emptySleepTime;
+    }
 }

webmagic-core/src/main/java/us/codecraft/webmagic/selector/thread/CountableThreadPool.java

@@ -51,9 +51,10 @@ public class CountableThreadPool {
     private ExecutorService executorService;
 
     public void execute(final Runnable runnable) {
-        try {
+
 
         if (threadAlive.get() >= threadNum) {
+            try {
                 reentrantLock.lock();
                 while (threadAlive.get() >= threadNum) {
                     try {
@@ -61,6 +62,9 @@ public class CountableThreadPool {
                     } catch (InterruptedException e) {
                     }
                 }
+            } finally {
+                reentrantLock.unlock();
+            }
         }
         threadAlive.incrementAndGet();
         executorService.execute(new Runnable() {
@@ -79,11 +83,6 @@ public class CountableThreadPool {
                 }
             }
         });
-        } finally {
-            if (reentrantLock.isLocked()) {
-                reentrantLock.unlock();
-            }
-        }
     }
 
     public boolean isShutdown() {

webmagic-core/src/test/java/us/codecraft/webmagic/SpiderTest.java

@@ -37,7 +37,7 @@ public class SpiderTest {
     @Test
     public void testWaitAndNotify() throws InterruptedException {
         for (int i = 0; i < 10000; i++) {
-            System.out.println("round" + i);
+            System.out.println("round " + i);
             testRound();
         }
     }