Commit be9130f3 authored by shenjunlin's avatar shenjunlin

fix bug

parent cf1b6113
......@@ -6,7 +6,7 @@
<version>7</version>
</parent>
<groupId>us.codecraft.duiba</groupId>
<version>0.7.8-sjl-SNAPSHOT</version>
<version>0.7.9-102-SNAPSHOT</version>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
<properties>
......
......@@ -3,7 +3,7 @@
<parent>
<groupId>us.codecraft.duiba</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.7.8-sjl-SNAPSHOT</version>
<version>0.7.9-102-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -305,6 +305,7 @@ public class Spider implements Runnable, Task {
initComponent();
logger.info("Spider {} started!",getUUID());
while (!Thread.currentThread().isInterrupted() && stat.get() == STAT_RUNNING) {
//while (!Thread.interrupted() && stat.get() == STAT_RUNNING) {
final Request request = scheduler.poll(this);
if (request == null) {
if (threadPool.getThreadAlive() == 0 && exitWhenComplete) {
......@@ -453,6 +454,7 @@ public class Spider implements Runnable, Task {
Thread.sleep(time);
} catch (InterruptedException e) {
logger.error("Thread interrupted when sleep",e);
Thread.currentThread().interrupt();
}
}
......@@ -559,6 +561,7 @@ public class Spider implements Runnable, Task {
newUrlCondition.await(emptySleepTime, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
logger.warn("waitNewUrl - interrupted, error {}", e);
Thread.currentThread().interrupt();
} finally {
newUrlLock.unlock();
}
......
......@@ -60,6 +60,7 @@ public class CountableThreadPool {
try {
condition.await();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
} finally {
......
......@@ -3,7 +3,7 @@
<parent>
<groupId>us.codecraft.duiba</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.7.8-sjl-SNAPSHOT</version>
<version>0.7.9-102-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -3,7 +3,7 @@
<parent>
<groupId>us.codecraft.duiba</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.7.8-sjl-SNAPSHOT</version>
<version>0.7.9-102-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -86,7 +86,9 @@ public class AnjukeSpider implements PageProcessor {
.addUrl("https://www.anjuke.com/sy-city.html")
// .addUrl("https://lvliang.anjuke.com/community/p1/")
.thread(2).addPipeline(new ConsolePipeline());
anjuke.start();
anjuke.run();
Thread.currentThread().interrupt();
}
}
......@@ -3,7 +3,7 @@
<parent>
<groupId>us.codecraft.duiba</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.7.8-sjl-SNAPSHOT</version>
<version>0.7.9-102-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -3,7 +3,7 @@
<parent>
<groupId>us.codecraft.duiba</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.7.8-sjl-SNAPSHOT</version>
<version>0.7.9-102-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment