添加等待

pom.xml

@@ -6,7 +6,7 @@
         <version>7</version>
     </parent>
     <groupId>us.codecraft.duiba</groupId>
-    <version>0.8.3-SNAPSHOT</version>
+    <version>0.8.4-SNAPSHOT</version>
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>
     <properties>

webmagic-core/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.3-SNAPSHOT</version>
+        <version>0.8.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

webmagic-extension/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.3-SNAPSHOT</version>
+        <version>0.8.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/SeleniumDownloader.java

 package us.codecraft.webmagic.downloader;
 
+import com.sun.tools.internal.xjc.reader.xmlschema.BindYellow;
 import io.github.bonigarcia.wdm.ChromeDriverManager;
 import io.github.bonigarcia.wdm.InternetExplorerDriverManager;
 import io.github.bonigarcia.wdm.PhantomJsDriverManager;
@@ -8,6 +9,8 @@ import org.openqa.selenium.By;
 import org.openqa.selenium.Cookie;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
+import org.openqa.selenium.support.ui.ExpectedConditions;
+import org.openqa.selenium.support.ui.WebDriverWait;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import us.codecraft.webmagic.Page;
@@ -136,10 +139,14 @@ public class SeleniumDownloader implements Downloader, Closeable {
 	 */
 	private Page downLoadPage(WebDriver webDriver, Task task, Request request){
 		logger.info("downloading page " + request.getUrl());
+		String content = "";
 		try {
-			webDriver.manage().timeouts().pageLoadTimeout(10, TimeUnit.SECONDS);
 			webDriver.get(request.getUrl());
-			Thread.sleep(sleepTime);
+			new WebDriverWait(webDriver, 10)
+					.until(ExpectedConditions.presenceOfElementLocated(By.xpath("/html")));
+			WebElement webElement = webDriver.findElement(By.xpath("/html"));
+			content = webElement.getAttribute("outerHTML");
+//			Thread.sleep(sleepTime);
 		} catch (Exception e) {
 			if (e instanceof InterruptedException) {
 				Thread.currentThread().interrupt();
@@ -160,8 +167,6 @@ public class SeleniumDownloader implements Downloader, Closeable {
 			}
 		}
 
-		WebElement webElement = webDriver.findElement(By.xpath("/html"));
-		String content = webElement.getAttribute("outerHTML");
 		Page page = new Page();
 		page.setRawText(content);
 		page.setHtml(new Html(content, request.getUrl()));

webmagic-samples/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.3-SNAPSHOT</version>
+        <version>0.8.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

webmagic-saxon/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.3-SNAPSHOT</version>
+        <version>0.8.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

webmagic-scripts/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft.duiba</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.8.3-SNAPSHOT</version>
+        <version>0.8.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>