Merge pull request #218 from bingoko/master

添加PhantomJS无界面浏览器支持

webmagic-selenium/config.ini

+# What WebDriver to use for the tests
+driver=phantomjs
+#driver=firefox
+#driver=chrome
+#driver=http://localhost:8910
+#driver=http://localhost:4444/wd/hub
+
+# PhantomJS specific config (change according to your installation)
+#phantomjs_exec_path=/Users/Bingo/bin/phantomjs-qt5
+phantomjs_exec_path=/Users/Bingo/Downloads/phantomjs-1.9.8-macosx/bin/phantomjs
+#phantomjs_driver_path=/Users/Bingo/Documents/workspace/webmagic/webmagic-selenium/src/main.js
+phantomjs_driver_loglevel=DEBUG
\ No newline at end of file

webmagic-selenium/pom.xml

 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<parent>
 		<artifactId>webmagic-parent</artifactId>
 		<groupId>us.codecraft</groupId>
@@ -13,13 +14,21 @@
 		<dependency>
 			<groupId>org.seleniumhq.selenium</groupId>
 			<artifactId>selenium-java</artifactId>
-            <version>2.33.0</version>
+			<version>2.46.0</version>
 		</dependency>
 		<dependency>
 			<groupId>us.codecraft</groupId>
 			<artifactId>webmagic-core</artifactId>
 			<version>${project.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>com.github.detro</groupId>
+			<artifactId>phantomjsdriver</artifactId>
+			<version>1.2.0</version>
+		</dependency>
+
+
+
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>

webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java

@@ -5,6 +5,7 @@ import org.openqa.selenium.By;
 import org.openqa.selenium.Cookie;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
+
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Request;
 import us.codecraft.webmagic.Site;
@@ -36,13 +37,26 @@ public class SeleniumDownloader implements Downloader, Closeable {
 
 	private int poolSize = 1;
 
+	private static final String DRIVER_PHANTOMJS = "phantomjs";
+
 	/**
 	 * 新建
 	 *
 	 * @param chromeDriverPath
 	 */
 	public SeleniumDownloader(String chromeDriverPath) {
-        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriverPath);
+		System.getProperties().setProperty("webdriver.chrome.driver",
+				chromeDriverPath);
+	}
+
+	/**
+	 * Constructor without any filed. Construct PhantomJS browser
+	 * 
+	 * @author bob.li.0718@gmail.com
+	 */
+	public SeleniumDownloader() {
+		// System.setProperty("phantomjs.binary.path",
+		// "/Users/Bingo/Downloads/phantomjs-1.9.7-macosx/bin/phantomjs");
 	}
 
 	/**
@@ -76,16 +90,26 @@ public class SeleniumDownloader implements Downloader, Closeable {
 		WebDriver.Options manage = webDriver.manage();
 		Site site = task.getSite();
 		if (site.getCookies() != null) {
-            for (Map.Entry<String, String> cookieEntry : site.getCookies().entrySet()) {
-                Cookie cookie = new Cookie(cookieEntry.getKey(), cookieEntry.getValue());
+			for (Map.Entry<String, String> cookieEntry : site.getCookies()
+					.entrySet()) {
+				Cookie cookie = new Cookie(cookieEntry.getKey(),
+						cookieEntry.getValue());
 				manage.addCookie(cookie);
 			}
 		}
+
+		/*
+		 * TODO You can add mouse event or other processes
+		 * 
+		 * @author: bob.li.0718@gmail.com
+		 */
+
 		WebElement webElement = webDriver.findElement(By.xpath("/html"));
 		String content = webElement.getAttribute("outerHTML");
 		Page page = new Page();
 		page.setRawText(content);
-        page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content, request.getUrl())));
+		page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content,
+				request.getUrl())));
 		page.setUrl(new PlainText(request.getUrl()));
 		page.setRequest(request);
 		webDriverPool.returnToPool(webDriver);
@@ -94,7 +118,7 @@ public class SeleniumDownloader implements Downloader, Closeable {
 
 	private void checkInit() {
 		if (webDriverPool == null) {
-            synchronized (this){
+			synchronized (this) {
 				webDriverPool = new WebDriverPool(poolSize);
 			}
 		}

webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java

@@ -3,10 +3,20 @@ package us.codecraft.webmagic.downloader.selenium;
 import org.apache.log4j.Logger;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.chrome.ChromeDriver;
-
+import org.openqa.selenium.firefox.FirefoxDriver;
+import org.openqa.selenium.phantomjs.PhantomJSDriver;
+import org.openqa.selenium.phantomjs.PhantomJSDriverService;
+import org.openqa.selenium.remote.DesiredCapabilities;
+import org.openqa.selenium.remote.RemoteWebDriver;
+
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Properties;
 import java.util.concurrent.BlockingDeque;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -29,10 +39,125 @@ class WebDriverPool {
 
 	private AtomicInteger stat = new AtomicInteger(STAT_RUNNING);
 
+	/*
+	 * new fields for configuring phantomJS
+	 */
+	private WebDriver mDriver = null;
+	private boolean mAutoQuitDriver = true;
+
+	private static final String CONFIG_FILE = "/Users/Bingo/Documents/workspace/webmagic/webmagic-selenium/config.ini";
+	private static final String DRIVER_FIREFOX = "firefox";
+	private static final String DRIVER_CHROME = "chrome";
+	private static final String DRIVER_PHANTOMJS = "phantomjs";
+
+	protected static Properties sConfig;
+	protected static DesiredCapabilities sCaps;
+
+	/**
+	 * Configure the GhostDriver, and initialize a WebDriver instance. This part
+	 * of code comes from GhostDriver.
+	 * https://github.com/detro/ghostdriver/tree/master/test/java/src/test/java/ghostdriver
+	 * 
+	 * @author bob.li.0718@gmail.com
+	 * @throws IOException
+	 */
+	public void configure() throws IOException {
+		// Read config file
+		sConfig = new Properties();
+		sConfig.load(new FileReader(CONFIG_FILE));
+
+		// Prepare capabilities
+		sCaps = new DesiredCapabilities();
+		sCaps.setJavascriptEnabled(true);
+		sCaps.setCapability("takesScreenshot", false);
+
+		String driver = sConfig.getProperty("driver", DRIVER_PHANTOMJS);
+
+		// Fetch PhantomJS-specific configuration parameters
+		if (driver.equals(DRIVER_PHANTOMJS)) {
+			// "phantomjs_exec_path"
+			if (sConfig.getProperty("phantomjs_exec_path") != null) {
+				sCaps.setCapability(
+						PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY,
+						sConfig.getProperty("phantomjs_exec_path"));
+			} else {
+				throw new IOException(
+						String.format(
+								"Property '%s' not set!",
+								PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY));
+			}
+			// "phantomjs_driver_path"
+			if (sConfig.getProperty("phantomjs_driver_path") != null) {
+				System.out.println("Test will use an external GhostDriver");
+				sCaps.setCapability(
+						PhantomJSDriverService.PHANTOMJS_GHOSTDRIVER_PATH_PROPERTY,
+						sConfig.getProperty("phantomjs_driver_path"));
+			} else {
+				System.out
+						.println("Test will use PhantomJS internal GhostDriver");
+			}
+		}
+
+		// Disable "web-security", enable all possible "ssl-protocols" and
+		// "ignore-ssl-errors" for PhantomJSDriver
+		// sCaps.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS, new
+		// String[] {
+		// "--web-security=false",
+		// "--ssl-protocol=any",
+		// "--ignore-ssl-errors=true"
+		// });
+
+		ArrayList<String> cliArgsCap = new ArrayList<String>();
+		cliArgsCap.add("--web-security=false");
+		cliArgsCap.add("--ssl-protocol=any");
+		cliArgsCap.add("--ignore-ssl-errors=true");
+		sCaps.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS,
+				cliArgsCap);
+
+		// Control LogLevel for GhostDriver, via CLI arguments
+		sCaps.setCapability(
+				PhantomJSDriverService.PHANTOMJS_GHOSTDRIVER_CLI_ARGS,
+				new String[] { "--logLevel="
+						+ (sConfig.getProperty("phantomjs_driver_loglevel") != null ? sConfig
+								.getProperty("phantomjs_driver_loglevel")
+								: "INFO") });
+
+		// String driver = sConfig.getProperty("driver", DRIVER_PHANTOMJS);
+
+		// Start appropriate Driver
+		if (isUrl(driver)) {
+			sCaps.setBrowserName("phantomjs");
+			mDriver = new RemoteWebDriver(new URL(driver), sCaps);
+		} else if (driver.equals(DRIVER_FIREFOX)) {
+			mDriver = new FirefoxDriver(sCaps);
+		} else if (driver.equals(DRIVER_CHROME)) {
+			mDriver = new ChromeDriver(sCaps);
+		} else if (driver.equals(DRIVER_PHANTOMJS)) {
+			mDriver = new PhantomJSDriver(sCaps);
+		}
+	}
+
+	/**
+	 * check whether input is a valid URL
+	 * 
+	 * @author bob.li.0718@gmail.com
+	 * @param urlString
+	 * @return true means yes, otherwise no.
+	 */
+	private boolean isUrl(String urlString) {
+		try {
+			new URL(urlString);
+			return true;
+		} catch (MalformedURLException mue) {
+			return false;
+		}
+	}
+
 	/**
 	 * store webDrivers created
 	 */
-    private List<WebDriver> webDriverList = Collections.synchronizedList(new ArrayList<WebDriver>());
+	private List<WebDriver> webDriverList = Collections
+			.synchronizedList(new ArrayList<WebDriver>());
 
 	/**
 	 * store webDrivers available
@@ -47,6 +172,11 @@ class WebDriverPool {
 		this(DEFAULT_CAPACITY);
 	}
 
+	/**
+	 * 
+	 * @return
+	 * @throws InterruptedException
+	 */
 	public WebDriver get() throws InterruptedException {
 		checkRunning();
 		WebDriver poll = innerQueue.poll();
@@ -56,9 +186,20 @@ class WebDriverPool {
 		if (webDriverList.size() < capacity) {
 			synchronized (webDriverList) {
 				if (webDriverList.size() < capacity) {
-                    ChromeDriver e = new ChromeDriver();
-                    innerQueue.add(e);
-                    webDriverList.add(e);
+
+					// add new WebDriver instance into pool
+					try {
+						configure();
+						innerQueue.add(mDriver);
+						webDriverList.add(mDriver);
+					} catch (IOException e) {
+						e.printStackTrace();
+					}
+
+					// ChromeDriver e = new ChromeDriver();
+					// WebDriver e = getWebDriver();
+					// innerQueue.add(e);
+					// webDriverList.add(e);
 				}
 			}
 
@@ -85,6 +226,7 @@ class WebDriverPool {
 		for (WebDriver webDriver : webDriverList) {
 			logger.info("Quit webDriver" + webDriver);
 			webDriver.quit();
+			webDriver = null;
 		}
 	}
 

webmagic-selenium/src/test/java/us/codecraft/webmagic/samples/GooglePlayProcessor.java

+package us.codecraft.webmagic.samples;
+
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
+import us.codecraft.webmagic.pipeline.FilePipeline;
+import us.codecraft.webmagic.processor.PageProcessor;
+
+/**
+ *
+ * Using Selenium with PhantomJS to fetch web-page with JS<br>
+ * 
+ * @author bob.li.0718@gmail.com <br>
+ *         Date: 15-7-11 <br>
+ */
+public class GooglePlayProcessor implements PageProcessor {
+
+	private Site site;
+
+	@Override
+	public void process(Page page) {
+
+		page.putField("whole-html", page.getHtml().toString());
+
+	}
+
+	@Override
+	public Site getSite() {
+		if (null == site) {
+			site = Site.me().setDomain("play.google.com").setSleepTime(300);
+		}
+		return site;
+	}
+
+	public static void main(String[] args) {
+		Spider.create(new GooglePlayProcessor())
+				.thread(5)
+				.addPipeline(
+						new FilePipeline(
+								"/Users/Bingo/Documents/workspace/webmagic/webmagic-selenium/data/"))
+				.setDownloader(new SeleniumDownloader())
+				.addUrl("https://play.google.com/store/apps/details?id=com.tencent.mm")
+				.runAsync();
+	}
+}