orgnize dep

webmagic-plugin/webmagic-misc/pom.xml

@@ -22,6 +22,11 @@
             <artifactId>jedis</artifactId>
             <version>2.0.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.seleniumhq.selenium</groupId>
+            <artifactId>selenium-java</artifactId>
+            <version>2.33.0</version>
+        </dependency>
     </dependencies>
 
 </project>
\ No newline at end of file

webmagic-plugin/webmagic-selenium/src/main/java/us/codecraft/webmagic/selenium/downloader/SeleniumDownloader.java → webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java

-package us.codecraft.webmagic.selenium.downloader;
+package us.codecraft.webmagic.downloader.selenium;
 
 import org.apache.log4j.Logger;
 import org.openqa.selenium.By;

webmagic-plugin/webmagic-selenium/src/main/java/us/codecraft/webmagic/selenium/downloader/WebDriverPool.java → webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java

-package us.codecraft.webmagic.selenium.downloader;
+package us.codecraft.webmagic.downloader.selenium;
 
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.chrome.ChromeDriver;

webmagic-plugin/webmagic-selenium/README.md

-webmagic-selenium
--------
-尝试使用selenium来进行页面动态渲染，开发中。
\ No newline at end of file

webmagic-plugin/webmagic-selenium/pom.xml

-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
-    <parent>
-        <groupId>us.codecraft</groupId>
-        <artifactId>webmagic-plugin</artifactId>
-        <version>0.1.0</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-    <artifactId>webmagic-selenium</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.seleniumhq.selenium</groupId>
-            <artifactId>selenium-java</artifactId>
-            <version>2.33.0</version>
-        </dependency>
-    </dependencies>
-
-
-</project>
\ No newline at end of file

webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/SeleniumTest.java

-package us.codecraft.webmagic.selenium;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import org.openqa.selenium.By;
-import org.openqa.selenium.WebDriver;
-import org.openqa.selenium.WebElement;
-import org.openqa.selenium.chrome.ChromeDriver;
-import org.openqa.selenium.remote.DesiredCapabilities;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * @author code4crafter@gmail.com <br>
- * @date: 13-7-26 <br>
- * Time: 下午12:27 <br>
- */
-public class SeleniumTest {
-
-    @Ignore("need chrome driver")
-    @Test
-    public void testSelenium() {
-        System.getProperties().setProperty("webdriver.chrome.driver", "/Users/yihua/Downloads/chromedriver");
-        Map<String, Object> contentSettings = new HashMap<String, Object>();
-        contentSettings.put("images", 2);
-
-        Map<String, Object> preferences = new HashMap<String, Object>();
-        preferences.put("profile.default_content_settings", contentSettings);
-
-        DesiredCapabilities caps = DesiredCapabilities.chrome();
-        caps.setCapability("chrome.prefs", preferences);
-        caps.setCapability("chrome.switches", Arrays.asList("--user-data-dir=/Users/yihua/temp/chrome"));
-        WebDriver webDriver = new ChromeDriver(caps);
-        webDriver.get("http://huaban.com/");
-        WebElement webElement = webDriver.findElement(By.xpath("/html"));
-        System.out.println(webElement.getAttribute("outerHTML"));
-        webDriver.close();
-    }
-}

webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/downloader/SeleniumDownloaderTest.java

-package us.codecraft.webmagic.selenium.downloader;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.Request;
-import us.codecraft.webmagic.Site;
-import us.codecraft.webmagic.Task;
-
-/**
- * @author code4crafter@gmail.com <br>
- * @date: 13-7-26 <br>
- *        Time: 下午2:46 <br>
- */
-public class SeleniumDownloaderTest {
-
-	private String chromeDriverPath = "/Users/yihua/Downloads/chromedriver";
-
-	@Ignore("need chrome driver")
-	@Test
-	public void test() {
-		SeleniumDownloader seleniumDownloader = new SeleniumDownloader(chromeDriverPath);
-		long time1 = System.currentTimeMillis();
-		for (int i = 0; i < 100; i++) {
-			Page page = seleniumDownloader.download(new Request("http://huaban.com/"), new Task() {
-				@Override
-				public String getUUID() {
-					return "huaban.com";
-				}
-
-				@Override
-				public Site getSite() {
-					return Site.me();
-				}
-			});
-			System.out.println(page.getHtml().$("#waterfall").links().regex(".*pins.*").all());
-		}
-		System.out.println(System.currentTimeMillis() - time1);
-	}
-
-    @Ignore
-	@Test
-	public void testBaiduWenku() {
-		SeleniumDownloader seleniumDownloader = new SeleniumDownloader(chromeDriverPath);
-        seleniumDownloader.setSleepTime(10000);
-		long time1 = System.currentTimeMillis();
-		Page page = seleniumDownloader.download(new Request("http://wenku.baidu.com/view/462933ff04a1b0717fd5ddc2.html"), new Task() {
-			@Override
-			public String getUUID() {
-				return "huaban.com";
-			}
-
-			@Override
-			public Site getSite() {
-				return Site.me();
-			}
-		});
-		System.out.println(page.getHtml().$("div.inner").replace("<[^<>]+>","").replace("&nsbp;","").all());
-	}
-
-}

webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/downloader/WebDriverPoolTest.java

-package us.codecraft.webmagic.selenium.downloader;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import org.openqa.selenium.WebDriver;
-
-/**
- * @author code4crafter@gmail.com <br>
- * @date: 13-7-26 <br>
- * Time: 下午2:12 <br>
- */
-public class WebDriverPoolTest {
-
-    private String chromeDriverPath = "/Users/yihua/Downloads/chromedriver";
-
-    @Ignore("need chrome driver")
-    @Test
-    public void test() {
-        System.getProperties().setProperty("webdriver.chrome.driver", chromeDriverPath);
-        WebDriverPool webDriverPool = new WebDriverPool(5);
-        for (int i = 0; i < 5; i++) {
-            try {
-                WebDriver webDriver = webDriverPool.get();
-                System.out.println(i);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-        webDriverPool.closeAll();
-    }
-}

webmagic-samples/pom.xml

@@ -22,11 +22,6 @@
             <artifactId>webmagic-misc</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>us.codecraft</groupId>
-            <artifactId>webmagic-selenium</artifactId>
-            <version>${project.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.freemarker</groupId>
             <artifactId>freemarker</artifactId>