add switch

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -9,6 +9,7 @@ import us.codecraft.webmagic.pipeline.Pipeline;
 import us.codecraft.webmagic.processor.PageProcessor;
 import us.codecraft.webmagic.scheduler.QueueScheduler;
 import us.codecraft.webmagic.scheduler.Scheduler;
+import us.codecraft.webmagic.utils.EnvironmentUtil;
 import us.codecraft.webmagic.utils.ThreadUtils;
 
 import java.io.Closeable;
@@ -368,6 +369,14 @@ public class Spider implements Runnable, Task {
         return this;
     }
 
+    /**
+     * switch off xsoup
+     * @return
+     */
+    public static void xsoupOff(){
+        EnvironmentUtil.setUseXsoup(false);
+    }
+
     @Override
     public String getUUID() {
         if (uuid != null) {

webmagic-core/src/main/java/us/codecraft/webmagic/selector/CacheElement.java

-package us.codecraft.webmagic.selector;
-
-import org.jsoup.nodes.Element;
-
-import java.util.List;
-
-/**
- * Cache parsed element for extract.
- *
- * @author code4crafter@gmail.com
- * @since 0.2.2
- */
-public class CacheElement {
-
-    public String text;
-
-    public Element element;
-
-    public String select(Selector selector) {
-        if (selector instanceof ElementSelector) {
-            ElementSelector elementSelector = (ElementSelector) selector;
-            return elementSelector.select(getElement());
-        } else {
-            return selector.select(getText());
-        }
-    }
-
-    public List<String> selectList(Selector selector) {
-        if (selector instanceof ElementSelector) {
-            ElementSelector elementSelector = (ElementSelector) selector;
-            return elementSelector.selectList(getElement());
-        } else {
-            return selector.selectList(getText());
-        }
-    }
-}

webmagic-core/src/main/java/us/codecraft/webmagic/selector/Html.java

@@ -2,6 +2,7 @@ package us.codecraft.webmagic.selector;
 
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
+import us.codecraft.webmagic.utils.EnvironmentUtil;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -72,17 +73,22 @@ public class Html extends PlainText {
 
     @Override
     public Selectable xpath(String xpath) {
-        XsoupSelector xsoupSelector = new XsoupSelector(xpath);
-        if (document!=null){
-            return new Html(xsoupSelector.selectList(document));
+        if (EnvironmentUtil.useXsoup()) {
+            XsoupSelector xsoupSelector = new XsoupSelector(xpath);
+            if (document != null) {
+                return new Html(xsoupSelector.selectList(document));
+            }
+            return selectList(xsoupSelector, strings);
+        } else {
+            XpathSelector xpathSelector = new XpathSelector(xpath);
+            return selectList(xpathSelector, strings);
         }
-        return selectList(xsoupSelector, strings);
     }
 
     @Override
     public Selectable $(String selector) {
         CssSelector cssSelector = Selectors.$(selector);
-        if (document!=null){
+        if (document != null) {
             return new Html(cssSelector.selectList(document));
         }
         return selectList(cssSelector, strings);
@@ -91,7 +97,7 @@ public class Html extends PlainText {
     @Override
     public Selectable $(String selector, String attrName) {
         CssSelector cssSelector = Selectors.$(selector, attrName);
-        if (document!=null){
+        if (document != null) {
             return new Html(cssSelector.selectList(document));
         }
         return selectList(cssSelector, strings);
@@ -102,15 +108,17 @@ public class Html extends PlainText {
     }
 
     public String getText() {
+        if (strings!=null&&strings.size()>0){
+            return strings.get(0);
+        }
         return document.html();
     }
 
     /**
-     *
      * @param selector
      * @return
      */
-    public String select(Selector selector) {
+    public String selectDocument(Selector selector) {
         if (selector instanceof ElementSelector) {
             ElementSelector elementSelector = (ElementSelector) selector;
             return elementSelector.select(getDocument());
@@ -119,7 +127,7 @@ public class Html extends PlainText {
         }
     }
 
-    public List<String> selectList(Selector selector) {
+    public List<String> selectDocumentForList(Selector selector) {
         if (selector instanceof ElementSelector) {
             ElementSelector elementSelector = (ElementSelector) selector;
             return elementSelector.selectList(getDocument());

webmagic-core/src/main/java/us/codecraft/webmagic/utils/EnvironmentUtil.java

+package us.codecraft.webmagic.utils;
+
+import org.apache.commons.lang3.BooleanUtils;
+
+import java.util.Properties;
+
+/**
+ * @author code4crafter@gmail.com
+ * @since 0.2.2
+ */
+public abstract class EnvironmentUtil {
+
+    private static final String USE_XSOUP = "xsoup";
+
+    public static boolean useXsoup() {
+        Properties properties = System.getProperties();
+        Object o = properties.get(USE_XSOUP);
+        if (o == null) {
+            return true;
+        }
+        return BooleanUtils.toBoolean(((String) o).toLowerCase());
+    }
+
+    public static void setUseXsoup(boolean useXsoup) {
+        Properties properties = System.getProperties();
+        properties.setProperty(USE_XSOUP, BooleanUtils.toString(useXsoup, "true", "false"));
+    }
+}

webmagic-core/src/test/java/us/codecraft/webmagic/utils/EnvironmentUtilTest.java

+package us.codecraft.webmagic.utils;
+
+import org.junit.Test;
+
+import static junit.framework.Assert.*;
+
+/**
+ * @author code4crafter@gmail.com
+ */
+public class EnvironmentUtilTest {
+
+    @Test
+    public void test() {
+        assertTrue(EnvironmentUtil.useXsoup());
+        EnvironmentUtil.setUseXsoup(false);
+        assertFalse(EnvironmentUtil.useXsoup());
+    }
+}

webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java

 package us.codecraft.webmagic.model;
 
 import org.apache.commons.lang3.StringUtils;
-import org.jsoup.nodes.Element;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.model.annotation.*;
 import us.codecraft.webmagic.selector.*;
@@ -185,13 +184,13 @@ class PageModelExtractor {
             return null;
         }
         if (objectExtractor == null) {
-            return processSingle(page, page.getHtml().toString());
+            return processSingle(page, null, false);
         } else {
             if (objectExtractor.multi) {
                 List<Object> os = new ArrayList<Object>();
                 List<String> list = objectExtractor.getSelector().selectList(page.getHtml().toString());
                 for (String s : list) {
-                    Object o = processSingle(page, s);
+                    Object o = processSingle(page, s, false);
                     if (o != null) {
                         os.add(o);
                     }
@@ -199,19 +198,13 @@ class PageModelExtractor {
                 return os;
             } else {
                 String select = objectExtractor.getSelector().select(page.getHtml().toString());
-                Object o = processSingle(page, select);
+                Object o = processSingle(page, select, false);
                 return o;
             }
         }
     }
 
-    private List<String> select(Selector selector,Element element,String html){
-        if (selector instanceof ElementSelector){
-
-        }
-    }
-
-    private Object processSingle(Page page, String html) {
+    private Object processSingle(Page page, String html, boolean isRaw) {
         Object o = null;
         try {
             o = clazz.newInstance();
@@ -220,10 +213,14 @@ class PageModelExtractor {
                     List<String> value;
                     switch (fieldExtractor.getSource()) {
                         case RawHtml:
-                            value = fieldExtractor.getSelector().selectList(page.getHtml().toString());
+                            value = page.getHtml().selectDocumentForList(fieldExtractor.getSelector());
                             break;
                         case Html:
-                            value = fieldExtractor.getSelector().selectList(html);
+                            if (isRaw) {
+                                value = page.getHtml().selectDocumentForList(fieldExtractor.getSelector());
+                            } else {
+                                value = fieldExtractor.getSelector().selectList(html);
+                            }
                             break;
                         case Url:
                             value = fieldExtractor.getSelector().selectList(page.getUrl().toString());
@@ -239,10 +236,14 @@ class PageModelExtractor {
                     String value;
                     switch (fieldExtractor.getSource()) {
                         case RawHtml:
-                            value = fieldExtractor.getSelector().select(page.getHtml().toString());
+                            value = page.getHtml().selectDocument(fieldExtractor.getSelector());
                             break;
                         case Html:
-                            value = fieldExtractor.getSelector().select(html);
+                            if (isRaw) {
+                                value = page.getHtml().selectDocument(fieldExtractor.getSelector());
+                            } else {
+                                value = fieldExtractor.getSelector().select(html);
+                            }
                             break;
                         case Url:
                             value = fieldExtractor.getSelector().select(page.getUrl().toString());

webmagic-extension/src/main/java/us/codecraft/webmagic/utils/ExtractorUtils.java

@@ -8,6 +8,7 @@ import java.util.List;
 
 /**
  * Tools for annotation converting. <br>
+ *
  * @author code4crafter@gmail.com <br>
  * @since 0.2.1
  */
@@ -24,17 +25,27 @@ public class ExtractorUtils {
                 selector = new RegexSelector(value);
                 break;
             case XPath:
-                selector = new XsoupSelector(value);
+                selector = getXpathSelector(value);
                 break;
             default:
-                selector = new XsoupSelector(value);
+                selector = getXpathSelector(value);
+        }
+        return selector;
+    }
+
+    private static Selector getXpathSelector(String value) {
+        Selector selector;
+        if (EnvironmentUtil.useXsoup()) {
+            selector = new XsoupSelector(value);
+        } else {
+            selector = new XpathSelector(value);
         }
         return selector;
     }
 
     public static List<Selector> getSelectors(ExtractBy[] extractBies) {
         List<Selector> selectors = new ArrayList<Selector>();
-        if (extractBies==null){
+        if (extractBies == null) {
             return selectors;
         }
         for (ExtractBy extractBy : extractBies) {