Merge pull request #193 from EdwardsBean/fix-mppipeline

Bug fix:MultiPagePipeline and DoubleKeyMap concurrent bug

webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/MultiPagePipeline.java

@@ -36,9 +36,15 @@ public class MultiPagePipeline implements Pipeline {
     private void handleObject(Iterator<Map.Entry<String, Object>> iterator) {
         Map.Entry<String, Object> objectEntry = iterator.next();
         Object o = objectEntry.getValue();
+        //需要拼凑
         if (o instanceof MultiPageModel) {
             MultiPageModel multiPageModel = (MultiPageModel) o;
+            //这次处理的部分，设置为完成
+            pageMap.put(multiPageModel.getPageKey(), multiPageModel.getPage(), Boolean.FALSE);
+            //每个key单独加锁
+            synchronized (pageMap.get(multiPageModel.getPageKey())) {
                 pageMap.put(multiPageModel.getPageKey(), multiPageModel.getPage(), Boolean.TRUE);
+                //其他需要拼凑的部分
                 if (multiPageModel.getOtherPages() != null) {
                     for (String otherPage : multiPageModel.getOtherPages()) {
                         Boolean aBoolean = pageMap.get(multiPageModel.getPageKey(), otherPage);
@@ -53,6 +59,7 @@ public class MultiPagePipeline implements Pipeline {
                 if (booleanMap == null) {
                     return;
                 }
+                // /过滤，这次完成的page item中，还未拼凑完整的item，不进入下一个pipeline
                 for (Map.Entry<String, Boolean> stringBooleanEntry : booleanMap.entrySet()) {
                     if (!stringBooleanEntry.getValue()) {
                         iterator.remove();
@@ -74,6 +81,7 @@ public class MultiPagePipeline implements Pipeline {
                             }
                         }
                     });
+                    // 合并
                     MultiPageModel value = entryList.get(0).getValue();
                     for (int i = 1; i < entryList.size(); i++) {
                         value = value.combine(entryList.get(i).getValue());
@@ -83,4 +91,6 @@ public class MultiPagePipeline implements Pipeline {
             }
         }
 
+    }
+
 }

webmagic-extension/src/main/java/us/codecraft/webmagic/utils/DoubleKeyMap.java

@@ -75,8 +75,9 @@ public class DoubleKeyMap<K1, K2, V> extends MultiKeyMapBase {
      * @param value
      * @return value
      */
-    public V put(K1 key1, K2 key2, V value) {
+    public synchronized V put(K1 key1, K2 key2, V value) {
         if (map.get(key1) == null) {
+            //不加锁的话，多个线程有可能都会执行到这里
             map.put(key1, this.<K2, V>newMap());
         }
         return get(key1).put(key2, value);
@@ -87,7 +88,7 @@ public class DoubleKeyMap<K1, K2, V> extends MultiKeyMapBase {
      * @param key2
      * @return value
      */
-    public V remove(K1 key1, K2 key2) {
+    public synchronized V remove(K1 key1, K2 key2) {
         if (get(key1) == null) {
             return null;
         }