Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
W
webmagic
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
沈俊林
webmagic
Commits
a3a868f5
Commit
a3a868f5
authored
Aug 04, 2013
by
yihua.huang
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
rename
parent
04a7fa03
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
22 additions
and
22 deletions
+22
-22
ModelPageProcessor.java
.../java/us/codecraft/webmagic/model/ModelPageProcessor.java
+7
-7
ModelPipeline.java
.../main/java/us/codecraft/webmagic/model/ModelPipeline.java
+3
-3
OOSpider.java
...e/src/main/java/us/codecraft/webmagic/model/OOSpider.java
+11
-11
package.html
...re/src/main/java/us/codecraft/webmagic/model/package.html
+1
-1
No files found.
webmagic-core/src/main/java/us/codecraft/webmagic/model/
Object
PageProcessor.java
→
webmagic-core/src/main/java/us/codecraft/webmagic/model/
Model
PageProcessor.java
View file @
a3a868f5
...
@@ -18,7 +18,7 @@ import java.util.regex.Pattern;
...
@@ -18,7 +18,7 @@ import java.util.regex.Pattern;
* @date: 13-8-1 <br>
* @date: 13-8-1 <br>
* Time: 下午8:46 <br>
* Time: 下午8:46 <br>
*/
*/
public
class
Object
PageProcessor
implements
PageProcessor
{
public
class
Model
PageProcessor
implements
PageProcessor
{
private
List
<
PageModelExtractor
>
pageModelExtractorList
=
new
ArrayList
<
PageModelExtractor
>();
private
List
<
PageModelExtractor
>
pageModelExtractorList
=
new
ArrayList
<
PageModelExtractor
>();
...
@@ -26,16 +26,16 @@ public class ObjectPageProcessor implements PageProcessor {
...
@@ -26,16 +26,16 @@ public class ObjectPageProcessor implements PageProcessor {
private
Set
<
Pattern
>
targetUrlPatterns
=
new
HashSet
<
Pattern
>();
private
Set
<
Pattern
>
targetUrlPatterns
=
new
HashSet
<
Pattern
>();
public
static
Object
PageProcessor
create
(
Site
site
,
Class
...
clazzs
)
{
public
static
Model
PageProcessor
create
(
Site
site
,
Class
...
clazzs
)
{
ObjectPageProcessor
objectPageProcessor
=
new
Object
PageProcessor
(
site
);
ModelPageProcessor
modelPageProcessor
=
new
Model
PageProcessor
(
site
);
for
(
Class
clazz
:
clazzs
)
{
for
(
Class
clazz
:
clazzs
)
{
object
PageProcessor
.
addPageModel
(
clazz
);
model
PageProcessor
.
addPageModel
(
clazz
);
}
}
return
object
PageProcessor
;
return
model
PageProcessor
;
}
}
public
Object
PageProcessor
addPageModel
(
Class
clazz
)
{
public
Model
PageProcessor
addPageModel
(
Class
clazz
)
{
PageModelExtractor
pageModelExtractor
=
PageModelExtractor
.
create
(
clazz
);
PageModelExtractor
pageModelExtractor
=
PageModelExtractor
.
create
(
clazz
);
targetUrlPatterns
.
addAll
(
pageModelExtractor
.
getTargetUrlPatterns
());
targetUrlPatterns
.
addAll
(
pageModelExtractor
.
getTargetUrlPatterns
());
targetUrlPatterns
.
addAll
(
pageModelExtractor
.
getHelpUrlPatterns
());
targetUrlPatterns
.
addAll
(
pageModelExtractor
.
getHelpUrlPatterns
());
...
@@ -43,7 +43,7 @@ public class ObjectPageProcessor implements PageProcessor {
...
@@ -43,7 +43,7 @@ public class ObjectPageProcessor implements PageProcessor {
return
this
;
return
this
;
}
}
private
Object
PageProcessor
(
Site
site
)
{
private
Model
PageProcessor
(
Site
site
)
{
this
.
site
=
site
;
this
.
site
=
site
;
}
}
...
...
webmagic-core/src/main/java/us/codecraft/webmagic/model/
Object
Pipeline.java
→
webmagic-core/src/main/java/us/codecraft/webmagic/model/
Model
Pipeline.java
View file @
a3a868f5
...
@@ -14,14 +14,14 @@ import java.util.concurrent.ConcurrentHashMap;
...
@@ -14,14 +14,14 @@ import java.util.concurrent.ConcurrentHashMap;
* @date: 13-8-2 <br>
* @date: 13-8-2 <br>
* Time: 上午10:47 <br>
* Time: 上午10:47 <br>
*/
*/
public
class
Object
Pipeline
implements
Pipeline
{
public
class
Model
Pipeline
implements
Pipeline
{
private
Map
<
Class
,
PageModelPipeline
>
pageModelPipelines
=
new
ConcurrentHashMap
<
Class
,
PageModelPipeline
>();
private
Map
<
Class
,
PageModelPipeline
>
pageModelPipelines
=
new
ConcurrentHashMap
<
Class
,
PageModelPipeline
>();
public
Object
Pipeline
()
{
public
Model
Pipeline
()
{
}
}
public
Object
Pipeline
put
(
Class
clazz
,
PageModelPipeline
pageModelPipeline
)
{
public
Model
Pipeline
put
(
Class
clazz
,
PageModelPipeline
pageModelPipeline
)
{
pageModelPipelines
.
put
(
clazz
,
pageModelPipeline
);
pageModelPipelines
.
put
(
clazz
,
pageModelPipeline
);
return
this
;
return
this
;
}
}
...
...
webmagic-core/src/main/java/us/codecraft/webmagic/model/OOSpider.java
View file @
a3a868f5
...
@@ -15,22 +15,22 @@ public class OOSpider extends Spider {
...
@@ -15,22 +15,22 @@ public class OOSpider extends Spider {
*
*
* @param pageProcessor 已定义的抽取规则
* @param pageProcessor 已定义的抽取规则
*/
*/
private
ObjectPageProcessor
object
PageProcessor
;
private
ModelPageProcessor
model
PageProcessor
;
private
ObjectPipeline
object
Pipeline
;
private
ModelPipeline
model
Pipeline
;
protected
OOSpider
(
ObjectPageProcessor
object
PageProcessor
)
{
protected
OOSpider
(
ModelPageProcessor
model
PageProcessor
)
{
super
(
object
PageProcessor
);
super
(
model
PageProcessor
);
this
.
objectPageProcessor
=
object
PageProcessor
;
this
.
modelPageProcessor
=
model
PageProcessor
;
}
}
public
OOSpider
(
Site
site
,
PageModelPipeline
pageModelPipeline
,
Class
...
pageModels
)
{
public
OOSpider
(
Site
site
,
PageModelPipeline
pageModelPipeline
,
Class
...
pageModels
)
{
this
(
Object
PageProcessor
.
create
(
site
,
pageModels
));
this
(
Model
PageProcessor
.
create
(
site
,
pageModels
));
this
.
objectPipeline
=
new
Object
Pipeline
();
this
.
modelPipeline
=
new
Model
Pipeline
();
super
.
pipeline
(
object
Pipeline
);
super
.
pipeline
(
model
Pipeline
);
if
(
pageModelPipeline
!=
null
){
if
(
pageModelPipeline
!=
null
){
for
(
Class
pageModel
:
pageModels
)
{
for
(
Class
pageModel
:
pageModels
)
{
this
.
object
Pipeline
.
put
(
pageModel
,
pageModelPipeline
);
this
.
model
Pipeline
.
put
(
pageModel
,
pageModelPipeline
);
}
}
}
}
}
}
...
@@ -45,8 +45,8 @@ public class OOSpider extends Spider {
...
@@ -45,8 +45,8 @@ public class OOSpider extends Spider {
public
OOSpider
addPageModel
(
PageModelPipeline
pageModelPipeline
,
Class
...
pageModels
)
{
public
OOSpider
addPageModel
(
PageModelPipeline
pageModelPipeline
,
Class
...
pageModels
)
{
for
(
Class
pageModel
:
pageModels
)
{
for
(
Class
pageModel
:
pageModels
)
{
object
PageProcessor
.
addPageModel
(
pageModel
);
model
PageProcessor
.
addPageModel
(
pageModel
);
object
Pipeline
.
put
(
pageModel
,
pageModelPipeline
);
model
Pipeline
.
put
(
pageModel
,
pageModelPipeline
);
}
}
return
this
;
return
this
;
}
}
...
...
webmagic-core/src/main/java/us/codecraft/webmagic/model/package.html
View file @
a3a868f5
<html>
<html>
<body>
<body>
webmagic对抓取器编写的面向
对象方式的封装。基于POJO(称为PageModel)
及注解即可实现一个PageProcessor。
webmagic对抓取器编写的面向
模型(称为PageModel)的封装。基于POJO
及注解即可实现一个PageProcessor。
</body>
</body>
</html>
</html>
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment