Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
W
webmagic
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
沈俊林
webmagic
Commits
30af23d0
Commit
30af23d0
authored
Apr 25, 2014
by
yihua.huang
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
split monitor to server and client mode #98
parent
ced79630
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
76 additions
and
15 deletions
+76
-15
SpiderMonitor.java
...ain/java/us/codecraft/webmagic/monitor/SpiderMonitor.java
+76
-15
No files found.
webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitor.java
View file @
30af23d0
...
@@ -26,7 +26,20 @@ import java.util.concurrent.atomic.AtomicInteger;
...
@@ -26,7 +26,20 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
*/
public
class
SpiderMonitor
{
public
class
SpiderMonitor
{
public
static
final
int
RMI_PORT
=
14721
;
private
enum
Type
{
Server
,
Client
,
Local
;
}
private
static
final
int
DEFAULT_SERVER_PORT
=
14721
;
private
static
final
String
DEFAULT_SERVER_HOST
=
"localhost"
;
private
int
serverPort
;
private
String
serverHost
;
private
Type
type
=
Type
.
Local
;
private
List
<
SpiderStatusMBean
>
spiderStatuses
=
new
ArrayList
<
SpiderStatusMBean
>();
private
List
<
SpiderStatusMBean
>
spiderStatuses
=
new
ArrayList
<
SpiderStatusMBean
>();
public
List
<
SpiderStatusMBean
>
getSpiders
()
{
public
List
<
SpiderStatusMBean
>
getSpiders
()
{
...
@@ -37,6 +50,11 @@ public class SpiderMonitor {
...
@@ -37,6 +50,11 @@ public class SpiderMonitor {
return
spiderStatuses
.
get
(
0
);
return
spiderStatuses
.
get
(
0
);
}
}
/**
* Register spider for monitor.
* @param spiders
* @return
*/
public
SpiderMonitor
register
(
Spider
...
spiders
)
{
public
SpiderMonitor
register
(
Spider
...
spiders
)
{
for
(
Spider
spider
:
spiders
)
{
for
(
Spider
spider
:
spiders
)
{
MonitorSpiderListener
monitorSpiderListener
=
new
MonitorSpiderListener
();
MonitorSpiderListener
monitorSpiderListener
=
new
MonitorSpiderListener
();
...
@@ -93,17 +111,59 @@ public class SpiderMonitor {
...
@@ -93,17 +111,59 @@ public class SpiderMonitor {
}
}
public
SpiderMonitor
jndiStart
(
int
port
)
throws
IOException
,
JMException
{
/**
* Start monitor as server mode.
* @param port
* @return
* @throws IOException
* @throws JMException
*/
public
SpiderMonitor
server
(
int
port
)
throws
IOException
,
JMException
{
Registry
registry
=
LocateRegistry
.
createRegistry
(
port
);
Registry
registry
=
LocateRegistry
.
createRegistry
(
port
);
serverPort
=
port
;
serverHost
=
"localhost"
;
type
=
Type
.
Server
;
return
this
;
return
this
;
}
}
public
SpiderMonitor
jndiStart
()
throws
IOException
,
JMException
{
/**
return
jndiStart
(
RMI_PORT
);
* Start monitor as server mode.
* @return
* @throws IOException
* @throws JMException
*/
public
SpiderMonitor
server
()
throws
IOException
,
JMException
{
return
server
(
DEFAULT_SERVER_PORT
);
}
/**
* Start monitor as client mode.
* @param serverHost
* @param serverPort
* @return
* @throws IOException
* @throws JMException
*/
public
SpiderMonitor
client
(
String
serverHost
,
int
serverPort
)
throws
IOException
,
JMException
{
type
=
Type
.
Client
;
this
.
serverHost
=
serverHost
;
this
.
serverPort
=
serverPort
;
return
this
;
}
/**
* Start monitor as client mode.
* @return
* @throws IOException
* @throws JMException
*/
public
SpiderMonitor
client
()
throws
IOException
,
JMException
{
return
client
(
DEFAULT_SERVER_HOST
,
DEFAULT_SERVER_PORT
);
}
}
public
SpiderMonitor
jmxStart
()
throws
IOException
,
JMException
{
public
SpiderMonitor
jmxStart
()
throws
IOException
,
JMException
{
return
jmxStart
(
"localhost"
,
RMI
_PORT
);
return
jmxStart
(
"localhost"
,
DEFAULT_SERVER
_PORT
);
}
}
public
SpiderMonitor
jmxStart
(
String
jndiServer
,
int
rmiPort
)
throws
IOException
,
JMException
{
public
SpiderMonitor
jmxStart
(
String
jndiServer
,
int
rmiPort
)
throws
IOException
,
JMException
{
...
@@ -114,12 +174,13 @@ public class SpiderMonitor {
...
@@ -114,12 +174,13 @@ public class SpiderMonitor {
ObjectName
objName
;
ObjectName
objName
;
JMXServiceURL
url
=
new
JMXServiceURL
(
"service:jmx:rmi:///jndi/rmi://"
+
jndiServer
+
":"
+
rmiPort
+
"/"
+
jmxServerName
);
if
(
type
!=
Type
.
Local
)
{
System
.
out
.
println
(
"JMXServiceURL: "
+
url
.
toString
());
JMXServiceURL
url
=
new
JMXServiceURL
(
"service:jmx:rmi:///jndi/rmi://"
+
jndiServer
+
":"
+
rmiPort
+
"/"
+
jmxServerName
);
System
.
out
.
println
(
"Please replace localhost of your ip if you want to connect it in remote server."
);
System
.
out
.
println
(
"JMXServiceURL: "
+
url
.
toString
());
JMXConnectorServer
jmxConnServer
=
JMXConnectorServerFactory
.
newJMXConnectorServer
(
url
,
null
,
localServer
);
System
.
out
.
println
(
"Please replace localhost of your ip if you want to connect it in remote server."
);
jmxConnServer
.
start
();
JMXConnectorServer
jmxConnServer
=
JMXConnectorServerFactory
.
newJMXConnectorServer
(
url
,
null
,
localServer
);
jmxConnServer
.
start
();
}
for
(
SpiderStatusMBean
spiderStatus
:
spiderStatuses
)
{
for
(
SpiderStatusMBean
spiderStatus
:
spiderStatuses
)
{
objName
=
new
ObjectName
(
jmxServerName
+
":name="
+
spiderStatus
.
getName
());
objName
=
new
ObjectName
(
jmxServerName
+
":name="
+
spiderStatus
.
getName
());
...
@@ -129,9 +190,7 @@ public class SpiderMonitor {
...
@@ -129,9 +190,7 @@ public class SpiderMonitor {
return
this
;
return
this
;
}
}
public
static
void
main
(
String
[]
args
)
throws
JMException
,
public
static
void
main
(
String
[]
args
)
throws
Exception
{
NullPointerException
,
IOException
{
Spider
oschinaSpider
=
Spider
.
create
(
new
OschinaBlogPageProcessor
())
Spider
oschinaSpider
=
Spider
.
create
(
new
OschinaBlogPageProcessor
())
.
addUrl
(
"http://my.oschina.net/flashsword/blog"
).
thread
(
2
);
.
addUrl
(
"http://my.oschina.net/flashsword/blog"
).
thread
(
2
);
...
@@ -140,7 +199,9 @@ public class SpiderMonitor {
...
@@ -140,7 +199,9 @@ public class SpiderMonitor {
SpiderMonitor
spiderMonitor
=
new
SpiderMonitor
();
SpiderMonitor
spiderMonitor
=
new
SpiderMonitor
();
spiderMonitor
.
register
(
oschinaSpider
,
githubSpider
);
spiderMonitor
.
register
(
oschinaSpider
,
githubSpider
);
spiderMonitor
.
jndiStart
().
jmxStart
();
spiderMonitor
.
jmxStart
();
oschinaSpider
.
start
();
githubSpider
.
start
();
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment