Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
W
webmagic
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
沈俊林
webmagic
Commits
94a67165
Commit
94a67165
authored
Apr 26, 2014
by
yihua.huang
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
remove jmx server for simplify #98
parent
86a45a66
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
8 additions
and
148 deletions
+8
-148
MonitorExample.java
...in/java/us/codecraft/webmagic/example/MonitorExample.java
+2
-10
SpiderMonitor.java
...ain/java/us/codecraft/webmagic/monitor/SpiderMonitor.java
+5
-136
SpiderMonitorTest.java
...java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java
+0
-1
Kr36NewsModel.java
...va/us/codecraft/webmagic/model/samples/Kr36NewsModel.java
+1
-1
No files found.
webmagic-extension/src/main/java/us/codecraft/webmagic/example/MonitorExample.java
View file @
94a67165
...
@@ -18,16 +18,8 @@ public class MonitorExample {
...
@@ -18,16 +18,8 @@ public class MonitorExample {
Spider
githubSpider
=
Spider
.
create
(
new
GithubRepoPageProcessor
())
Spider
githubSpider
=
Spider
.
create
(
new
GithubRepoPageProcessor
())
.
addUrl
(
"https://github.com/code4craft"
);
.
addUrl
(
"https://github.com/code4craft"
);
SpiderMonitor
spiderMonitor
=
SpiderMonitor
.
instance
();
SpiderMonitor
.
instance
().
register
(
oschinaSpider
);
spiderMonitor
.
register
(
oschinaSpider
);
SpiderMonitor
.
instance
().
register
(
githubSpider
);
spiderMonitor
.
register
(
githubSpider
);
//If you want to connect it from remote, use spiderMonitor.server().jmxStart();
//ONLY ONE server can start for a machine.
//Others will be registered without start a server.
//You can also register a server by spiderMonitor.client(host,port).jmxStart().
spiderMonitor
.
server
().
jmxStart
();
oschinaSpider
.
start
();
githubSpider
.
start
();
}
}
}
}
webmagic-extension/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitor.java
View file @
94a67165
...
@@ -6,17 +6,9 @@ import us.codecraft.webmagic.Request;
...
@@ -6,17 +6,9 @@ import us.codecraft.webmagic.Request;
import
us.codecraft.webmagic.Spider
;
import
us.codecraft.webmagic.Spider
;
import
us.codecraft.webmagic.SpiderListener
;
import
us.codecraft.webmagic.SpiderListener
;
import
us.codecraft.webmagic.utils.Experimental
;
import
us.codecraft.webmagic.utils.Experimental
;
import
us.codecraft.webmagic.utils.IPUtils
;
import
javax.management.*
;
import
javax.management.*
;
import
javax.management.remote.JMXConnectorServer
;
import
javax.management.remote.JMXConnectorServerFactory
;
import
javax.management.remote.JMXServiceURL
;
import
java.io.IOException
;
import
java.lang.management.ManagementFactory
;
import
java.lang.management.ManagementFactory
;
import
java.rmi.registry.LocateRegistry
;
import
java.rmi.registry.Registry
;
import
java.rmi.server.ExportException
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.Collections
;
import
java.util.Collections
;
import
java.util.List
;
import
java.util.List
;
...
@@ -30,41 +22,21 @@ import java.util.concurrent.atomic.AtomicInteger;
...
@@ -30,41 +22,21 @@ import java.util.concurrent.atomic.AtomicInteger;
@Experimental
@Experimental
public
class
SpiderMonitor
{
public
class
SpiderMonitor
{
private
enum
Type
{
Server
,
Client
,
Local
;
}
private
static
SpiderMonitor
INSTANCE
=
new
SpiderMonitor
();
private
static
SpiderMonitor
INSTANCE
=
new
SpiderMonitor
();
private
AtomicBoolean
started
=
new
AtomicBoolean
(
false
);
private
AtomicBoolean
started
=
new
AtomicBoolean
(
false
);
private
Logger
logger
=
LoggerFactory
.
getLogger
(
getClass
());
private
Logger
logger
=
LoggerFactory
.
getLogger
(
getClass
());
private
static
final
int
DEFAULT_SERVER_PORT
=
14721
;
private
static
final
String
DEFAULT_SERVER_HOST
=
"localhost"
;
private
int
serverPort
;
private
MBeanServer
mbeanServer
;
private
MBeanServer
mbeanServer
;
private
String
jmxServerName
;
private
String
jmxServerName
;
private
String
serverHost
;
private
Type
type
=
Type
.
Local
;
private
List
<
SpiderStatusMXBean
>
spiderStatuses
=
new
ArrayList
<
SpiderStatusMXBean
>();
private
List
<
SpiderStatusMXBean
>
spiderStatuses
=
new
ArrayList
<
SpiderStatusMXBean
>();
public
List
<
SpiderStatusMXBean
>
getSpiders
()
{
protected
SpiderMonitor
()
{
return
spiderStatuses
;
jmxServerName
=
"WebMagic"
;
}
mbeanServer
=
ManagementFactory
.
getPlatformMBeanServer
();
public
SpiderStatusMXBean
getSpider
()
{
return
spiderStatuses
.
get
(
0
);
}
private
SpiderMonitor
()
{
}
}
/**
/**
...
@@ -84,9 +56,7 @@ public class SpiderMonitor {
...
@@ -84,9 +56,7 @@ public class SpiderMonitor {
spider
.
getSpiderListeners
().
add
(
monitorSpiderListener
);
spider
.
getSpiderListeners
().
add
(
monitorSpiderListener
);
}
}
SpiderStatusMXBean
spiderStatusMBean
=
getSpiderStatusMBean
(
spider
,
monitorSpiderListener
);
SpiderStatusMXBean
spiderStatusMBean
=
getSpiderStatusMBean
(
spider
,
monitorSpiderListener
);
if
(
started
.
get
())
{
registerMBean
(
spiderStatusMBean
);
registerMBean
(
spiderStatusMBean
);
}
spiderStatuses
.
add
(
spiderStatusMBean
);
spiderStatuses
.
add
(
spiderStatusMBean
);
}
}
return
this
;
return
this
;
...
@@ -132,109 +102,8 @@ public class SpiderMonitor {
...
@@ -132,109 +102,8 @@ public class SpiderMonitor {
}
}
}
}
/**
* Start monitor as server mode.
*
* @param port
* @return
* @throws IOException
* @throws JMException
*/
public
synchronized
SpiderMonitor
server
(
int
port
)
throws
IOException
,
JMException
{
try
{
Registry
registry
=
LocateRegistry
.
createRegistry
(
port
);
}
catch
(
ExportException
e
)
{
logger
.
warn
(
"Start server fail, maybe the address is in using."
,
e
);
}
serverPort
=
port
;
serverHost
=
"localhost"
;
type
=
Type
.
Server
;
return
this
;
}
/**
* Start monitor as server mode.
*
* @return
* @throws IOException
* @throws JMException
*/
public
SpiderMonitor
server
()
throws
IOException
,
JMException
{
return
server
(
DEFAULT_SERVER_PORT
);
}
/**
* Local mode: the monitor will be bound to the JVM instance.<br></br>
* Use jconsole to check your application.
*
* @return
*/
public
synchronized
SpiderMonitor
local
()
{
this
.
type
=
Type
.
Local
;
return
this
;
}
/**
* Start monitor as client mode.
*
* @param serverHost
* @param serverPort
* @return
* @throws IOException
* @throws JMException
*/
public
synchronized
SpiderMonitor
client
(
String
serverHost
,
int
serverPort
)
throws
IOException
,
JMException
{
type
=
Type
.
Client
;
this
.
serverHost
=
serverHost
;
this
.
serverPort
=
serverPort
;
return
this
;
}
/**
* Start monitor as client mode.
*
* @return
* @throws IOException
* @throws JMException
*/
public
SpiderMonitor
client
()
throws
IOException
,
JMException
{
return
client
(
DEFAULT_SERVER_HOST
,
DEFAULT_SERVER_PORT
);
}
public
synchronized
SpiderMonitor
jmxStart
()
throws
IOException
,
JMException
{
if
(!
started
.
compareAndSet
(
false
,
true
))
{
logger
.
error
(
"Monitor has already started!"
);
return
this
;
}
jmxServerName
=
"WebMagic-"
+
IPUtils
.
getFirstNoLoopbackIPAddresses
();
// start JNDI
mbeanServer
=
ManagementFactory
.
getPlatformMBeanServer
();
ObjectName
objName
;
if
(
type
!=
Type
.
Local
)
{
JMXServiceURL
url
=
new
JMXServiceURL
(
"service:jmx:rmi:///jndi/rmi://"
+
serverHost
+
":"
+
serverPort
+
"/"
+
jmxServerName
);
System
.
out
.
println
(
"JMXServiceURL: "
+
url
.
toString
());
System
.
out
.
println
(
"Please replace localhost of your ip if you want to connect it in remote server."
);
JMXConnectorServer
jmxConnServer
=
JMXConnectorServerFactory
.
newJMXConnectorServer
(
url
,
null
,
mbeanServer
);
jmxConnServer
.
start
();
objName
=
new
ObjectName
(
jmxServerName
+
":name=WebMagicMonitor"
);
mbeanServer
.
registerMBean
(
jmxConnServer
,
objName
);
}
for
(
SpiderStatusMXBean
spiderStatus
:
spiderStatuses
)
{
registerMBean
(
spiderStatus
);
}
return
this
;
}
protected
void
registerMBean
(
SpiderStatusMXBean
spiderStatus
)
throws
MalformedObjectNameException
,
InstanceAlreadyExistsException
,
MBeanRegistrationException
,
NotCompliantMBeanException
{
protected
void
registerMBean
(
SpiderStatusMXBean
spiderStatus
)
throws
MalformedObjectNameException
,
InstanceAlreadyExistsException
,
MBeanRegistrationException
,
NotCompliantMBeanException
{
ObjectName
objName
;
ObjectName
objName
=
new
ObjectName
(
jmxServerName
+
":name="
+
spiderStatus
.
getName
());
objName
=
new
ObjectName
(
jmxServerName
+
":name="
+
spiderStatus
.
getName
());
mbeanServer
.
registerMBean
(
spiderStatus
,
objName
);
mbeanServer
.
registerMBean
(
spiderStatus
,
objName
);
}
}
...
...
webmagic-extension/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java
View file @
94a67165
...
@@ -26,7 +26,6 @@ public class SpiderMonitorTest {
...
@@ -26,7 +26,6 @@ public class SpiderMonitorTest {
.
addUrl
(
"https://github.com/code4craft"
);
.
addUrl
(
"https://github.com/code4craft"
);
spiderMonitor
.
register
(
oschinaSpider
,
githubSpider
);
spiderMonitor
.
register
(
oschinaSpider
,
githubSpider
);
spiderMonitor
.
jmxStart
();
}
}
}
}
webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/Kr36NewsModel.java
View file @
94a67165
...
@@ -40,7 +40,7 @@ public class Kr36NewsModel {
...
@@ -40,7 +40,7 @@ public class Kr36NewsModel {
},
Kr36NewsModel
.
class
).
thread
(
20
);
},
Kr36NewsModel
.
class
).
thread
(
20
);
thread
.
start
();
thread
.
start
();
SpiderMonitor
spiderMonitor
=
SpiderMonitor
.
instance
();
SpiderMonitor
spiderMonitor
=
SpiderMonitor
.
instance
();
spiderMonitor
.
server
().
register
(
thread
).
jmxStart
(
);
spiderMonitor
.
register
(
thread
);
}
}
public
String
getTitle
()
{
public
String
getTitle
()
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment