Commit dc9f574e authored by yihua.huang's avatar yihua.huang

update request

parent d56c681b
package us.codecraft.webmagic; package us.codecraft.webmagic;
import java.util.HashMap;
import java.util.Map;
/** /**
* Request对象封装了待抓取的url信息。<br/> * Request对象封装了待抓取的url信息。<br/>
* 在PageProcessor中,Request对象可以通过{@link us.codecraft.webmagic.Page#getRequest()} 获取。<br/> * 在PageProcessor中,Request对象可以通过{@link us.codecraft.webmagic.Page#getRequest()} 获取。<br/>
...@@ -18,6 +21,7 @@ package us.codecraft.webmagic; ...@@ -18,6 +21,7 @@ package us.codecraft.webmagic;
* String linktext = (String)page.getRequest().getExtra()[0]; * String linktext = (String)page.getRequest().getExtra()[0];
* } * }
* </pre> * </pre>
*
* @author code4crafter@gmail.com <br> * @author code4crafter@gmail.com <br>
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午11:37 * Time: 上午11:37
...@@ -26,18 +30,20 @@ public class Request { ...@@ -26,18 +30,20 @@ public class Request {
private String url; private String url;
private Object[] extra; /**
* 额外参数,可以保存一些需要的上下文信息
*/
private Map<String, Object> extras = new HashMap<String, Object>();
private double priority; private double priority;
/** /**
* 构建一个request对象 * 构建一个request对象
*
* @param url 必须参数,待抓取的url * @param url 必须参数,待抓取的url
* @param extra 额外参数,可以保存一些需要的上下文信息
*/ */
public Request(String url, Object... extra) { public Request(String url) {
this.url = url; this.url = url;
this.extra = extra;
} }
public double getPriority() { public double getPriority() {
...@@ -49,16 +55,18 @@ public class Request { ...@@ -49,16 +55,18 @@ public class Request {
return this; return this;
} }
/** public Object getExtra(String key) {
* 获取预存的对象 return extras.get(key);
* @return object[] 预存的对象数组 }
*/
public Object[] getExtra() { public Request putExtra(String key,Object value) {
return extra; extras.put(key,value);
return this;
} }
/** /**
* 获取待抓取的url * 获取待抓取的url
*
* @return url 待抓取的url * @return url 待抓取的url
*/ */
public String getUrl() { public String getUrl() {
......
...@@ -33,7 +33,7 @@ public class RedisScheduler implements Scheduler { ...@@ -33,7 +33,7 @@ public class RedisScheduler implements Scheduler {
if (jedis.zrank(SET_PREFIX + task.getUUID(), request.getUrl()) == null) { if (jedis.zrank(SET_PREFIX + task.getUUID(), request.getUrl()) == null) {
//使用List保存队列 //使用List保存队列
jedis.rpush(QUEUE_PREFIX + task.getUUID(), request.getUrl()); jedis.rpush(QUEUE_PREFIX + task.getUUID(), request.getUrl());
jedis.zadd(SET_PREFIX + task.getUUID(), System.currentTimeMillis(), request.getUrl()); jedis.zadd(SET_PREFIX + task.getUUID(), request.getPriority(), request.getUrl());
} }
pool.returnResource(jedis); pool.returnResource(jedis);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment