public class FetchItemQueues extends Object
Modifier and Type | Field and Description |
---|---|
static String |
DEFAULT_ID |
static String |
QUEUE_MODE_DOMAIN |
static String |
QUEUE_MODE_HOST |
static String |
QUEUE_MODE_IP |
Constructor and Description |
---|
FetchItemQueues(Configuration conf) |
Modifier and Type | Method and Description |
---|---|
org.apache.nutch.fetcher.FetchItemQueues.QueuingStatus |
addFetchItem(FetchItem it) |
org.apache.nutch.fetcher.FetchItemQueues.QueuingStatus |
addFetchItem(Text url,
CrawlDatum datum) |
int |
checkExceptionThreshold(String queueid)
Increment the exception counter of a queue in case of an exception e.g.
|
protected static String |
checkQueueMode(String queueMode)
Check whether queue mode is valid, fall-back to default mode if not.
|
int |
checkTimelimit() |
void |
dump() |
int |
emptyQueues() |
void |
finishFetchItem(FetchItem it) |
void |
finishFetchItem(FetchItem it,
boolean asap) |
FetchItem |
getFetchItem() |
FetchItemQueue |
getFetchItemQueue(String id) |
int |
getQueueCount() |
int |
getQueueCountMaxExceptions() |
int |
getTotalSize() |
boolean |
redirectIsQueuedRecently(Text redirUrl) |
public static final String DEFAULT_ID
public static final String QUEUE_MODE_HOST
public static final String QUEUE_MODE_DOMAIN
public static final String QUEUE_MODE_IP
public FetchItemQueues(Configuration conf)
protected static String checkQueueMode(String queueMode)
queueMode
- queue mode to checkpublic int getTotalSize()
public int getQueueCount()
public int getQueueCountMaxExceptions()
public org.apache.nutch.fetcher.FetchItemQueues.QueuingStatus addFetchItem(Text url, CrawlDatum datum)
public org.apache.nutch.fetcher.FetchItemQueues.QueuingStatus addFetchItem(FetchItem it)
public void finishFetchItem(FetchItem it)
public void finishFetchItem(FetchItem it, boolean asap)
public FetchItemQueue getFetchItemQueue(String id)
public FetchItem getFetchItem()
public int checkTimelimit()
public int emptyQueues()
public int checkExceptionThreshold(String queueid)
queueid
- public boolean redirectIsQueuedRecently(Text redirUrl)
redirUrl
- redirect targetpublic void dump()
Copyright © 2021 The Apache Software Foundation