public class UpdateHostDbReducer extends Reducer<Text,NutchWritable,Text,HostDatum>
Reducer.Context
Modifier and Type | Field and Description |
---|---|
protected static boolean |
checkFailed |
protected static boolean |
checkKnown |
protected static boolean |
checkNew |
protected ThreadPoolExecutor |
executor |
protected static boolean |
force |
protected static long |
now |
protected static String[] |
numericFields |
protected static Text[] |
numericFieldWritables |
protected Integer |
numResolverThreads |
protected static int[] |
percentiles |
protected static Integer |
purgeFailedHostsThreshold |
protected BlockingQueue<Runnable> |
queue |
protected static Integer |
recheckInterval |
protected ResolverThread |
resolverThread |
protected static String[] |
stringFields |
protected static Text[] |
stringFieldWritables |
Constructor and Description |
---|
UpdateHostDbReducer() |
Modifier and Type | Method and Description |
---|---|
void |
cleanup(Reducer.Context context)
Shut down all running threads and wait for completion.
|
protected boolean |
isEligibleForCheck(HostDatum datum)
Determines whether a record is eligible for recheck.
|
void |
reduce(Text key,
Iterable<NutchWritable> values,
Reducer.Context context) |
void |
setup(Reducer.Context context)
Configures the thread pool and prestarts all resolver threads.
|
protected boolean |
shouldCheck(HostDatum datum)
Determines whether a record should be checked.
|
protected ResolverThread resolverThread
protected Integer numResolverThreads
protected static Integer purgeFailedHostsThreshold
protected static Integer recheckInterval
protected static boolean checkFailed
protected static boolean checkNew
protected static boolean checkKnown
protected static boolean force
protected static long now
protected static String[] numericFields
protected static String[] stringFields
protected static int[] percentiles
protected static Text[] numericFieldWritables
protected static Text[] stringFieldWritables
protected BlockingQueue<Runnable> queue
protected ThreadPoolExecutor executor
public void setup(Reducer.Context context)
public void reduce(Text key, Iterable<NutchWritable> values, Reducer.Context context) throws IOException, InterruptedException
reduce
in class Reducer<Text,NutchWritable,Text,HostDatum>
IOException
InterruptedException
protected boolean shouldCheck(HostDatum datum)
datum
- protected boolean isEligibleForCheck(HostDatum datum)
datum
- public void cleanup(Reducer.Context context)
Copyright © 2021 The Apache Software Foundation