@InterfaceAudience.Private @InterfaceStability.Unstable public class LeafQueue extends AbstractCSQueue
Modifier and Type | Field and Description |
---|---|
protected int |
maxApplications |
protected int |
maxApplicationsPerUser |
activitiesManager, authorizer, csContext, queueEntity, readLock, writeLock
Constructor and Description |
---|
LeafQueue(CapacitySchedulerContext cs,
String queueName,
CSQueue parent,
CSQueue old) |
Modifier and Type | Method and Description |
---|---|
boolean |
accept(org.apache.hadoop.yarn.api.records.Resource cluster,
ResourceCommitRequest<FiCaSchedulerApp,FiCaSchedulerNode> request) |
void |
apply(org.apache.hadoop.yarn.api.records.Resource cluster,
ResourceCommitRequest<FiCaSchedulerApp,FiCaSchedulerNode> request) |
CSAssignment |
assignContainers(org.apache.hadoop.yarn.api.records.Resource clusterResource,
PlacementSet<FiCaSchedulerNode> ps,
ResourceLimits currentResourceLimits,
SchedulingMode schedulingMode)
Assign containers to applications in the queue or it's children (if any).
|
void |
attachContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application,
RMContainer rmContainer)
Attach a container to this queue
|
org.apache.hadoop.yarn.api.records.Resource |
calculateAndGetAMResourceLimit() |
org.apache.hadoop.yarn.api.records.Resource |
calculateAndGetAMResourceLimitPerPartition(String nodePartition) |
protected boolean |
canAssignToUser(org.apache.hadoop.yarn.api.records.Resource clusterResource,
String userName,
org.apache.hadoop.yarn.api.records.Resource limit,
FiCaSchedulerApp application,
String nodePartition,
ResourceLimits currentResourceLimits) |
void |
collectSchedulerApplications(Collection<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> apps)
Adds all applications in the queue and its subqueues to the given collection.
|
void |
completedContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application,
FiCaSchedulerNode node,
RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event,
CSQueue childQueue,
boolean sortQueues)
A container assigned to the queue has completed.
|
void |
decAMUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToDec,
SchedulerApplicationAttempt application) |
void |
decUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToDec,
SchedulerApplicationAttempt application)
When partition of node updated, we will update queue's resource usage if it
has container(s) running on that.
|
void |
detachContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application,
RMContainer rmContainer)
Detach a container from this queue
|
void |
finishApplication(org.apache.hadoop.yarn.api.records.ApplicationId application,
String user)
An application submitted to this queue has finished.
|
void |
finishApplicationAttempt(FiCaSchedulerApp application,
String queue)
An application attempt submitted to this queue has finished.
|
AbstractUsersManager |
getAbstractUsersManager()
Get the
AbstractUsersManager for the queue. |
Collection<FiCaSchedulerApp> |
getAllApplications()
Obtain (read-only) collection of all applications.
|
Set<String> |
getAllUsers()
Get all valid users in this queue.
|
org.apache.hadoop.yarn.api.records.Resource |
getAMResourceLimit() |
org.apache.hadoop.yarn.api.records.Resource |
getAMResourceLimitPerPartition(String nodePartition) |
Collection<FiCaSchedulerApp> |
getApplications()
Obtain (read-only) collection of active applications.
|
List<CSQueue> |
getChildQueues()
Get child queues
|
org.apache.hadoop.yarn.api.records.Priority |
getDefaultApplicationPriority()
Get the Default Application Priority for this queue
|
protected org.apache.hadoop.yarn.api.records.Resource |
getHeadroom(UsersManager.User user,
org.apache.hadoop.yarn.api.records.Resource queueCurrentLimit,
org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application) |
protected org.apache.hadoop.yarn.api.records.Resource |
getHeadroom(UsersManager.User user,
org.apache.hadoop.yarn.api.records.Resource queueCurrentLimit,
org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerApp application,
String partition) |
Map<String,TreeSet<RMContainer>> |
getIgnoreExclusivityRMContainers() |
float |
getMaxAMResourcePerQueuePercent()
Used only by tests.
|
int |
getMaxApplications() |
int |
getMaxApplicationsPerUser() |
float |
getMinimumAllocationFactor()
Used only by tests.
|
int |
getNodeLocalityDelay() |
int |
getNumActiveApplications() |
int |
getNumActiveApplications(String user) |
int |
getNumApplications()
Get the number of applications in the queue.
|
int |
getNumPendingApplications() |
int |
getNumPendingApplications(String user) |
OrderingPolicy<FiCaSchedulerApp> |
getOrderingPolicy() |
Collection<FiCaSchedulerApp> |
getPendingApplications()
Obtain (read-only) collection of pending applications.
|
OrderingPolicy<FiCaSchedulerApp> |
getPendingAppsOrderingPolicy() |
List<AppPriorityACLGroup> |
getPriorityACLs() |
org.apache.hadoop.yarn.api.records.QueueInfo |
getQueueInfo(boolean includeChildQueues,
boolean recursive)
Get queue information
|
List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> |
getQueueUserAclInfo(org.apache.hadoop.security.UserGroupInformation user)
Get queue ACLs for given
user . |
int |
getRackLocalityAdditionalDelay() |
boolean |
getRackLocalityFullReset() |
org.apache.hadoop.yarn.api.records.Resource |
getResourceLimitForActiveUsers(String userName,
org.apache.hadoop.yarn.api.records.Resource clusterResource,
String nodePartition,
SchedulingMode schedulingMode) |
org.apache.hadoop.yarn.api.records.Resource |
getResourceLimitForAllUsers(String userName,
org.apache.hadoop.yarn.api.records.Resource clusterResource,
String nodePartition,
SchedulingMode schedulingMode) |
org.apache.hadoop.yarn.api.records.Resource |
getTotalPendingResourcesConsideringUserLimit(org.apache.hadoop.yarn.api.records.Resource clusterResources,
String partition,
boolean deductReservedFromPending)
Get total pending resource considering user limit for the leaf queue.
|
UsersManager.User |
getUser(String userName) |
org.apache.hadoop.yarn.api.records.Resource |
getUserAMResourceLimit() |
org.apache.hadoop.yarn.api.records.Resource |
getUserAMResourceLimitPerPartition(String nodePartition,
String userName) |
int |
getUserLimit() |
float |
getUserLimitFactor() |
UsersManager |
getUsersManager() |
void |
incAMUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToInc,
SchedulerApplicationAttempt application) |
void |
incUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToInc,
SchedulerApplicationAttempt application)
When partition of node updated, we will update queue's resource usage if it
has container(s) running on that.
|
void |
recalculateQueueUsageRatio(org.apache.hadoop.yarn.api.records.Resource clusterResource,
String nodePartition)
Recalculate QueueUsage Ratio.
|
void |
recoverContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource,
SchedulerApplicationAttempt attempt,
RMContainer rmContainer)
Recover the state of the queue for a given container.
|
void |
reinitialize(CSQueue newlyParsedQueue,
org.apache.hadoop.yarn.api.records.Resource clusterResource)
Reinitialize the queue.
|
void |
setAbsoluteCapacity(float absoluteCapacity) |
void |
setCapacity(float capacity) |
void |
setMaxApplications(int maxApplications) |
protected void |
setupQueueConfigs(org.apache.hadoop.yarn.api.records.Resource clusterResource) |
void |
stopQueue()
Stop the queue.
|
void |
submitApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String userName,
String queue)
Submit a new application to the queue.
|
void |
submitApplicationAttempt(FiCaSchedulerApp application,
String userName)
Submit an application attempt to the queue.
|
String |
toString() |
void |
updateApplicationPriority(SchedulerApplication<FiCaSchedulerApp> app,
org.apache.hadoop.yarn.api.records.Priority newAppPriority) |
void |
updateClusterResource(org.apache.hadoop.yarn.api.records.Resource clusterResource,
ResourceLimits currentResourceLimits)
Update the cluster resource for queues as we add/remove nodes
|
void |
validateSubmitApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String userName,
String queue)
Validate submitApplication api so that moveApplication do a pre-check.
|
accessibleToPartition, activeQueue, appFinished, assignContainers, decPendingResource, decReservedResource, getAbsoluteCapacity, getAbsoluteMaximumCapacity, getAbsoluteUsedCapacity, getAccessibleNodeLabels, getACLs, getCapacity, getDefaultApplicationLifetime, getDefaultAppLifetimeWasSpecifiedInConfig, getDefaultNodeLabelExpression, getIntraQueuePreemptionDisabled, getIntraQueuePreemptionDisabledInHierarchy, getKillableContainers, getMaximumAllocation, getMaximumApplicationLifetime, getMaximumCapacity, getMetrics, getMinimumAllocation, getNodeLabelsForQueue, getNumContainers, getParent, getPreemptionDisabled, getPriority, getPrivilegedEntity, getQueueCapacities, getQueueConfigurations, getQueueInfo, getQueueName, getQueuePath, getQueueResourceUsage, getQueueStatistics, getReadLock, getReservationContinueLooking, getState, getTotalKillableResource, getUsedCapacity, getUsedResources, getUserWeights, hasAccess, hasChildQueues, incPendingResource, incReservedResource, releaseResource, setParent, setupConfigurableCapacities, updateQueueState
protected int maxApplications
protected volatile int maxApplicationsPerUser
public LeafQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException
IOException
protected void setupQueueConfigs(org.apache.hadoop.yarn.api.records.Resource clusterResource) throws IOException
IOException
@InterfaceAudience.Private public float getMinimumAllocationFactor()
@InterfaceAudience.Private public float getMaxAMResourcePerQueuePercent()
public int getMaxApplications()
public int getMaxApplicationsPerUser()
public UsersManager getUsersManager()
public AbstractUsersManager getAbstractUsersManager()
CSQueue
AbstractUsersManager
for the queue.AbstractUsersManager
for the queuepublic List<CSQueue> getChildQueues()
CSQueue
public int getNumApplications()
CSQueue
public int getNumPendingApplications()
public int getNumActiveApplications()
@InterfaceAudience.Private public int getNumPendingApplications(String user)
@InterfaceAudience.Private public int getNumActiveApplications(String user)
@InterfaceAudience.Private public int getUserLimit()
@InterfaceAudience.Private public float getUserLimitFactor()
public org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive)
Queue
includeChildQueues
- include child queues?recursive
- recursively get child queue information?public List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> getQueueUserAclInfo(org.apache.hadoop.security.UserGroupInformation user)
Queue
user
.user
- usernamepublic UsersManager.User getUser(String userName)
@InterfaceAudience.Private public List<AppPriorityACLGroup> getPriorityACLs()
public void reinitialize(CSQueue newlyParsedQueue, org.apache.hadoop.yarn.api.records.Resource clusterResource) throws IOException
CSQueue
newlyParsedQueue
- new queue to re-initalize fromclusterResource
- resources in the clusterIOException
public void submitApplicationAttempt(FiCaSchedulerApp application, String userName)
CSQueue
public void submitApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId, String userName, String queue) throws org.apache.hadoop.security.AccessControlException
CSQueue
applicationId
- the applicationId of the application being submitteduserName
- user who submitted the applicationqueue
- queue to which the application is submittedorg.apache.hadoop.security.AccessControlException
public void validateSubmitApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId, String userName, String queue) throws org.apache.hadoop.security.AccessControlException
CSQueue
validateSubmitApplication
in interface CSQueue
validateSubmitApplication
in class AbstractCSQueue
applicationId
- Application IDuserName
- User Namequeue
- Queue Nameorg.apache.hadoop.security.AccessControlException
- if any acl violation is there.public org.apache.hadoop.yarn.api.records.Resource getAMResourceLimit()
public org.apache.hadoop.yarn.api.records.Resource getAMResourceLimitPerPartition(String nodePartition)
public org.apache.hadoop.yarn.api.records.Resource calculateAndGetAMResourceLimit()
public org.apache.hadoop.yarn.api.records.Resource getUserAMResourceLimit()
public org.apache.hadoop.yarn.api.records.Resource getUserAMResourceLimitPerPartition(String nodePartition, String userName)
public org.apache.hadoop.yarn.api.records.Resource calculateAndGetAMResourceLimitPerPartition(String nodePartition)
public void finishApplication(org.apache.hadoop.yarn.api.records.ApplicationId application, String user)
CSQueue
user
- user who submitted the applicationpublic void finishApplicationAttempt(FiCaSchedulerApp application, String queue)
CSQueue
public CSAssignment assignContainers(org.apache.hadoop.yarn.api.records.Resource clusterResource, PlacementSet<FiCaSchedulerNode> ps, ResourceLimits currentResourceLimits, SchedulingMode schedulingMode)
CSQueue
clusterResource
- the resource of the cluster.ps
- PlacementSet
of nodes which resources are availablecurrentResourceLimits
- how much overall resource of this queue can use.schedulingMode
- Type of exclusive check when assign container on a
NodeManager, see SchedulingMode
.public boolean accept(org.apache.hadoop.yarn.api.records.Resource cluster, ResourceCommitRequest<FiCaSchedulerApp,FiCaSchedulerNode> request)
accept
in interface CSQueue
accept
in class AbstractCSQueue
public void apply(org.apache.hadoop.yarn.api.records.Resource cluster, ResourceCommitRequest<FiCaSchedulerApp,FiCaSchedulerNode> request)
protected org.apache.hadoop.yarn.api.records.Resource getHeadroom(UsersManager.User user, org.apache.hadoop.yarn.api.records.Resource queueCurrentLimit, org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application)
protected org.apache.hadoop.yarn.api.records.Resource getHeadroom(UsersManager.User user, org.apache.hadoop.yarn.api.records.Resource queueCurrentLimit, org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application, String partition)
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public int getNodeLocalityDelay()
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public int getRackLocalityAdditionalDelay()
@Lock(value=org.apache.hadoop.yarn.server.utils.Lock.NoLock.class) public boolean getRackLocalityFullReset()
public org.apache.hadoop.yarn.api.records.Resource getResourceLimitForActiveUsers(String userName, org.apache.hadoop.yarn.api.records.Resource clusterResource, String nodePartition, SchedulingMode schedulingMode)
userName
- Name of user who has submitted one/more app to given queue.clusterResource
- total cluster resourcenodePartition
- partition nameschedulingMode
- scheduling mode
RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITYpublic org.apache.hadoop.yarn.api.records.Resource getResourceLimitForAllUsers(String userName, org.apache.hadoop.yarn.api.records.Resource clusterResource, String nodePartition, SchedulingMode schedulingMode)
userName
- Name of user who has submitted one/more app to given queue.clusterResource
- total cluster resourcenodePartition
- partition nameschedulingMode
- scheduling mode
RESPECT_PARTITION_EXCLUSIVITY/IGNORE_PARTITION_EXCLUSIVITY@InterfaceAudience.Private protected boolean canAssignToUser(org.apache.hadoop.yarn.api.records.Resource clusterResource, String userName, org.apache.hadoop.yarn.api.records.Resource limit, FiCaSchedulerApp application, String nodePartition, ResourceLimits currentResourceLimits)
public void recalculateQueueUsageRatio(org.apache.hadoop.yarn.api.records.Resource clusterResource, String nodePartition)
clusterResource
- Total Cluster ResourcenodePartition
- Partitionpublic void completedContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue, boolean sortQueues)
CSQueue
clusterResource
- the resource of the clusterapplication
- application to which the container was assignednode
- node on which the container completedrmContainer
- completed container,
null
if it was just a reservationcontainerStatus
- ContainerStatus
for the completed
containerevent
- event to be sent to the containerchildQueue
- CSQueue
to reinsert in childQueuessortQueues
- indicates whether it should re-sort the queuespublic void updateClusterResource(org.apache.hadoop.yarn.api.records.Resource clusterResource, ResourceLimits currentResourceLimits)
CSQueue
clusterResource
- the current cluster resourcecurrentResourceLimits
- the current ResourceLimitspublic void incUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToInc, SchedulerApplicationAttempt application)
CSQueue
incUsedResource
in interface CSQueue
incUsedResource
in class AbstractCSQueue
public void decUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToDec, SchedulerApplicationAttempt application)
CSQueue
decUsedResource
in interface CSQueue
decUsedResource
in class AbstractCSQueue
public void incAMUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToInc, SchedulerApplicationAttempt application)
public void decAMUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToDec, SchedulerApplicationAttempt application)
public void recoverContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, SchedulerApplicationAttempt attempt, RMContainer rmContainer)
Queue
clusterResource
- the resource of the clusterattempt
- the application for which the container was allocatedrmContainer
- the container that was recovered.public Collection<FiCaSchedulerApp> getPendingApplications()
public Collection<FiCaSchedulerApp> getApplications()
public Collection<FiCaSchedulerApp> getAllApplications()
public org.apache.hadoop.yarn.api.records.Resource getTotalPendingResourcesConsideringUserLimit(org.apache.hadoop.yarn.api.records.Resource clusterResources, String partition, boolean deductReservedFromPending)
clusterResources
- clusterResourcepartition
- node partitiondeductReservedFromPending
- When a container is reserved in CS,
pending resource will not be deducted.
This could lead to double accounting when
doing preemption:
In normal cases, we should deduct reserved
resource from pending to avoid
excessive preemption.public void collectSchedulerApplications(Collection<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> apps)
CSQueue
apps
- the collection to add the applications topublic void attachContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application, RMContainer rmContainer)
CSQueue
clusterResource
- the current cluster resourceapplication
- application to which the container was assignedrmContainer
- the container to attachpublic void detachContainer(org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerApp application, RMContainer rmContainer)
CSQueue
clusterResource
- the current cluster resourceapplication
- application to which the container was assignedrmContainer
- the container to detachpublic Map<String,TreeSet<RMContainer>> getIgnoreExclusivityRMContainers()
public void setCapacity(float capacity)
public void setAbsoluteCapacity(float absoluteCapacity)
public void setMaxApplications(int maxApplications)
public OrderingPolicy<FiCaSchedulerApp> getOrderingPolicy()
public org.apache.hadoop.yarn.api.records.Priority getDefaultApplicationPriority()
Queue
getDefaultApplicationPriority
in interface Queue
getDefaultApplicationPriority
in class AbstractCSQueue
public void updateApplicationPriority(SchedulerApplication<FiCaSchedulerApp> app, org.apache.hadoop.yarn.api.records.Priority newAppPriority)
public OrderingPolicy<FiCaSchedulerApp> getPendingAppsOrderingPolicy()
public void stopQueue()
SchedulerQueue
Copyright © 2020 Apache Software Foundation. All rights reserved.