Skip to content

YARN-11253. Add Configuration to delegationToken RemoverScanInterval. #4751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 27, 2022
Merged

YARN-11253. Add Configuration to delegationToken RemoverScanInterval. #4751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9a42bf9
YARN-11253. Add Configuration to delegationToken RemoverScanInterval.
Aug 17, 2022
f4f4cef
YARN-11253. Fix CheckStyle.
Aug 17, 2022
11c0724
YARN-11253. Add Junit Test.
Aug 17, 2022
d1a76ac
Merge branch 'apache:trunk' into YARN-11253
slfan1989 Aug 24, 2022
4113ca9
YARN-11253. Fix CheckStyle.
Aug 25, 2022
70abbaa
Merge branch 'apache:trunk' into YARN-11253
slfan1989 Aug 25, 2022
1cc1b35
YARN-11253. Fix CheckStyle.
Aug 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ...oop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -793,6 +793,10 @@ public static boolean isAclEnabled(Configuration conf) {
RM_PREFIX + "delegation.token.max-lifetime";
public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
7*24*60*60*1000; // 7 days
public static final String RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY =
RM_PREFIX + "delegation.token.remove-scan-interval";
public static final long RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT =
60*60*1000; // 1 hour

public static final String RM_DELEGATION_TOKEN_MAX_CONF_SIZE =
RM_PREFIX + "delegation-token.max-conf-size-bytes";
Expand Down
12 changes: 12 additions & 0 deletions hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1077,6 +1077,18 @@
<value>86400000</value>
</property>

<property>
<description>
This configuration is used for
how often the tokens are scanned for expired tokens in milliseconds.
the background thread(delegation token remover thread)
will delete expired tokens after the configured time.
the default value is 1h.
</description>
<name>yarn.resourcemanager.delegation.token.remove-scan-interval</name>
<value>1h</value>
</property>

<property>
<description>
RM DelegationTokenRenewer thread timeout
Expand Down
7 changes: 6 additions & 1 deletion ...r/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;

import java.io.IOException;
import java.util.concurrent.TimeUnit;

public class RMSecretManagerService extends AbstractService {

Expand Down Expand Up @@ -135,9 +136,13 @@ protected RMDelegationTokenSecretManager createRMDelegationTokenSecretManager(
long tokenRenewInterval =
conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
long removeScanInterval =
conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY,
YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);

return new RMDelegationTokenSecretManager(secretKeyInterval,
tokenMaxLifetime, tokenRenewInterval, 3600000, rmContext);
tokenMaxLifetime, tokenRenewInterval, removeScanInterval, rmContext);
}

}
12 changes: 9 additions & 3 deletions ...nager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.TimeUnit;

import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.thirdparty.protobuf.InvalidProtocolBufferException;
Expand Down Expand Up @@ -124,9 +125,13 @@ public void testDelegationToken() throws Exception {
long initialInterval = 10000l;
long maxLifetime= 20000l;
long renewInterval = 10000l;
long delegationTokenRemoverScanInterval =
conf.getTimeDuration(YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_KEY,
YarnConfiguration.RM_DELEGATION_TOKEN_REMOVE_SCAN_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);

RMDelegationTokenSecretManager rmDtSecretManager = createRMDelegationTokenSecretManager(
initialInterval, maxLifetime, renewInterval);
initialInterval, maxLifetime, renewInterval, delegationTokenRemoverScanInterval);
rmDtSecretManager.startThreads();
LOG.info("Creating DelegationTokenSecretManager with initialInterval: "
+ initialInterval + ", maxLifetime: " + maxLifetime
Expand Down Expand Up @@ -574,7 +579,8 @@ private static ResourceScheduler createMockScheduler(Configuration conf) {

private static RMDelegationTokenSecretManager
createRMDelegationTokenSecretManager(long secretKeyInterval,
long tokenMaxLifetime, long tokenRenewInterval) {
long tokenMaxLifetime, long tokenRenewInterval,
long delegationTokenRemoverScanInterval) {
ResourceManager rm = mock(ResourceManager.class);
RMContext rmContext = mock(RMContext.class);
when(rmContext.getStateStore()).thenReturn(new NullRMStateStore());
Expand All @@ -583,7 +589,7 @@ private static ResourceScheduler createMockScheduler(Configuration conf) {

RMDelegationTokenSecretManager rmDtSecretManager =
new RMDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime,
tokenRenewInterval, 3600000, rmContext);
tokenRenewInterval, delegationTokenRemoverScanInterval, rmContext);
return rmDtSecretManager;
}
}