Skip to content

Commit bd8240b

Browse files
slfan1989slfan1989
authored
Merge branch 'apache:trunk' into YARN-11252 
2 parents 0c23966 + cd72f7e commit bd8240b

File tree

12 files changed

+265
-13
lines changed

12 files changed

+265
-13
lines changed

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -489,4 +489,7 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
489489
public static final boolean IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT =
490490
true;
491491

492+
public static final String HADOOP_SECURITY_RESOLVER_IMPL =
493+
"hadoop.security.resolver.impl";
494+
492495
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818

1919
package org.apache.hadoop.net;
2020

21+
import org.slf4j.Logger;
22+
import org.slf4j.LoggerFactory;
23+
24+
import javax.naming.NamingException;
2125
import java.net.InetAddress;
2226
import java.net.UnknownHostException;
2327

@@ -27,6 +31,10 @@
2731
* fully qualified domain names belonging to the IPs from this host name
2832
*/
2933
public class DNSDomainNameResolver implements DomainNameResolver {
34+
35+
private final static Logger LOG =
36+
LoggerFactory.getLogger(DNSDomainNameResolver.class.getName());
37+
3038
@Override
3139
public InetAddress[] getAllByDomainName(String domainName)
3240
throws UnknownHostException {
@@ -40,6 +48,16 @@ public String getHostnameByIP(InetAddress address) {
4048
&& host.charAt(host.length()-1) == '.') {
4149
host = host.substring(0, host.length()-1);
4250
}
51+
// Protect against the Java behaviour of returning the IP address as a string from a cache
52+
// instead of performing a reverse lookup.
53+
if (host != null && host.equals(address.getHostAddress())) {
54+
LOG.debug("IP address returned for FQDN detected: {}", address.getHostAddress());
55+
try {
56+
return DNS.reverseDns(address, null);
57+
} catch (NamingException lookupFailure) {
58+
LOG.warn("Failed to perform reverse lookup: {}", address);
59+
}
60+
}
4361
return host;
4462
}
4563

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444
import org.apache.hadoop.fs.CommonConfigurationKeys;
4545
import org.apache.hadoop.io.Text;
4646
import org.apache.hadoop.net.DNS;
47+
import org.apache.hadoop.net.DomainNameResolver;
48+
import org.apache.hadoop.net.DomainNameResolverFactory;
4749
import org.apache.hadoop.net.NetUtils;
4850
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
4951
import org.apache.hadoop.security.token.Token;
@@ -81,6 +83,8 @@ private SecurityUtil() {
8183
@VisibleForTesting
8284
static HostResolver hostResolver;
8385

86+
private static DomainNameResolver domainNameResolver;
87+
8488
private static boolean logSlowLookups;
8589
private static int slowLookupThresholdMs;
8690

@@ -112,6 +116,9 @@ private static void setConfigurationInternal(Configuration conf) {
112116
.HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_KEY,
113117
CommonConfigurationKeys
114118
.HADOOP_SECURITY_DNS_LOG_SLOW_LOOKUPS_THRESHOLD_MS_DEFAULT);
119+
120+
domainNameResolver = DomainNameResolverFactory.newInstance(conf,
121+
CommonConfigurationKeys.HADOOP_SECURITY_RESOLVER_IMPL);
115122
}
116123

117124
/**
@@ -212,7 +219,7 @@ public static String getServerPrincipal(String principalConfig,
212219
throw new IOException("Can't replace " + HOSTNAME_PATTERN
213220
+ " pattern since client address is null");
214221
}
215-
return replacePattern(components, addr.getCanonicalHostName());
222+
return replacePattern(components, domainNameResolver.getHostnameByIP(addr));
216223
}
217224
}
218225

‎hadoop-common-project/hadoop-common/src/main/resources/core-default.xml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,14 @@
130130
</description>
131131
</property>
132132

133+
<property>
134+
<name>hadoop.security.resolver.impl</name>
135+
<value>org.apache.hadoop.net.DNSDomainNameResolver</value>
136+
<description>
137+
The resolver implementation used to resolve FQDN for Kerberos
138+
</description>
139+
</property>
140+
133141
<property>
134142
<name>hadoop.security.dns.log-slow-lookups.enabled</name>
135143
<value>false</value>

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNSDomainNameResolver.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.net;
19+
20+
import org.junit.Test;
21+
22+
import java.net.InetAddress;
23+
import java.net.UnknownHostException;
24+
import java.util.Objects;
25+
26+
import static org.junit.Assert.assertEquals;
27+
import static org.junit.Assert.assertNotEquals;
28+
import static org.junit.Assume.assumeFalse;
29+
30+
31+
public class TestDNSDomainNameResolver {
32+
33+
static DNSDomainNameResolver DNR = new DNSDomainNameResolver();
34+
35+
@Test
36+
public void testGetHostNameByIP() throws UnknownHostException {
37+
InetAddress localhost = InetAddress.getLocalHost();
38+
assumeFalse("IP lookup support required",
39+
Objects.equals(localhost.getCanonicalHostName(), localhost.getHostAddress()));
40+
41+
// Precondition: host name and canonical host name for unresolved returns an IP address.
42+
InetAddress unresolved = InetAddress.getByAddress(localhost.getHostAddress(),
43+
localhost.getAddress());
44+
assertEquals(localhost.getHostAddress(), unresolved.getHostName());
45+
46+
// Test: Get the canonical name despite InetAddress caching
47+
String canonicalHostName = DNR.getHostnameByIP(unresolved);
48+
49+
// Verify: The canonical host name doesn't match the host address but does match the localhost.
50+
assertNotEquals(localhost.getHostAddress(), canonicalHostName);
51+
assertEquals(localhost.getCanonicalHostName(), canonicalHostName);
52+
}
53+
54+
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@ private void check() {
201201
iterkey).iterator();
202202
final List<DatanodeDescriptor> toRemove = new ArrayList<>();
203203
final List<DatanodeDescriptor> unhealthyDns = new ArrayList<>();
204+
boolean isValidState = true;
204205

205206
while (it.hasNext() && !exceededNumBlocksPerCheck() && namesystem
206207
.isRunning()) {
@@ -265,6 +266,7 @@ private void check() {
265266
// to track maintenance expiration.
266267
dnAdmin.setInMaintenance(dn);
267268
} else {
269+
isValidState = false;
268270
Preconditions.checkState(false,
269271
"Node %s is in an invalid state! "
270272
+ "Invalid state: %s %s blocks are on this dn.",
@@ -288,7 +290,11 @@ private void check() {
288290
// an invalid state.
289291
LOG.warn("DatanodeAdminMonitor caught exception when processing node "
290292
+ "{}.", dn, e);
291-
getPendingNodes().add(dn);
293+
if(isValidState){
294+
getPendingNodes().add(dn);
295+
} else {
296+
LOG.warn("Ignoring the node {} which is in invalid state", dn);
297+
}
292298
toRemove.add(dn);
293299
unhealthyDns.remove(dn);
294300
} finally {

‎hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ Certain HDFS operations, i.e., `hflush`, `hsync`, `concat`, `setReplication`, `t
234234
are not supported on erasure coded files due to substantial technical
235235
challenges.
236236

237-
* `append()` and `truncate()` on an erasure coded file will throw `IOException`.
237+
* `append()` and `truncate()` on an erasure coded file will throw `IOException`. However, with `NEW_BLOCK` flag enabled, append to a closed striped file is supported.
238238
* `concat()` will throw `IOException` if files are mixed with different erasure
239239
coding policies or with replicated files.
240240
* `setReplication()` is no-op on erasure coded files.

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1005,13 +1005,17 @@ protected RequestFactory createRequestFactory() {
10051005
String storageClassConf = getConf()
10061006
.getTrimmed(STORAGE_CLASS, "")
10071007
.toUpperCase(Locale.US);
1008-
StorageClass storageClass;
1009-
try {
1010-
storageClass = StorageClass.fromValue(storageClassConf);
1011-
} catch (IllegalArgumentException e) {
1012-
LOG.warn("Unknown storage class property {}: {}; falling back to default storage class",
1013-
STORAGE_CLASS, storageClassConf);
1014-
storageClass = null;
1008+
StorageClass storageClass = null;
1009+
if (!storageClassConf.isEmpty()) {
1010+
try {
1011+
storageClass = StorageClass.fromValue(storageClassConf);
1012+
} catch (IllegalArgumentException e) {
1013+
LOG.warn("Unknown storage class property {}: {}; falling back to default storage class",
1014+
STORAGE_CLASS, storageClassConf);
1015+
}
1016+
} else {
1017+
LOG.debug("Unset storage class property {}; falling back to default storage class",
1018+
STORAGE_CLASS);
10151019
}
10161020

10171021
return RequestFactoryImpl.builder()

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
5656
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
5757
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
58+
import org.apache.hadoop.yarn.server.api.records.NodeAction;
5859
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
5960
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
6061
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -82,6 +83,7 @@
8283
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
8384
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
8485
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
86+
import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
8587
import org.apache.hadoop.yarn.util.Records;
8688
import org.apache.hadoop.yarn.util.resource.Resources;
8789
import org.junit.After;
@@ -1502,4 +1504,27 @@ public void testCalculateHeartBeatInterval() {
15021504
calcIntervalTest(rmNode, nodeUtil, hbDefault, hbMin, hbMax,
15031505
speedup, slowdown, vcoreUnit * 1.0F, hbDefault); // 100%
15041506
}
1507+
1508+
@Test
1509+
public void testFinishedContainersPulledByAmOnDecommissioningNode() {
1510+
RMNodeImpl rMNodeImpl = getRunningNode();
1511+
rMNodeImpl.handle(
1512+
new RMNodeEvent(rMNodeImpl.getNodeID(), RMNodeEventType.GRACEFUL_DECOMMISSION));
1513+
Assert.assertEquals(NodeState.DECOMMISSIONING, rMNodeImpl.getState());
1514+
1515+
ContainerId containerId = BuilderUtils.newContainerId(
1516+
BuilderUtils.newApplicationAttemptId(BuilderUtils.newApplicationId(0, 0), 0), 0);
1517+
List<ContainerId> containerIds = Arrays.asList(containerId);
1518+
1519+
rMNodeImpl.handle(
1520+
new RMNodeFinishedContainersPulledByAMEvent(rMNodeImpl.getNodeID(), containerIds));
1521+
Assert.assertEquals(NodeState.DECOMMISSIONING, rMNodeImpl.getState());
1522+
1523+
// Verify expected containersToBeRemovedFromNM from NodeHeartbeatResponse.
1524+
NodeHeartbeatResponse response =
1525+
YarnServerBuilderUtils.newNodeHeartbeatResponse(1, NodeAction.NORMAL, null, null, null,
1526+
null, 1000);
1527+
rMNodeImpl.setAndUpdateNodeHeartbeatResponse(response);
1528+
Assert.assertEquals(1, response.getContainersToBeRemovedFromNM().size());
1529+
}
15051530
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1312,14 +1312,52 @@ public Response updateApplicationPriority(AppPriority targetPriority,
13121312
@Override
13131313
public AppQueue getAppQueue(HttpServletRequest hsr, String appId)
13141314
throws AuthorizationException {
1315-
throw new NotImplementedException("Code is not implemented");
1315+
1316+
if (appId == null || appId.isEmpty()) {
1317+
throw new IllegalArgumentException("Parameter error, the appId is empty or null.");
1318+
}
1319+
1320+
try {
1321+
SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
1322+
DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
1323+
subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
1324+
return interceptor.getAppQueue(hsr, appId);
1325+
} catch (IllegalArgumentException e) {
1326+
RouterServerUtil.logAndThrowRunTimeException(e,
1327+
"Unable to get queue by appId: %s.", appId);
1328+
} catch (YarnException e) {
1329+
RouterServerUtil.logAndThrowRunTimeException("getAppQueue Failed.", e);
1330+
}
1331+
1332+
return null;
13161333
}
13171334

13181335
@Override
13191336
public Response updateAppQueue(AppQueue targetQueue, HttpServletRequest hsr,
13201337
String appId) throws AuthorizationException, YarnException,
13211338
InterruptedException, IOException {
1322-
throw new NotImplementedException("Code is not implemented");
1339+
1340+
if (appId == null || appId.isEmpty()) {
1341+
throw new IllegalArgumentException("Parameter error, the appId is empty or null.");
1342+
}
1343+
1344+
if (targetQueue == null) {
1345+
throw new IllegalArgumentException("Parameter error, the targetQueue is null.");
1346+
}
1347+
1348+
try {
1349+
SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
1350+
DefaultRequestInterceptorREST interceptor = getOrCreateInterceptorForSubCluster(
1351+
subClusterInfo.getSubClusterId(), subClusterInfo.getRMWebServiceAddress());
1352+
return interceptor.updateAppQueue(targetQueue, hsr, appId);
1353+
} catch (IllegalArgumentException e) {
1354+
RouterServerUtil.logAndThrowRunTimeException(e,
1355+
"Unable to update app queue by appId: %s.", appId);
1356+
} catch (YarnException e) {
1357+
RouterServerUtil.logAndThrowRunTimeException("updateAppQueue Failed.", e);
1358+
}
1359+
1360+
return null;
13231361
}
13241362

13251363
@Override

0 commit comments

Comments
 (0)