Skip to content

Commit 04a73e0

Browse files
sumangala17sumangala17
committed
grp createpath args, pre-computed key configs, x-ms-version
1 parent ae6fce6 commit 04a73e0

File tree

11 files changed

+117
-1060
lines changed

11 files changed

+117
-1060
lines changed

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1039,8 +1039,13 @@ public boolean enableAbfsListIterator() {
10391039
return this.enableAbfsListIterator;
10401040
}
10411041

1042-
public String getClientProvidedEncryptionKey() {
1043-
String accSpecEncKey = accountConf(FS_AZURE_ENCRYPTION_CLIENT_PROVIDED_KEY);
1042+
public String getEncodedClientProvidedEncryptionKey() {
1043+
String accSpecEncKey = accountConf(FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY);
1044+
return rawConfig.get(accSpecEncKey, null);
1045+
}
1046+
1047+
public String getEncodedClientProvidedEncryptionKeySHA() {
1048+
String accSpecEncKey = accountConf(FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY_SHA);
10441049
return rawConfig.get(accSpecEncKey, null);
10451050
}
10461051

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java‎

Lines changed: 46 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -563,8 +563,7 @@ public OutputStream createFile(final Path path,
563563
if (triggerConditionalCreateOverwrite) {
564564
op = conditionalCreateOverwriteFile(relativePath,
565565
statistics,
566-
isNamespaceEnabled ? getOctalNotation(permission) : null,
567-
isNamespaceEnabled ? getOctalNotation(umask) : null,
566+
new Permissions(isNamespaceEnabled, permission, umask),
568567
isAppendBlob,
569568
encryptionAdapter,
570569
tracingContext
@@ -573,8 +572,7 @@ public OutputStream createFile(final Path path,
573572
} else {
574573
op = client.createPath(relativePath, true,
575574
overwrite,
576-
isNamespaceEnabled ? getOctalNotation(permission) : null,
577-
isNamespaceEnabled ? getOctalNotation(umask) : null,
575+
new Permissions(isNamespaceEnabled, permission, umask),
578576
isAppendBlob,
579577
null,
580578
encryptionAdapter,
@@ -603,16 +601,14 @@ public OutputStream createFile(final Path path,
603601
* only if there is match for eTag of existing file.
604602
* @param relativePath
605603
* @param statistics
606-
* @param permission
607-
* @param umask
604+
* @param permissions contains permission and umask
608605
* @param isAppendBlob
609606
* @return
610607
* @throws AzureBlobFileSystemException
611608
*/
612609
private AbfsRestOperation conditionalCreateOverwriteFile(final String relativePath,
613610
final FileSystem.Statistics statistics,
614-
final String permission,
615-
final String umask,
611+
Permissions permissions,
616612
final boolean isAppendBlob,
617613
EncryptionAdapter encryptionAdapter,
618614
TracingContext tracingContext) throws IOException {
@@ -622,7 +618,7 @@ private AbfsRestOperation conditionalCreateOverwriteFile(final String relativePa
622618
// Trigger a create with overwrite=false first so that eTag fetch can be
623619
// avoided for cases when no pre-existing file is present (major portion
624620
// of create file traffic falls into the case of no pre-existing file).
625-
op = client.createPath(relativePath, true, false, permission, umask,
621+
op = client.createPath(relativePath, true, false, permissions,
626622
isAppendBlob, null, encryptionAdapter, tracingContext);
627623

628624
} catch (AbfsRestOperationException e) {
@@ -647,7 +643,7 @@ private AbfsRestOperation conditionalCreateOverwriteFile(final String relativePa
647643

648644
try {
649645
// overwrite only if eTag matches with the file properties fetched befpre
650-
op = client.createPath(relativePath, true, true, permission, umask,
646+
op = client.createPath(relativePath, true, true, permissions,
651647
isAppendBlob, eTag, encryptionAdapter, tracingContext);
652648
} catch (AbfsRestOperationException ex) {
653649
if (ex.getStatusCode() == HttpURLConnection.HTTP_PRECON_FAILED) {
@@ -734,11 +730,10 @@ public void createDirectory(final Path path, final FsPermission permission,
734730

735731
boolean overwrite =
736732
!isNamespaceEnabled || abfsConfiguration.isEnabledMkdirOverwrite();
733+
Permissions permissions = new Permissions(isNamespaceEnabled,
734+
permission, umask);
737735
final AbfsRestOperation op = client.createPath(getRelativePath(path),
738-
false, overwrite,
739-
isNamespaceEnabled ? getOctalNotation(permission) : null,
740-
isNamespaceEnabled ? getOctalNotation(umask) : null, false,
741-
null, null, tracingContext);
736+
false, overwrite, permissions, false, null, null, tracingContext);
742737
perfInfo.registerResult(op.getResult()).registerSuccess(true);
743738
}
744739
}
@@ -1632,16 +1627,21 @@ private void initializeClient(URI uri, String fileSystemName,
16321627
encryptionContextProvider =
16331628
abfsConfiguration.createEncryptionContextProvider();
16341629
if (encryptionContextProvider != null) {
1635-
if (abfsConfiguration.getClientProvidedEncryptionKey() != null) {
1630+
if (abfsConfiguration.getEncodedClientProvidedEncryptionKey() != null) {
16361631
throw new IOException(
16371632
"Both global key and encryption context are set, only one allowed");
16381633
}
16391634
encryptionContextProvider.initialize(
16401635
abfsConfiguration.getRawConfiguration(), accountName,
16411636
fileSystemName);
16421637
encryptionType = EncryptionType.ENCRYPTION_CONTEXT;
1643-
} else if (abfsConfiguration.getClientProvidedEncryptionKey() != null) {
1644-
encryptionType = EncryptionType.GLOBAL_KEY;
1638+
} else if (abfsConfiguration.getEncodedClientProvidedEncryptionKey() != null) {
1639+
if (abfsConfiguration.getEncodedClientProvidedEncryptionKeySHA() != null) {
1640+
encryptionType = EncryptionType.GLOBAL_KEY;
1641+
} else {
1642+
throw new IOException(
1643+
"Encoded SHA256 hash must be provided for global encryption");
1644+
}
16451645
}
16461646
}
16471647

@@ -1655,7 +1655,6 @@ private void initializeClient(URI uri, String fileSystemName,
16551655
sasTokenProvider, encryptionContextProvider,
16561656
populateAbfsClientContext());
16571657
}
1658-
client.setEncryptionType(encryptionType);
16591658

16601659
LOG.trace("AbfsClient init complete");
16611660
}
@@ -1674,11 +1673,6 @@ private AbfsClientContext populateAbfsClientContext() {
16741673
.build();
16751674
}
16761675

1677-
private String getOctalNotation(FsPermission fsPermission) {
1678-
Preconditions.checkNotNull(fsPermission, "fsPermission");
1679-
return String.format(AbfsHttpConstants.PERMISSION_FORMAT, fsPermission.toOctal());
1680-
}
1681-
16821676
public String getRelativePath(final Path path) {
16831677
Preconditions.checkNotNull(path, "path");
16841678
return path.toUri().getPath();
@@ -1866,6 +1860,35 @@ public String toString() {
18661860
}
18671861
}
18681862

1863+
public static class Permissions {
1864+
private final String permission;
1865+
private final String umask;
1866+
1867+
Permissions(boolean isNamespaceEnabled, FsPermission permission,
1868+
FsPermission umask) {
1869+
if (isNamespaceEnabled) {
1870+
this.permission = getOctalNotation(permission);
1871+
this.umask = getOctalNotation(umask);
1872+
} else {
1873+
this.permission = null;
1874+
this.umask = null;
1875+
}
1876+
}
1877+
1878+
private String getOctalNotation(FsPermission fsPermission) {
1879+
Preconditions.checkNotNull(fsPermission, "fsPermission");
1880+
return String.format(AbfsHttpConstants.PERMISSION_FORMAT, fsPermission.toOctal());
1881+
}
1882+
1883+
public String getPermission() {
1884+
return permission;
1885+
}
1886+
1887+
public String getUmask() {
1888+
return umask;
1889+
}
1890+
}
1891+
18691892
/**
18701893
* A builder class for AzureBlobFileSystemStore.
18711894
*/

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,8 +188,12 @@ public final class ConfigurationKeys {
188188
public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script";
189189
/** Setting this true will make the driver use it's own RemoteIterator implementation */
190190
public static final String FS_AZURE_ENABLE_ABFS_LIST_ITERATOR = "fs.azure.enable.abfslistiterator";
191-
/** Server side encryption key */
192-
public static final String FS_AZURE_ENCRYPTION_CLIENT_PROVIDED_KEY = "fs.azure.encryption.client-provided-key";
191+
/** Server side encryption key encoded in Base6format */
192+
public static final String FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY =
193+
"fs.azure.encryption.encoded.client-provided-key";
194+
/** SHA256 hash of encryption key encoded in Base6format */
195+
public static final String FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY_SHA =
196+
"fs.azure.encryption.encoded.client-provided-key-sha";
193197
/** Custom EncryptionContextProvider type */
194198
public static final String FS_AZURE_ENCRYPTION_CONTEXT_PROVIDER_TYPE = "fs.azure.encryption.context.provider.type";
195199

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/security/EncryptionAdapter.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,9 @@ public SecretKey createEncryptionContext() throws IOException {
7676
public void computeKeys() throws IOException {
7777
SecretKey key = getEncryptionKey();
7878
Preconditions.checkNotNull(key, "Encryption key should not be null.");
79-
encodedKey = getBase64EncodedString(new String(key.getEncoded(),
80-
StandardCharsets.UTF_8));
79+
// encodedKey = getBase64EncodedString(new String(key.getEncoded(),
80+
// StandardCharsets.UTF_8));
81+
encodedKey = getBase64EncodedString(key.getEncoded());
8182
encodedKeySHA = getBase64EncodedString(getSHA256Hash(new String(key.getEncoded(),
8283
StandardCharsets.UTF_8)));
8384
}

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java‎

Lines changed: 28 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,6 @@
2626
import java.net.URL;
2727
import java.net.URLEncoder;
2828
import java.nio.charset.StandardCharsets;
29-
import java.time.Instant;
30-
import java.security.MessageDigest;
31-
import java.security.NoSuchAlgorithmException;
3229
import java.util.ArrayList;
3330
import java.util.List;
3431
import java.util.Locale;
@@ -39,6 +36,7 @@
3936

4037
import org.apache.hadoop.classification.VisibleForTesting;
4138
import org.apache.hadoop.fs.Path;
39+
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.Permissions;
4240
import org.apache.hadoop.fs.azurebfs.extensions.EncryptionContextProvider;
4341
import org.apache.hadoop.fs.azurebfs.security.EncryptionAdapter;
4442
import org.apache.hadoop.fs.azurebfs.utils.EncryptionType;
@@ -88,22 +86,22 @@ public class AbfsClient implements Closeable {
8886

8987
private final URL baseUrl;
9088
private final SharedKeyCredentials sharedKeyCredentials;
91-
private final String xMsVersion = "2021-04-10";//"2019-12-12";
89+
private String xMsVersion = "2019-12-12";
9290
private final ExponentialRetryPolicy retryPolicy;
9391
private final String filesystem;
9492
private final AbfsConfiguration abfsConfiguration;
9593
private final String userAgent;
9694
private final AbfsPerfTracker abfsPerfTracker;
97-
private String clientProvidedEncryptionKey;
98-
private final String clientProvidedEncryptionKeySHA;
95+
private String clientProvidedEncryptionKey = null;
96+
private String clientProvidedEncryptionKeySHA = null;
9997

10098
private final String accountName;
10199
private final AuthType authType;
102100
private AccessTokenProvider tokenProvider;
103101
private SASTokenProvider sasTokenProvider;
104102
private final AbfsCounters abfsCounters;
105-
private EncryptionContextProvider encryptionContextProvider;
106-
private EncryptionType encryptionType;
103+
private EncryptionContextProvider encryptionContextProvider = null;
104+
private EncryptionType encryptionType = EncryptionType.NONE;
107105

108106
private final ListeningScheduledExecutorService executorService;
109107

@@ -115,22 +113,23 @@ private AbfsClient(final URL baseUrl,
115113
this.baseUrl = baseUrl;
116114
this.sharedKeyCredentials = sharedKeyCredentials;
117115
String baseUrlString = baseUrl.toString();
118-
this.filesystem = baseUrlString.substring(baseUrlString.lastIndexOf(FORWARD_SLASH) + 1);
116+
this.filesystem = baseUrlString.substring(
117+
baseUrlString.lastIndexOf(FORWARD_SLASH) + 1);
119118
this.abfsConfiguration = abfsConfiguration;
120119
this.retryPolicy = abfsClientContext.getExponentialRetryPolicy();
121-
this.accountName = abfsConfiguration.getAccountName().substring(0, abfsConfiguration.getAccountName().indexOf(AbfsHttpConstants.DOT));
120+
this.accountName = abfsConfiguration.getAccountName().substring(0,
121+
abfsConfiguration.getAccountName().indexOf(AbfsHttpConstants.DOT));
122122
this.authType = abfsConfiguration.getAuthType(accountName);
123-
this.encryptionContextProvider = encryptionContextProvider;
124-
125-
String encryptionKey = this.abfsConfiguration
126-
.getClientProvidedEncryptionKey();
127-
if (encryptionKey != null) {
128-
this.clientProvidedEncryptionKey = EncryptionAdapter.getBase64EncodedString(encryptionKey);
129-
this.clientProvidedEncryptionKeySHA = EncryptionAdapter.getBase64EncodedString(
130-
EncryptionAdapter.getSHA256Hash(encryptionKey));
131-
} else {
132-
this.clientProvidedEncryptionKey = null;
133-
this.clientProvidedEncryptionKeySHA = null;
123+
124+
if (encryptionContextProvider != null) {
125+
this.encryptionContextProvider = encryptionContextProvider;
126+
xMsVersion = "2021-04-10"; // will be default once server change deployed
127+
encryptionType = EncryptionType.ENCRYPTION_CONTEXT;
128+
} else if ((clientProvidedEncryptionKey =
129+
abfsConfiguration.getEncodedClientProvidedEncryptionKey()) != null) {
130+
this.clientProvidedEncryptionKeySHA =
131+
abfsConfiguration.getEncodedClientProvidedEncryptionKeySHA();
132+
encryptionType = EncryptionType.GLOBAL_KEY;
134133
}
135134

136135
String sslProviderName = null;
@@ -389,7 +388,7 @@ public AbfsRestOperation deleteFilesystem(TracingContext tracingContext) throws
389388
}
390389

391390
public AbfsRestOperation createPath(final String path, final boolean isFile,
392-
final boolean overwrite, final String permission, final String umask,
391+
final boolean overwrite, final Permissions permissions,
393392
final boolean isAppendBlob, final String eTag,
394393
EncryptionAdapter encryptionAdapter, TracingContext tracingContext)
395394
throws IOException {
@@ -402,12 +401,15 @@ public AbfsRestOperation createPath(final String path, final boolean isFile,
402401
requestHeaders.add(new AbfsHttpHeader(IF_NONE_MATCH, AbfsHttpConstants.STAR));
403402
}
404403

405-
if (permission != null && !permission.isEmpty()) {
406-
requestHeaders.add(new AbfsHttpHeader(HttpHeaderConfigurations.X_MS_PERMISSIONS, permission));
404+
if (permissions.getPermission() != null && !permissions.getPermission().isEmpty()) {
405+
requestHeaders.add(
406+
new AbfsHttpHeader(HttpHeaderConfigurations.X_MS_PERMISSIONS,
407+
permissions.getPermission()));
407408
}
408409

409-
if (umask != null && !umask.isEmpty()) {
410-
requestHeaders.add(new AbfsHttpHeader(HttpHeaderConfigurations.X_MS_UMASK, umask));
410+
if (permissions.getUmask() != null && !permissions.getUmask().isEmpty()) {
411+
requestHeaders.add(new AbfsHttpHeader(HttpHeaderConfigurations.X_MS_UMASK,
412+
permissions.getUmask()));
411413
}
412414

413415
if (eTag != null && !eTag.isEmpty()) {

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.EnumSet;
2626
import java.util.UUID;
2727

28+
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.Permissions;
2829
import org.junit.Test;
2930

3031
import org.apache.hadoop.conf.Configuration;
@@ -398,9 +399,8 @@ public void testNegativeScenariosForCreateOverwriteDisabled()
398399
serverErrorResponseEx) // Scn5: create overwrite=false fails with Http500
399400
.when(mockClient)
400401
.createPath(any(String.class), eq(true), eq(false),
401-
isNamespaceEnabled ? any(String.class) : eq(null),
402-
isNamespaceEnabled ? any(String.class) : eq(null),
403-
any(boolean.class), eq(null), any(), any(TracingContext.class));
402+
any(Permissions.class), any(boolean.class), eq(null), any(),
403+
any(TracingContext.class));
404404

405405
doThrow(fileNotFoundResponseEx) // Scn1: GFS fails with Http404
406406
.doThrow(serverErrorResponseEx) // Scn2: GFS fails with Http500
@@ -416,9 +416,8 @@ public void testNegativeScenariosForCreateOverwriteDisabled()
416416
serverErrorResponseEx) // Scn4: create overwrite=true fails with Http500
417417
.when(mockClient)
418418
.createPath(any(String.class), eq(true), eq(true),
419-
isNamespaceEnabled ? any(String.class) : eq(null),
420-
isNamespaceEnabled ? any(String.class) : eq(null),
421-
any(boolean.class), eq(null), any(), any(TracingContext.class));
419+
any(Permissions.class), any(boolean.class), eq(null), any(),
420+
any(TracingContext.class));
422421

423422
// Scn1: GFS fails with Http404
424423
// Sequence of events expected:

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomEncryption.java‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,9 @@
4747
import org.apache.hadoop.test.LambdaTestUtils;
4848
import org.apache.hadoop.util.Lists;
4949

50-
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_CLIENT_PROVIDED_KEY;
5150
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_CONTEXT_PROVIDER_TYPE;
51+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY;
52+
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY_SHA;
5253
import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_ENCRYPTION_KEY_SHA256;
5354
import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_REQUEST_SERVER_ENCRYPTED;
5455
import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_SERVER_ENCRYPTED;
@@ -252,15 +253,22 @@ private AzureBlobFileSystem getECProviderEnabledFS() throws Exception {
252253
Configuration configuration = getRawConfiguration();
253254
configuration.set(FS_AZURE_ENCRYPTION_CONTEXT_PROVIDER_TYPE,
254255
MockEncryptionContextProvider.class.getCanonicalName());
255-
configuration.unset(
256-
FS_AZURE_ENCRYPTION_CLIENT_PROVIDED_KEY + "." + getAccountName());
256+
configuration.unset(FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY + "."
257+
+ getAccountName());
258+
configuration.unset(FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY_SHA + "."
259+
+ getAccountName());
257260
return (AzureBlobFileSystem) FileSystem.newInstance(configuration);
258261
}
259262

260263
private AzureBlobFileSystem getCPKenabledFS() throws IOException {
261264
Configuration conf = getRawConfiguration();
262-
conf.set(FS_AZURE_ENCRYPTION_CLIENT_PROVIDED_KEY + "." + getAccountName(),
263-
cpk);
265+
String cpkEncoded = EncryptionAdapter.getBase64EncodedString(cpk);
266+
String cpkEncodedSHA = EncryptionAdapter.getBase64EncodedString(
267+
EncryptionAdapter.getSHA256Hash(cpk));
268+
conf.set(FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY + "."
269+
+ getAccountName(), cpkEncoded);
270+
conf.set(FS_AZURE_ENCRYPTION_ENCODED_CLIENT_PROVIDED_KEY_SHA + "."
271+
+ getAccountName(), cpkEncodedSHA);
264272
conf.unset(FS_AZURE_ENCRYPTION_CONTEXT_PROVIDER_TYPE);
265273
return (AzureBlobFileSystem) FileSystem.newInstance(conf);
266274
}

0 commit comments

Comments
 (0)