From 345dc503e2f200c85bc453bf1fb717400eac99a8 Mon Sep 17 00:00:00 2001 From: Piotr Czarnas Date: Tue, 27 Feb 2024 11:45:09 +0100 Subject: [PATCH] Synchronization of files with a windows backslash. --- .../com/dqops/core/filesystem/metadata/FileMetadata.java | 7 +++++++ .../com/dqops/core/filesystem/metadata/FolderMetadata.java | 7 +++++++ .../dqops/core/filesystem/virtual/FileNameSanitizer.java | 1 + .../GSRemoteFileSystemSynchronizationOperationsImpl.java | 2 +- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dqops/src/main/java/com/dqops/core/filesystem/metadata/FileMetadata.java b/dqops/src/main/java/com/dqops/core/filesystem/metadata/FileMetadata.java index 3af8454810..a8d7c649df 100644 --- a/dqops/src/main/java/com/dqops/core/filesystem/metadata/FileMetadata.java +++ b/dqops/src/main/java/com/dqops/core/filesystem/metadata/FileMetadata.java @@ -236,4 +236,11 @@ public boolean equals(Object o) { public int hashCode() { return relativePath.hashCode(); } + + @Override + public String toString() { + return "FileMetadata{" + + "relativePath=" + relativePath + + '}'; + } } diff --git a/dqops/src/main/java/com/dqops/core/filesystem/metadata/FolderMetadata.java b/dqops/src/main/java/com/dqops/core/filesystem/metadata/FolderMetadata.java index 5ed4cfa1c0..77b90921c0 100644 --- a/dqops/src/main/java/com/dqops/core/filesystem/metadata/FolderMetadata.java +++ b/dqops/src/main/java/com/dqops/core/filesystem/metadata/FolderMetadata.java @@ -704,4 +704,11 @@ public void truncateToLicenseLimits(DqoCloudApiKeyPayload apiKeyPayload) { } } } + + @Override + public String toString() { + return "FolderMetadata{" + + "relativePath=" + relativePath + + '}'; + } } diff --git a/dqops/src/main/java/com/dqops/core/filesystem/virtual/FileNameSanitizer.java b/dqops/src/main/java/com/dqops/core/filesystem/virtual/FileNameSanitizer.java index 3c55041748..0e97485400 100644 --- a/dqops/src/main/java/com/dqops/core/filesystem/virtual/FileNameSanitizer.java +++ b/dqops/src/main/java/com/dqops/core/filesystem/virtual/FileNameSanitizer.java @@ -186,6 +186,7 @@ public static Path convertEncodedPathToFullyUrlEncodedPath(Path path) { for (int i = 0; i < pathElements.length; i++) { String rawName = decodeFileSystemName(pathElements[i]); urlEncodedPathElements[i] = URLEncoder.encode(rawName, StandardCharsets.UTF_8) + .replace("%", "%25") .replace("+", "%20"); // fixing spaces, because GCP storage bucket does not use '+' for a space } diff --git a/dqops/src/main/java/com/dqops/core/synchronization/filesystems/gcp/GSRemoteFileSystemSynchronizationOperationsImpl.java b/dqops/src/main/java/com/dqops/core/synchronization/filesystems/gcp/GSRemoteFileSystemSynchronizationOperationsImpl.java index 823e5c40b7..30e2832448 100644 --- a/dqops/src/main/java/com/dqops/core/synchronization/filesystems/gcp/GSRemoteFileSystemSynchronizationOperationsImpl.java +++ b/dqops/src/main/java/com/dqops/core/synchronization/filesystems/gcp/GSRemoteFileSystemSynchronizationOperationsImpl.java @@ -248,7 +248,7 @@ public FolderMetadata listFilesInFolder(FileSystemSynchronizationRoot fileSystem for (Blob blob : blogPage.iterateAll()) { String blobFileName = blob.getName(); - Path fullBlobFilePathInsideBucket = FileNameSanitizer.convertRawPathToEncodedPath(blobFileName); + Path fullBlobFilePathInsideBucket = Path.of(blobFileName.replace("%25", "%")); Map metadata = blob.getMetadata(); if (metadata != null && Objects.equals(metadata.get("DQOFileType"), "empty-parquet")) { // ignoring because it is a special empty file to ensure that the schema of an external table could be detected from a parquet file