Skip to content

Commit

Permalink
Merge branch 'release' into feat/37759-unified-response-view
Browse files Browse the repository at this point in the history
  • Loading branch information
@alex-golovanov
alex-golovanov committed Dec 4, 2024
2 parents cdc41fe + 1f07be3 commit c9bcbc0
Show file tree
Hide file tree
Showing 36 changed files with 1,639 additions and 196 deletions.
219 changes: 216 additions & 3 deletions app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsCEImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
import com.appsmith.external.git.FileInterface;
import com.appsmith.external.git.GitExecutor;
import com.appsmith.external.git.constants.GitSpan;
import com.appsmith.external.git.models.GitResourceIdentity;
import com.appsmith.external.git.models.GitResourceMap;
import com.appsmith.external.git.models.GitResourceType;
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.external.helpers.Stopwatch;
Expand All @@ -14,10 +17,12 @@
import com.appsmith.git.configurations.GitServiceConfig;
import com.appsmith.git.constants.CommonConstants;
import com.appsmith.git.helpers.DSLTransformerHelper;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.micrometer.tracing.Span;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.json.JSONObject;
Expand All @@ -28,6 +33,8 @@
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;
import reactor.util.function.Tuple2;
import reactor.util.function.Tuples;

import java.io.BufferedWriter;
import java.io.File;
Expand All @@ -48,7 +55,10 @@
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static com.appsmith.external.git.constants.GitConstants.ACTION_COLLECTION_LIST;
import static com.appsmith.external.git.constants.GitConstants.ACTION_LIST;
Expand All @@ -74,6 +84,7 @@ public class FileUtilsCEImpl implements FileInterface {
private final GitExecutor gitExecutor;
protected final FileOperations fileOperations;
private final ObservationHelper observationHelper;
protected final ObjectMapper objectMapper;

private static final String EDIT_MODE_URL_TEMPLATE = "{{editModeUrl}}";

Expand All @@ -90,11 +101,23 @@ public FileUtilsCEImpl(
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
FileOperations fileOperations,
ObservationHelper observationHelper) {
ObservationHelper observationHelper,
ObjectMapper objectMapper) {
this.gitServiceConfig = gitServiceConfig;
this.gitExecutor = gitExecutor;
this.fileOperations = fileOperations;
this.observationHelper = observationHelper;
this.objectMapper = objectMapper;
}

protected Map<GitResourceType, GitResourceType> getModifiedResourcesTypes() {
return Map.of(
GitResourceType.JSLIB_CONFIG, GitResourceType.JSLIB_CONFIG,
GitResourceType.CONTEXT_CONFIG, GitResourceType.CONTEXT_CONFIG,
GitResourceType.QUERY_CONFIG, GitResourceType.QUERY_CONFIG,
GitResourceType.QUERY_DATA, GitResourceType.QUERY_CONFIG,
GitResourceType.JSOBJECT_CONFIG, GitResourceType.JSOBJECT_CONFIG,
GitResourceType.JSOBJECT_DATA, GitResourceType.JSOBJECT_CONFIG);
}

/**
Expand Down Expand Up @@ -215,6 +238,101 @@ public Mono<Path> saveApplicationToGitRepo(
.subscribeOn(scheduler);
}

@Override
public Mono<Path> saveArtifactToGitRepo(Path baseRepoSuffix, GitResourceMap gitResourceMap, String branchName)
throws GitAPIException, IOException {

// Repo path will be:
// baseRepo : root/orgId/defaultAppId/repoName/{applicationData}
// Checkout to mentioned branch if not already checked-out
return gitExecutor
.resetToLastCommit(baseRepoSuffix, branchName)
.flatMap(isSwitched -> {
Path baseRepo = Paths.get(gitServiceConfig.getGitRootPath()).resolve(baseRepoSuffix);

try {
updateEntitiesInRepo(gitResourceMap, baseRepo);
} catch (IOException e) {
return Mono.error(e);
}

return Mono.just(baseRepo);
})
.subscribeOn(scheduler);
}

protected Set<String> getExistingFilesInRepo(Path baseRepo) throws IOException {
try (Stream<Path> stream = Files.walk(baseRepo).parallel()) {
return stream.filter(path -> {
try {
return Files.isRegularFile(path) || FileUtils.isEmptyDirectory(path.toFile());
} catch (IOException e) {
log.error("Unable to find file details. Please check the file at file path: {}", path);
log.error("Assuming that it does not exist for now ...");
return false;
}
})
.map(baseRepo::relativize)
.map(Path::toString)
.collect(Collectors.toSet());
}
}

protected Set<String> updateEntitiesInRepo(GitResourceMap gitResourceMap, Path baseRepo) throws IOException {
ModifiedResources modifiedResources = gitResourceMap.getModifiedResources();
Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();

Set<String> filesInRepo = getExistingFilesInRepo(baseRepo);

Set<String> updatedFilesToBeSerialized = resourceMap.keySet().parallelStream()
.map(gitResourceIdentity -> gitResourceIdentity.getFilePath())
.collect(Collectors.toSet());

// Remove all files that need to be serialized from the existing files list, as well as the README file
// What we are left with are all the files to be deleted
filesInRepo.removeAll(updatedFilesToBeSerialized);
filesInRepo.remove("README.md");

// Delete all the files because they are no longer needed
// This covers both older structures of storing files and,
// legitimate changes in the artifact that might cause deletions
filesInRepo.stream().parallel().forEach(filePath -> {
try {
Files.deleteIfExists(baseRepo.resolve(filePath));
} catch (IOException e) {
// We ignore files that could not be deleted and expect to come back to this at a later point
// Just log the path for now
log.error("Unable to delete file at path: {}", filePath);
}
});

// Now go through the resource map and based on resource type, check if the resource is modified before
// serialization
// Or simply choose the mechanism for serialization
Map<GitResourceType, GitResourceType> modifiedResourcesTypes = getModifiedResourcesTypes();
return resourceMap.entrySet().parallelStream()
.map(entry -> {
GitResourceIdentity key = entry.getKey();
boolean resourceUpdated = true;
if (modifiedResourcesTypes.containsKey(key.getResourceType()) && modifiedResources != null) {
GitResourceType comparisonType = modifiedResourcesTypes.get(key.getResourceType());

resourceUpdated =
modifiedResources.isResourceUpdatedNew(comparisonType, key.getResourceIdentifier());
}

if (resourceUpdated) {
String filePath = key.getFilePath();
saveResourceCommon(entry.getValue(), baseRepo.resolve(filePath));

return filePath;
}
return null;
})
.filter(Objects::nonNull)
.collect(Collectors.toSet());
}

protected Set<String> updateEntitiesInRepo(ApplicationGitReference applicationGitReference, Path baseRepo) {

Set<String> validDatasourceFileNames = new HashSet<>();
Expand Down Expand Up @@ -434,6 +552,23 @@ protected boolean saveResource(Object sourceEntity, Path path) {
return false;
}

protected void saveResourceCommon(Object sourceEntity, Path path) {
try {
Files.createDirectories(path.getParent());
if (sourceEntity instanceof String s) {
writeStringToFile(s, path);
return;
}
if (sourceEntity instanceof JSONObject) {
sourceEntity = objectMapper.readTree(sourceEntity.toString());
}
fileOperations.writeToFile(sourceEntity, path);
} catch (IOException e) {
log.error("Error while writing resource to file {} with {}", path, e.getMessage());
log.debug(e.getMessage());
}
}

/**
* This method is used to write actionCollection specific resource to file system. We write the data in two steps
* 1. Actual js code
Expand Down Expand Up @@ -514,9 +649,9 @@ private void writeStringToFile(String sourceEntity, Path path) throws IOExceptio
/**
* This will reconstruct the application from the repo
*
* @param organisationId To which organisation application needs to be rehydrated
* @param organisationId To which organisation application needs to be rehydrated
* @param baseApplicationId To which organisation application needs to be rehydrated
* @param branchName for which the application needs to be rehydrate
* @param branchName for which the application needs to be rehydrate
* @return application reference from which entire application can be rehydrated
*/
public Mono<ApplicationGitReference> reconstructApplicationReferenceFromGitRepo(
Expand Down Expand Up @@ -672,6 +807,84 @@ private Object readPageMetadata(Path directoryPath) {
directoryPath.resolve(directoryPath.toFile().getName() + CommonConstants.JSON_EXTENSION));
}

protected GitResourceMap fetchGitResourceMap(Path baseRepoPath) throws IOException {
// Extract application metadata from the json
Object metadata = fileOperations.readFile(
baseRepoPath.resolve(CommonConstants.METADATA + CommonConstants.JSON_EXTENSION));
Integer fileFormatVersion = fileOperations.getFileFormatVersion(metadata);
// Check if fileFormat of the saved files in repo is compatible
if (!isFileFormatCompatible(fileFormatVersion)) {
throw new AppsmithPluginException(AppsmithPluginError.INCOMPATIBLE_FILE_FORMAT);
}

GitResourceMap gitResourceMap = new GitResourceMap();
Map<GitResourceIdentity, Object> resourceMap = gitResourceMap.getGitResourceMap();

Set<String> filesInRepo = getExistingFilesInRepo(baseRepoPath);

filesInRepo.parallelStream()
.filter(path -> !Files.isDirectory(baseRepoPath.resolve(path)))
.forEach(filePath -> {
Tuple2<GitResourceIdentity, Object> identity = getGitResourceIdentity(baseRepoPath, filePath);

resourceMap.put(identity.getT1(), identity.getT2());
});

return gitResourceMap;
}

protected Tuple2<GitResourceIdentity, Object> getGitResourceIdentity(Path baseRepoPath, String filePath) {
Path path = baseRepoPath.resolve(filePath);
GitResourceIdentity identity;
Object contents = fileOperations.readFile(path);
if (!filePath.contains("/")) {
identity = new GitResourceIdentity(GitResourceType.ROOT_CONFIG, filePath, filePath);
} else if (filePath.matches(DATASOURCE_DIRECTORY + "/.*")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.DATASOURCE_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(JS_LIB_DIRECTORY + "/.*")) {
String fileName = FilenameUtils.getBaseName(filePath);
identity = new GitResourceIdentity(GitResourceType.JSLIB_CONFIG, fileName, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/[^/]*.json]")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.CONTEXT_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*/metadata.json")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.QUERY_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_DIRECTORY + "/.*\\.txt")) {
Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json"));
String gitSyncId =
objectMapper.valueToTree(configContents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.QUERY_DATA, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*/metadata.json")) {
String gitSyncId =
objectMapper.valueToTree(contents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.JSOBJECT_CONFIG, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/" + ACTION_COLLECTION_DIRECTORY + "/.*\\.js")) {
Object configContents = fileOperations.readFile(path.getParent().resolve("metadata.json"));
String gitSyncId =
objectMapper.valueToTree(configContents).get("gitSyncId").asText();
identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId, filePath);
} else if (filePath.matches(PAGE_DIRECTORY + "/[^/]*/widgets/.*\\.json")) {
Pattern pageDirPattern = Pattern.compile("(" + PAGE_DIRECTORY + "/([^/]*))/widgets/.*\\.json");
Matcher matcher = pageDirPattern.matcher(filePath);
matcher.find();
String pageDirectory = matcher.group(1);
String pageName = matcher.group(2) + ".json";
Object configContents =
fileOperations.readFile(baseRepoPath.resolve(pageDirectory).resolve(pageName));
String gitSyncId =
objectMapper.valueToTree(configContents).get("gitSyncId").asText();
String widgetId = objectMapper.valueToTree(contents).get("widgetId").asText();
identity = new GitResourceIdentity(GitResourceType.JSOBJECT_DATA, gitSyncId + "-" + widgetId, filePath);
} else return null;

return Tuples.of(identity, contents);
}

private ApplicationGitReference fetchApplicationReference(Path baseRepoPath) {
ApplicationGitReference applicationGitReference = new ApplicationGitReference();
// Extract application metadata from the json
Expand Down
6 changes: 4 additions & 2 deletions app/server/appsmith-git/src/main/java/com/appsmith/git/files/FileUtilsImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.appsmith.external.git.operations.FileOperations;
import com.appsmith.external.helpers.ObservationHelper;
import com.appsmith.git.configurations.GitServiceConfig;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Import;
Expand All @@ -22,7 +23,8 @@ public FileUtilsImpl(
GitServiceConfig gitServiceConfig,
GitExecutor gitExecutor,
FileOperations fileOperations,
ObservationHelper observationHelper) {
super(gitServiceConfig, gitExecutor, fileOperations, observationHelper);
ObservationHelper observationHelper,
ObjectMapper objectMapper) {
super(gitServiceConfig, gitExecutor, fileOperations, observationHelper, objectMapper);
}
}
Loading

0 comments on commit c9bcbc0

Please sign in to comment.