[#320] migration updates

extensions/extensions-helm/aissemble-jenkins-chart/values.yaml

@@ -0,0 +1,10 @@
+# Default values for aissemble-jenkins.
+jenkins:
+  controller:
+    image: ghcr.io/boozallen/aissemble-jenkins-controller
+    # note the official chart does not use the controller tag value for testing, so disabling here
+    testEnabled: false
+    tag: "1.8.1-SNAPSHOT"
+  agent:
+    image: ghcr.io/boozallen/aissemble-jenkins-agent
+    tag: "1.8.1-SNAPSHOT"

foundation/foundation-mda/src/main/resources/templates/general-docker/it.java.docker.file.vm

@@ -4,7 +4,7 @@
 #
 # Generated from: ${templateName}
 ARG DOCKER_BASELINE_REPO_ID
-FROM ${DOCKER_BASELINE_REPO_ID}boozallen/openjdk:11-slim
+FROM openjdk:11-slim
 
 COPY ./target/specifications/ ./specifications
 COPY ./target/${artifactId}.jar ./

foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/AbstractPomMigration.java

@@ -0,0 +1,45 @@
+package com.boozallen.aissemble.upgrade.migration;
+
+/*-
+ * #%L
+ * aiSSEMBLE::Foundation::Upgrade
+ * %%
+ * Copyright (C) 2021 Booz Allen
+ * %%
+ * This software package is licensed under the Booz Allen Public License. All Rights Reserved.
+ * #L%
+ */
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.technologybrewery.baton.BatonException;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.stream.Stream;
+
+public abstract class AbstractPomMigration extends AbstractAissembleMigration {
+    protected static final Logger logger = LoggerFactory.getLogger(AbstractPomMigration.class);
+
+    public static final String POM = "pom";
+    public static final String GUARANTEED_TAG = "<modelVersion>";
+
+    protected String indent;
+
+    protected void detectAndSetIndent(File file) {
+        try (Stream<String> lines = Files.lines(file.toPath())) {
+            indent = lines.filter(line -> line.contains(GUARANTEED_TAG))
+                    .findFirst()
+                    .map(artifact -> artifact.substring(0, artifact.indexOf(GUARANTEED_TAG)))
+                    .orElse(null);
+            if (StringUtils.isEmpty(indent)) {
+                logger.info("Failed to detect indent for POM. Using default. {}", file);
+                indent = "    ";
+            }
+        } catch (IOException e) {
+            throw new BatonException("Failed to get indent from POM:" + file, e);
+        }
+    }
+}

foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/util/AissembleFileUtils.java

@@ -0,0 +1,39 @@
+package com.boozallen.aissemble.upgrade.util;
+
+/*-
+ * #%L
+ * aiSSEMBLE::Foundation::Upgrade
+ * %%
+ * Copyright (C) 2021 Booz Allen
+ * %%
+ * This software package is licensed under the Booz Allen Public License. All Rights Reserved.
+ * #L%
+ */
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class AissembleFileUtils {
+
+    /**
+     * Copy lines between startLine and endLine, inclusive, from the file.
+     *
+     * @param file the file to copy lines from
+     * @param startLine the index of the first line to copy
+     * @param endLine the index of the last line to copy
+     * @return a list of the copied lines
+     */
+    public static List<String> getLines(Path file, int startLine, int endLine) {
+        try (Stream<String> lines = Files.lines(file)) {
+            return lines.skip(startLine)
+                    .limit(endLine - startLine + 1)
+                    .collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/AbstractMigrationTest.java

@@ -10,31 +10,48 @@
  * #L%
  */
 
-import org.apache.commons.io.FileUtils;
-import org.technologybrewery.baton.BatonException;
-
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public abstract class AbstractMigrationTest {
     protected File testFile;
-
     protected boolean shouldExecute;
     protected boolean successful;
 
-    private static final String TEST_FILES_FOLDER = Paths.get("target", "test-classes", "test-files").toString();
+    private static final Path TEST_FILES_FOLDER = Paths.get("target", "test-classes", "test-files");
+    private static final String MIGRATION_DIR = File.separator + "migration" + File.separator;
+    private static final String VALIDATION_DIR = File.separator + "validation" + File.separator;
 
-    protected static boolean validateMigration(File original, File migrated) {
+    protected static void assertLinesMatch(String message, File expectedFile, File actualFile) {
         try {
-            return FileUtils.contentEquals(original, migrated);
-        } catch (Exception e) {
-            return false;
+            List<String> actualLines = Files.readAllLines(actualFile.toPath());
+            List<String> expectedLines = Files.readAllLines(expectedFile.toPath());
+            for (int i = 0; i < actualLines.size() || i < expectedLines.size(); i++) {
+                String actualLine = null;
+                String expectedLine = null;
+                if (i < actualLines.size()) {
+                    actualLine = actualLines.get(i);
+                }
+                if (i < expectedLines.size()) {
+                    expectedLine = expectedLines.get(i);
+                }
+                assertEquals(message + ": Ln " + (i+1), expectedLine, actualLine);
+            }
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to compare test and validation file contents", e);
         }
     }
 
     protected static void addTestFile(String subPath) throws IOException {
-        File testFile = Paths.get(TEST_FILES_FOLDER, subPath).toFile();
+        File testFile = TEST_FILES_FOLDER.resolve(subPath).toFile();
         if(testFile.exists()) {
             throw new RuntimeException(String.format("Test file at %s already exists", subPath));
         }
@@ -43,7 +60,10 @@ protected static void addTestFile(String subPath) throws IOException {
     }
 
     protected static File getTestFile(String subPath) {
-        File testFile = Paths.get("target", "test-classes", "test-files", subPath).toFile();
+        if (subPath.startsWith(File.separator)) {
+            subPath = subPath.substring(1);
+        }
+        File testFile = TEST_FILES_FOLDER.resolve(subPath).toFile();
         File dir = testFile.getParentFile();
         if (!dir.mkdirs() && !dir.isDirectory()) {
             throw new RuntimeException("Parent directory of test file is already a regular file: " + dir);
@@ -55,4 +75,32 @@ protected void performMigration(AbstractAissembleMigration migration) {
         shouldExecute = migration.shouldExecuteOnFile(testFile);
         successful = shouldExecute && migration.performMigration(testFile);
     }
+
+    protected void assertMigrationSuccess() {
+        assertTrue("Migration was incorrectly skipped", shouldExecute);
+        assertTrue("Migration did not execute successfully", successful);
+    }
+
+    protected void assertMigrationSkipped() {
+        assertFalse("Migration was not skipped as expected", shouldExecute);
+    }
+
+    /**
+     * Asserts that the test file matches the expected validation file. Calculates the validation file path based on the
+     * current test file path by replacing the migration directory with the validation directory. Throws an exception if
+     * this calculation fails or if the validation file does not exist.
+     *
+     * @param message the message to display if the assertion fails
+     */
+    protected void assertTestFileMatchesExpectedFile(String message) {
+        String validationFilePath = testFile.getPath().replace(MIGRATION_DIR, VALIDATION_DIR);
+        File validationFile = new File(validationFilePath);
+        if( testFile.equals(validationFile)) {
+            throw new RuntimeException(String.format("Test/validation files not using expected structure: %s", testFile));
+        }
+        if(!validationFile.exists()) {
+            throw new RuntimeException(String.format("Validation file does not exist: %s", validationFilePath));
+        }
+        assertLinesMatch(message, validationFile, testFile);
+    }
 }

foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/utils/AissembleFileUtilSteps.java

@@ -0,0 +1,46 @@
+package com.boozallen.aissemble.upgrade.migration.utils;
+
+/*-
+ * #%L
+ * aiSSEMBLE::Foundation::Upgrade
+ * %%
+ * Copyright (C) 2021 Booz Allen
+ * %%
+ * This software package is licensed under the Booz Allen Public License. All Rights Reserved.
+ * #L%
+ */
+
+import com.boozallen.aissemble.upgrade.util.AissembleFileUtils;
+import io.cucumber.java.en.Given;
+import io.cucumber.java.en.Then;
+import io.cucumber.java.en.When;
+import org.junit.Assert;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class AissembleFileUtilSteps {
+
+    private Path testFile = Path.of("target/test.txt");
+    private List<String> fetchedLines;
+
+    @Given("a file with the contents:")
+    public void aFileWithTheContents(String contents) throws IOException {
+        Files.write(testFile, contents.getBytes(), StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
+    }
+
+    @When("I retrieve the lines {int} to {int} of the file")
+    public void iRetrieveTheLinesStartToEndOfTheFile(int start, int end) {
+        fetchedLines = AissembleFileUtils.getLines(testFile, start, end);
+    }
+
+    @Then("the lines should be {string}")
+    public void theLinesShouldBe(String expected) {
+        List<String> expectedLines = expected.lines().collect(Collectors.toList());
+        Assert.assertEquals("getLines fetched incorrect content", expectedLines, fetchedLines);
+    }
+}

foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_9_0/AirflowDockerfileMigrationSteps.java

@@ -49,7 +49,7 @@ public void theAirflowDockerImageMigrationExecutes() {
     @Then("the Dockerfile will pull the community docker Airflow image")
     public void theDockerfileWillPullTheCommunityDockerAirflowImage() {
         validatedFile = getTestFile("/v1_9_0/AirflowDockerfileMigration/validation/Dockerfile");
-        assertTrue("Dockerfile is still referencing aissemble-airflow instead of community airflow Docker image.", validateMigration(testFile, validatedFile));
+        assertLinesMatch("Dockerfile is still referencing aissemble-airflow instead of community airflow Docker image.", validatedFile, testFile);
     }
 
     @Then("the airflow migration is skipped")

foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_9_0/ArgocdTemplateSyncPolicyMigrationStep.java

@@ -57,13 +57,13 @@ public void theArgoCDTemplateSyncPolicyMigrationExecutes() {
     @Then("the syncPolicy template function is included in the template")
     public void theSyncPolicyTemplateFunctionIsIncludedInTheTemplate() {
         validatedFile = getTestFile("/v1_9_0/ArgocdTemplateSyncPolicyMigration/validation/" + testFile.getName());
-        assertTrue("The syncPolicy helm function is included in the template.", validateMigration(testFile, validatedFile));
+        assertLinesMatch("The syncPolicy helm function is included in the template.", validatedFile, testFile);
     }
 
     @Then("the template is unchanged")
     public void theTemplateIsUnchanged() {
         validatedFile = getTestFile("/v1_9_0/ArgocdTemplateSyncPolicyMigration/validation/" + testFile.getName());
-        assertTrue("The ArgoCD template is not impacted by migration!", validateMigration(testFile, validatedFile));
+        assertLinesMatch("The ArgoCD template is not impacted by migration!", validatedFile, testFile);
     }
 
     @Then("the application template still has indent with 4 spaces")

foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_9_0/ArgocdValueFileSyncPolicyMigrationStep.java

@@ -44,7 +44,7 @@ public void theArgoCDTemplateSyncPolicyMigrationExecutes() {
     @Then("the value file is updated with expected syncPolicy value function")
     public void theSyncPolicyTemplateFunctionIsIncludedInTheTemplate() {
         File validatedFile = getTestFile(String.format("v1_9_0/ArgocdValueFileSyncPolicyMigration/validation/%s-values.yaml", configuration));
-        assertTrue("the value file is updated with expected syncPolicy value function.", validateMigration(testFile, validatedFile));
+        assertLinesMatch("the value file is updated with expected syncPolicy value function.", validatedFile, testFile);
     }
 
     @Given("an ArgoCD value file has indent with 4 spaces")

foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_9_0/MLFlowDockerfileMigrationSteps.java

@@ -46,12 +46,12 @@ public void theMLFlowDockerImageMigrationExecutes() {
     @Then("the image will pull the community docker image")
     public void theImageWillPullTheCommunityDockerImage() {
         validatedFile = getTestFile("/v1_9_0/MLFlowDockerfileMigration/validation/applicable-Dockerfile");
-        assertTrue("Dockerfile is still referencing aissemble-mlflow instead of community mlflow Docker image.", validateMigration(testFile, validatedFile));
+        assertLinesMatch("Dockerfile is still referencing aissemble-mlflow instead of community mlflow Docker image.", validatedFile, testFile);
     }
 
     @Then("the image is unchanged")
     public void the_image_is_unchanged() {
         validatedFile = getTestFile("/v1_9_0/MLFlowDockerfileMigration/validation/inapplicable-Dockerfile");
-        assertTrue("The migration is processing Dockerfile it should NOT be migrating!", validateMigration(testFile, validatedFile));
+        assertLinesMatch("The migration is processing Dockerfile it should NOT be migrating!", validatedFile, testFile);
     }
 }

foundation/foundation-upgrade/src/test/resources/specifications/file-util.feature

@@ -0,0 +1,20 @@
+Feature: Common File Operations
+
+  Scenario Outline: Retrieve lines
+    Given a file with the contents:
+      """
+      first line
+      second line
+      third line
+      fourth line
+      """
+    When I retrieve the lines <start> to <end> of the file
+    Then the lines should be "<lines>"
+
+    Examples:
+      | start | end | lines                                            |
+      | 0     | 2   | first line\nsecond line\nthird line              |
+      | 1     | 2   | second line\nthird line                          |
+      | 2     | 3   | third line\nfourth line                          |
+      | 0     | 99  | first line\nsecond line\nthird line\nfourth line |
+