feat!: Integrate ESDK-Java with AWS Cryptographic Material Providers Library (MPL) for Keyring and CMM Support.

.github/workflows/ci.yml

@@ -58,6 +58,37 @@ jobs:
           env-vars-for-codebuild: JAVA_ENV_VERSION
         env:
           JAVA_ENV_VERSION: ${{ matrix.platform.distribution }}${{ matrix.version }}
+  generateTestVectors:
+    name: Generate Vectors
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1
+      fail-fast: true
+      matrix:
+        platform:
+          - distribution: openjdk
+            image: "aws/codebuild/standard:3.0"
+          - distribution: corretto
+            image: "aws/codebuild/amazonlinux2-x86_64-standard:3.0" # Corretto only runs on AL2
+        version: [ 8, 11 ]
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
+          aws-region: us-west-2
+          role-duration-seconds: 3600
+      - name: Generate Test Vectors
+        uses: aws-actions/aws-codebuild-run-build@v1
+        timeout-minutes: 60
+        with:
+          project-name: AWS-ESDK-Java-CI
+          buildspec-override: codebuild/ci/vectors-generator.yml
+          compute-type-override: BUILD_GENERAL1_LARGE
+          image-override: ${{ matrix.platform.image }}
+          env-vars-for-codebuild: JAVA_ENV_VERSION
+        env:
+          JAVA_ENV_VERSION: ${{ matrix.platform.distribution }}${{ matrix.version }}
   releaseCI:
     name: Release CI
     runs-on: ubuntu-latest

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "aws-encryption-sdk-specification"]
 	path = aws-encryption-sdk-specification
 	url = https://github.com/awslabs/aws-encryption-sdk-specification.git
+[submodule "submodules/MaterialProviders"]
+	path = submodules/MaterialProviders
+	url = https://github.com/aws/aws-cryptographic-material-providers-library-java.git

codebuild/ci/release-ci.yml

@@ -17,13 +17,34 @@ phases:
   install:
     runtime-versions:
       java: openjdk11
+    commands:
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
   pre_build:
     commands:
       - export SETTINGS_FILE=$(pwd)/codebuild/release/settings.xml
       - export CODEARTIFACT_TOKEN=$(aws codeartifact get-authorization-token --domain $DOMAIN --domain-owner $ACCOUNT --query authorizationToken --output text --region ${REGION})
       - export CODEARTIFACT_REPO_URL=https://${DOMAIN}-${ACCOUNT}.d.codeartifact.${REGION}.amazonaws.com/maven/${REPOSITORY}
       - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-CI --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz
       - tar -xvf ~/mvn_gpg.tgz -C ~
+
+      # Build and deploy to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
+
   build:
     commands:
       - VERSION_HASH="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)-$CODEBUILD_RESOLVED_SOURCE_VERSION-$GITHUB_EVENT_NAME"

codebuild/ci/vectors-ci.yml

@@ -4,6 +4,35 @@ phases:
   install:
     runtime-versions:
       java: $JAVA_ENV_VERSION
+    commands:
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
+  pre_build:
+    commands:
+      # Assume Role to access non-prod resources
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+
+      # Build and deploy TestVectors to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
   build:
     commands:
       - mvn install -T 8 -Dgpg.skip=true -ntp "-DtestVectorZip=file://$CODEBUILD_SRC_DIR/src/test/resources/aws-encryption-sdk-test-vectors/vectors/awses-decrypt/python-2.3.0.zip"

codebuild/ci/vectors-generator.yml

@@ -0,0 +1,53 @@
+version: 0.2
+
+phases:
+  install:
+    runtime-versions:
+      java: $JAVA_ENV_VERSION
+    commands:
+      - n 16
+      # Install the Javascript ESDK run test vectors
+      - npm install -g @aws-crypto/integration-node
+
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
+  pre_build:
+    commands:
+      # Assume Role to access non-prod resources
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+
+      # Build and deploy to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
+  build:
+    commands:
+      - export VECTORS_ZIP="$CODEBUILD_SRC_DIR/generated_vectors.zip"
+      # Generate test vectors by encrypting with Keyrings
+      # Ignore Testing coverage requirement by skipping jacoco
+      - mvn -B -ntp install -Dgpg.skip=true -Djacoco.skip=true "-Dtest=TestVectorGenerator" "-DzipFilePath=$VECTORS_ZIP" "-DkeysManifest=$CODEBUILD_SRC_DIR/src/test/resources/keys.json"
+      # Decrypt generated vectors with Javascript ESDK
+      - integration-node decrypt -v $VECTORS_ZIP
+
+      - rm $VECTORS_ZIP
+      # Generate test vectors by encrypting with MasterKeys
+      - mvn -B -ntp install -Dgpg.skip=true -Djacoco.skip=true -Dmasterkey=true "-Dtest=TestVectorGenerator" "-DzipFilePath=$VECTORS_ZIP" "-DkeysManifest=$CODEBUILD_SRC_DIR/src/test/resources/keys.json"
+      # Decrypt generated vectors with Javascript ESDK
+      - integration-node decrypt -v $VECTORS_ZIP

codebuild/release/release-prod.yml

@@ -16,12 +16,31 @@ phases:
   install:
     runtime-versions:
       java: corretto11
+    commands:
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
   pre_build:
     commands:
       - git checkout $BRANCH
       - export SETTINGS_FILE=$(pwd)/codebuild/release/settings.xml
       - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-Release --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz
       - tar -xvf ~/mvn_gpg.tgz -C ~
+      # Build and deploy TestVectorsAwsCryptographicMaterialProviders to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
   build:
     commands:
       - |

codebuild/release/release-staging.yml

@@ -18,13 +18,32 @@ phases:
   install:
     runtime-versions:
       java: corretto11
+    commands:
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
   pre_build:
     commands:
       - export SETTINGS_FILE=$(pwd)/codebuild/release/settings.xml
       - export CODEARTIFACT_TOKEN=$(aws codeartifact get-authorization-token --domain $DOMAIN --domain-owner $ACCOUNT --query authorizationToken --output text --region ${REGION})
       - export CODEARTIFACT_REPO_URL=https://${DOMAIN}-${ACCOUNT}.d.codeartifact.${REGION}.amazonaws.com/maven/${REPOSITORY}
       - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-Release --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz
       - tar -xvf ~/mvn_gpg.tgz -C ~
+      # Build and deploy TestVectorsAwsCryptographicMaterialProviders to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
   build:
     commands:
       - VERSION_HASH="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)-$CODEBUILD_RESOLVED_SOURCE_VERSION"