Added support for snapshot builds.

Signed-off-by: dblock <dblock@amazon.com>

bundle-workflow/README.md

@@ -10,6 +10,12 @@ The OpenSearch repo is built first, followed by common-utils. These dependencies
 ./bundle-workflow/build.sh manifests/opensearch-1.0.0.yml
 ```
 
+The following options are available.
+
+| name       | description                                                         |
+|------------|---------------------------------------------------------------------|
+| --snapshot | Build a snapshot instead of a release artifact, default is `false`. |
+
 ### Scripts
 
 Each component build relies on a `build.sh` script that is used to prepare bundle artifacts for a particular bundle version that takes two arguments: version and target architecture. By default the tool will look for a script in [scripts/bundle-build/components](scripts/bundle-build/components), then in the checked-out repository in `build/build.sh`, then default to a Gradle build implemented in [scripts/bundle-build/standard-gradle-build](scripts/bundle-build/standard-gradle-build).

bundle-workflow/python/build.py

@@ -15,6 +15,7 @@
 
 parser = argparse.ArgumentParser(description = "Build an OpenSearch Bundle")
 parser.add_argument('manifest', type = argparse.FileType('r'), help="Manifest file.")
+parser.add_argument('-s', '--snapshot', action = 'store_true', default = False, help="Build snapshot.")
 args = parser.parse_args()
 
 component_scripts_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../scripts/bundle-build/components')
@@ -35,12 +36,12 @@ def get_arch():
 os.makedirs(output_dir, exist_ok = True)
 build_id = os.getenv('OPENSEARCH_BUILD_ID', uuid.uuid4().hex)
 
-with tempfile.TemporaryDirectory() as work_dir:
+with tempfile.TemporaryDirectory() as work_dir: 
     print(f'Building in {work_dir}')
 
     os.chdir(work_dir)
 
-    build_recorder = BuildRecorder(build_id, output_dir, manifest.build.name, manifest.build.version, arch)
+    build_recorder = BuildRecorder(build_id, output_dir, manifest.build.name, manifest.build.version, arch, args.snapshot)
 
     print(f'Building {manifest.build.name} ({arch}) into {output_dir}')
 
@@ -52,7 +53,7 @@ def get_arch():
                           component_scripts_path,
                           default_build_path,
                           build_recorder)
-        builder.build(manifest.build.version, arch)
+        builder.build(manifest.build.version, arch, args.snapshot)
         builder.export_artifacts()
 
     build_recorder.write_manifest(output_dir)

bundle-workflow/python/build_workflow/build_recorder.py

@@ -7,9 +7,9 @@
 from manifests.build_manifest import BuildManifest
 
 class BuildRecorder:
-    def __init__(self, build_id, output_dir, name, version, arch):
+    def __init__(self, build_id, output_dir, name, version, arch, snapshot):
         self.output_dir = output_dir
-        self.build_manifest = self.BuildManifestBuilder(build_id, name, version, arch)
+        self.build_manifest = self.BuildManifestBuilder(build_id, name, version, arch, snapshot)
 
     def record_component(self, component_name, git_repo):
         self.build_manifest.append_component(component_name, git_repo.url, git_repo.ref, git_repo.sha)
@@ -35,13 +35,14 @@ def write_manifest(self, folder):
             yaml.dump(output_manifest.to_dict(), file)
 
     class BuildManifestBuilder:
-        def __init__(self, build_id, name, version, arch):
+        def __init__(self, build_id, name, version, arch, snapshot):
             self.data = {}
             self.data['build'] = {}
             self.data['build']['id'] = build_id
             self.data['build']['name'] = name
             self.data['build']['version'] = str(version)
             self.data['build']['architecture'] = arch
+            self.data['build']['snapshot'] = str(snapshot)
             self.data['schema-version'] = '1.0'
             # We need to store components as a hash so that we can append artifacts by component name
             # When we convert to a BuildManifest this will get converted back into a list

bundle-workflow/python/build_workflow/builder.py

@@ -24,8 +24,9 @@ def __init__(self, component_name, git_repo, component_scripts_path, default_bui
         self.component_scripts_path = component_scripts_path
         self.default_build_script = default_build_script
         self.build_recorder = build_recorder
+        self.output_path = 'artifacts'
 
-    def build(self, version, arch):
+    def build(self, version, arch, snapshot):
         paths = [os.path.realpath(os.path.join(self.git_repo.dir.name, 'scripts/build.sh')),
                  os.path.realpath(os.path.join(self.component_scripts_path, self.component_name, 'build.sh')),
                  self.default_build_script]
@@ -34,12 +35,12 @@ def build(self, version, arch):
         if build_script is None:
             raise RuntimeError(f'Could not find build.sh script. Looked in {paths}')
 
-        build_command = f'{build_script} {version} {arch}'
+        build_command = f'{build_script} -v {version} -a {arch} -s {str(snapshot).lower()} -o {self.output_path}'
         self.git_repo.execute(build_command)
         self.build_recorder.record_component(self.component_name, self.git_repo)
 
     def export_artifacts(self):
-        artifacts_dir = os.path.realpath(os.path.join(self.git_repo.dir.name, 'artifacts'))
+        artifacts_dir = os.path.realpath(os.path.join(self.git_repo.dir.name, self.output_path))
         for artifact_type in ["maven", "bundle", "plugins", "libs"]:
             for dir, dirs, files in os.walk(os.path.join(artifacts_dir, artifact_type)):
                 for file_name in files:

bundle-workflow/scripts/bundle-build/components/OpenSearch/build.sh

@@ -5,27 +5,71 @@
 
 set -e
 
-version="$1"
-ARCHITECTURE="$2"
+function usage() {
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Arguments:"
+    echo -e "-v VERSION\t[Required] OpenSearch version."
+    echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'."
+    echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored."
+    echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'."
+    echo -e "-h help"
+}
 
-outputDir=artifacts
+while getopts ":h:v:s:o:a:" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        v)
+            VERSION=$OPTARG
+            ;;
+        s)
+            SNAPSHOT=$OPTARG
+            ;;
+        o)
+            OUTPUT=$OPTARG
+            ;;
+        a)
+            ARCHITECTURE=$OPTARG
+            ;;
+        :)
+            echo "Error: -${OPTARG} requires an argument"
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${arg}"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -z "$VERSION" ]; then
+    echo "Error: You must specify the OpenSearch version"
+    usage
+    exit 1
+fi
+
+[ -z "$OUTPUT" ] && OUTPUT=artifacts
+
+outputDir=$OUTPUT
 mkdir -p "${outputDir}/maven"
 
 # Build project and publish to maven local.
-./gradlew publishToMavenLocal -Dbuild.snapshot=false
+./gradlew publishToMavenLocal -Dbuild.snapshot=$SNAPSHOT
 
 # Publish to existing test repo, using this to stage release versions of the artifacts that can be released from the same build.
-./gradlew publishNebulaPublicationToTestRepository -Dbuild.snapshot=false
+./gradlew publishNebulaPublicationToTestRepository -Dbuild.snapshot=$SNAPSHOT
 
 # Copy maven publications to be promoted
 cp -r ./build/local-test-repo/org/opensearch "${outputDir}"/maven
 
-if [ "${ARCHITECTURE}" = "x64" ]
-then
+if [ "${ARCHITECTURE}" = "x64" ]; then
   ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=false
   cp -r distribution/archives/linux-tar/build/distributions/. "${outputDir}"/bundle
-elif [ "${ARCHITECTURE}" == "arm64" ]
-then
+elif [ "${ARCHITECTURE}" == "arm64" ]; then
   ./gradlew :distribution:archives:linux-arm64-tar:assemble -Dbuild.snapshot=false
   cp -r distribution/archives/linux-arm64-tar/build/distributions/ "${outputDir}"/bundle
 fi

bundle-workflow/scripts/bundle-build/components/alerting/build.sh

@@ -5,16 +5,64 @@
 
 set -e
 
-echo ${PWD}
-outputDir=artifacts
+function usage() {
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Arguments:"
+    echo -e "-v VERSION\t[Required] OpenSearch version."
+    echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'."
+    echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored."
+    echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'."
+    echo -e "-h help"
+}
 
-mkdir -p $outputDir/plugins
-./gradlew assemble --no-daemon --refresh-dependencies -Dbuild.snapshot=false -DskipTests=true -Dopensearch.version=$1
+while getopts ":h:v:s:o:a:" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        v)
+            VERSION=$OPTARG
+            ;;
+        s)
+            SNAPSHOT=$OPTARG
+            ;;
+        o)
+            OUTPUT=$OPTARG
+            ;;
+        a)
+            ARCHITECTURE=$OPTARG
+            ;;
+        :)
+            echo "Error: -${OPTARG} requires an argument"
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${arg}"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -z "$VERSION" ]; then
+    echo "Error: You must specify the OpenSearch version"
+    usage
+    exit 1
+fi
+
+[[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT
+[ -z "$OUTPUT" ] && OUTPUT=artifacts
+
+mkdir -p $OUTPUT/plugins
+
+./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT
 
 zipPath=$(find . -path \*build/distributions/*.zip)
 distributions="$(dirname "${zipPath}")"
 
 echo "COPY ${distributions}/*.zip"
-cp ${distributions}/*.zip ./$outputDir/plugins
+cp ${distributions}/*.zip ./$OUTPUT/plugins
 
-./gradlew publishToMavenLocal -Dopensearch.version=$1 -Dbuild.snapshot=false
+./gradlew publishToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT

bundle-workflow/scripts/bundle-build/components/common-utils/build.sh

@@ -5,9 +5,58 @@
 
 set -e
 
-opensearch_version=$1
+function usage() {
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Arguments:"
+    echo -e "-v VERSION\t[Required] OpenSearch version."
+    echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'."
+    echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored."
+    echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'."
+    echo -e "-h help"
+}
 
-./gradlew build -Dopensearch.version=$1
-./gradlew publishToMavenLocal -Dopensearch.version=$1
+while getopts ":h:v:s:o:a:" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        v)
+            VERSION=$OPTARG
+            ;;
+        s)
+            SNAPSHOT=$OPTARG
+            ;;
+        o)
+            OUTPUT=$OPTARG
+            ;;
+        a)
+            ARCHITECTURE=$OPTARG
+            ;;
+        :)
+            echo "Error: -${OPTARG} requires an argument"
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${arg}"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -z "$VERSION" ]; then
+    echo "Error: You must specify the OpenSearch version"
+    usage
+    exit 1
+fi
+
+[[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT
+[ -z "$OUTPUT" ] && OUTPUT=artifacts
+
+echo ./gradlew build -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT
+./gradlew build -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT
+./gradlew publishToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT
 mkdir -p artifacts/maven
-cp -r ~/.m2/repository/org/opensearch/common-utils artifacts/maven
+cp -r ~/.m2/repository/org/opensearch/common-utils $OUTPUT/maven

bundle-workflow/scripts/bundle-build/components/dashboards-notebooks/build.sh

@@ -5,10 +5,60 @@
 
 set -e
 
+function usage() {
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Arguments:"
+    echo -e "-v VERSION\t[Required] OpenSearch version."
+    echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'."
+    echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored."
+    echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'."
+    echo -e "-h help"
+}
+
+while getopts ":h:v:s:o:a:" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        v)
+            VERSION=$OPTARG
+            ;;
+        s)
+            SNAPSHOT=$OPTARG
+            ;;
+        o)
+            OUTPUT=$OPTARG
+            ;;
+        a)
+            ARCHITECTURE=$OPTARG
+            ;;
+        :)
+            echo "Error: -${OPTARG} requires an argument"
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${arg}"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -z "$VERSION" ]; then
+    echo "Error: You must specify the OpenSearch version"
+    usage
+    exit 1
+fi
+
+[[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT
+[ -z "$OUTPUT" ] && OUTPUT=artifacts
+
 cd opensearch-notebooks
-./gradlew assemble --no-daemon --refresh-dependencies -Dbuild.snapshot=false -DskipTests=true -Dopensearch.version=$1
+./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT
 
-outputDir=../artifacts
+outputDir=../$OUTPUT
 mkdir -p $outputDir
 
 zipPath=$(find . -path \*build/distributions/*.zip)