Skip to content

Commit

Permalink
Re-adding the windows snapshot with new changes (#2550)
Browse files Browse the repository at this point in the history 
* Re-adding the windows snapshot with new changes

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Update manifests

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Add more tests

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Restore Pipefile.lock

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Tweak comments

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
  • Loading branch information
@peterzhuamazon
peterzhuamazon authored Sep 2, 2022
1 parent b657fba commit e1c5883
Show file tree
Hide file tree
Showing 5 changed files with 50 additions and 44 deletions.
62 changes: 31 additions & 31 deletions jenkins/opensearch/distribution-build.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,37 +199,37 @@ pipeline {
}
}
}
//stage('build-snapshot-windows-x64-tar') {
// agent {
// node {
// label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host'
// }
// }
// tools {
// jdk dockerAgent.javaVersion
// }
// steps {
// script {
// buildManifest(
// componentName: "${COMPONENT_NAME}",
// inputManifest: "manifests/${INPUT_MANIFEST}",
// distribution: 'zip',
// platform: 'windows',
// snapshot: true
// )
// echo("Uploading windows min snapshots to S3")
// uploadMinSnapshotsToS3(
// fileActions: [createSha512Checksums()],
// distribution: 'zip'
// )
// }
// }
// post {
// always {
// postCleanup()
// }
// }
//}
stage('build-snapshot-windows-x64-zip') {
agent {
node {
label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host'
}
}
tools {
jdk dockerAgent.javaVersion
}
steps {
script {
buildManifest(
componentName: "${COMPONENT_NAME}",
inputManifest: "manifests/${INPUT_MANIFEST}",
distribution: 'zip',
platform: 'windows',
snapshot: true
)
echo("Uploading windows min snapshots to S3")
uploadMinSnapshotsToS3(
fileActions: [createSha512Checksums()],
distribution: 'zip'
)
}
}
post {
always {
postCleanup()
}
}
}
stage('build-and-test-x64-tar') {
agent {
docker {
Expand Down
10 changes: 5 additions & 5 deletions src/system/temporary_directory.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,17 @@ def g__handleRemoveReadonly(func: FunctionType, path: str, exc: Any) -> Any:
logging.debug(f"excvalue {excvalue}")
logging.debug(f"func {func.__name__}")
if func in (os.rmdir, os.remove, os.unlink):
os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO | stat.S_IWRITE | stat.S_IREAD) # 0777 nix* / +wr windows
os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO | stat.S_IREAD | stat.S_IWRITE) # 0777 nix* / +rw windows
retry_total = 3
for retry_count in range(retry_total): # Re-run func to force deletion especially on windows
try:
logging.debug(f'Try count: {retry_count + 1}/{retry_total}')
logging.warn(f'Removing try count: {retry_count + 1}/{retry_total} for {path}')
func_result = func(path)
logging.debug(f'func_result: {func_result}')
if func_result is None:
break
except Exception as ex:
logging.warn(func_result)
logging.warn(ex)
logging.warn(f'Exception: {ex}')
else:
raise

Expand Down Expand Up @@ -58,5 +58,5 @@ def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None:
if self.keep:
logging.info(f"Keeping {self.name}")
else:
logging.debug(f"Removing {self.name}")
logging.info(f"Removing {self.name}")
shutil.rmtree(self.name, ignore_errors=False, onerror=g__handleRemoveReadonly)
5 changes: 3 additions & 2 deletions tests/jenkins/jobs/uploadMinSnapshotsToS3_Jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@ ccc
bbb
ccc
})
uploadMinSnapshotsToS3.echo(Start copying files)
uploadMinSnapshotsToS3.sh(
cp tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz
cp tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz
cp -v tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64.tar.gz.sha512 tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512
sed -i "s/.tar.gz/-latest.tar.gz/g" tests/data/tar/builds/opensearch/dist/opensearch-min-1.2.2-SNAPSHOT-linux-x64-latest.tar.gz.sha512
)
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
Expand Down
2 changes: 1 addition & 1 deletion vars/createSha512Checksums.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Closure call() {

return { argsMap -> body: {

final foundFiles = sh(script: "find $argsMap.artifactPath -type f", returnStdout: true).split()
final foundFiles = sh(script: "find ${argsMap.artifactPath} -type f", returnStdout: true).split()

for (file in foundFiles) {
acceptTypeFound = false
Expand Down
15 changes: 10 additions & 5 deletions vars/uploadMinSnapshotsToS3.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,12 @@ void call(Map args = [:]) {
extension = buildManifest.build.getExtension()

// Setup src & dst variables for artifacts
srcDir = "${WORKSPACE}/${args.distribution}/builds/${productName}/dist"
dstDir = "snapshots/core/${productName}/${version}"
baseName = "${productName}-min-${version}-${platform}-${architecture}"
// Replace backslash with forward slash ('\' to '/') in path
// Compatible with both Windows as well as any nix* env
// Else the map in groovy will treat '\' as escape char on Windows
String srcDir = "${WORKSPACE}/${args.distribution}/builds/${productName}/dist".replace("\\", "/")
String dstDir = "snapshots/core/${productName}/${version}"
String baseName = "${productName}-min-${version}-${platform}-${architecture}"

// Create checksums
echo('Create .sha512 for Min Snapshots Artifacts')
Expand All @@ -34,9 +37,11 @@ void call(Map args = [:]) {
action(argsMap)
}

echo('Start copying files')

sh """
cp ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
cp ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
cp -v ${srcDir}/${baseName}.${extension} ${srcDir}/${baseName}-latest.${extension}
cp -v ${srcDir}/${baseName}.${extension}.sha512 ${srcDir}/${baseName}-latest.${extension}.sha512
sed -i "s/.${extension}/-latest.${extension}/g" ${srcDir}/${baseName}-latest.${extension}.sha512
"""
withCredentials([
Expand Down

0 comments on commit e1c5883

Please sign in to comment.