From f61136998b0c51bf854e40a8ace643e7cc8dce68 Mon Sep 17 00:00:00 2001
From: Jeremy Lewi <jeremy@lewi.us>
Date: Tue, 24 Jul 2018 11:07:55 -0700
Subject: [PATCH] Copy artifacts should make the file names unique by appending
 a suffix. (#183)

* Copy artifacts should make the file names unique by appending a suffix.

* See kubeflow/tf-operator#748
* A test can run multiple instances of a workflow but with different parameters.
* In this case we need to make sure the junit files and other artifacts
  copied to GCS for gubernator have unique names.

* One way to make this easier is to have copy-artifacts automatically
  append a unique suffix to each file before copying it to GCS.

* Fix lint.
---
 py/kubeflow/testing/prow_artifacts.py    | 18 ++++++++++++++++++
 workflows/components/workflows.libsonnet |  1 +
 2 files changed, 19 insertions(+)

diff --git a/py/kubeflow/testing/prow_artifacts.py b/py/kubeflow/testing/prow_artifacts.py
index 07c40810aec..74dff0e5dc4 100644
--- a/py/kubeflow/testing/prow_artifacts.py
+++ b/py/kubeflow/testing/prow_artifacts.py
@@ -134,6 +134,17 @@ def copy_artifacts(args):
 
   output = get_gcs_dir(args.bucket)
 
+  if args.suffix:
+    logging.info("Renaming all artifact files to include %s", args.suffix)
+    for dirpath, _, files in os.walk(args.artifacts_dir):
+      for filename in files:
+        full_path = os.path.join(dirpath, filename)
+
+        name, ext = os.path.splitext(filename)
+        new_name = "{0}-{1}{2}".format(name, args.suffix, ext)
+        new_path = os.path.join(dirpath, new_name)
+        logging.info("Rename %s to %s", full_path, new_path)
+        os.rename(full_path, new_path)
   util.maybe_activate_service_account()
   util.run(["gsutil", "-m", "rsync", "-r", args.artifacts_dir, output])
 
@@ -255,6 +266,13 @@ def main(unparsed_args=None):  # pylint: disable=too-many-locals
     type=str,
     help="Bucket to copy the artifacts to.")
 
+  parser_copy.add_argument(
+    "--suffix",
+    default="",
+    type=str,
+    help=("Optional if supplied add this suffix to the names of all artifact "
+          "files before copying them to the GCS bucket."))
+
   parser_copy.set_defaults(func=copy_artifacts)
 
   #############################################################################
diff --git a/workflows/components/workflows.libsonnet b/workflows/components/workflows.libsonnet
index 2b7bd2867de..b6d47064019 100644
--- a/workflows/components/workflows.libsonnet
+++ b/workflows/components/workflows.libsonnet
@@ -213,6 +213,7 @@
               "--artifacts_dir=" + outputDir,
               "copy_artifacts",
               "--bucket=" + bucket,
+              "--suffix=fakesuffix",
             ]),  // copy-artifacts
           ],  // templates
         },