Update benchpark list To Remove Legacy Information (#634)

* Return current modifiers, systems, experiments, and benchmarks instead of legacy versions

* Exclude certain experiment variants

* Print systems instead of hardware descriptions

* Add clusters to system print out

* Add coloring of output

* black

* Update commands in the docs

---------

Co-authored-by: Mckinsey <mckinsey1@llnl.gov>
Co-authored-by: pearce8 <pearce8@llnl.gov>

docs/basic-usage.rst

@@ -3,13 +3,9 @@
 
    SPDX-License-Identifier: Apache-2.0
 
-===========
-Basic Usage
-===========
-
-------------------
+==================
 Benchpark Commands
-------------------
+==================
 
 The easiest way to get started is to run existing experiments one existing systems, or 
 to modify one that is similar. You can search through the existing experiments and benchmarks with the below commands. 
@@ -24,14 +20,20 @@ Search for available system and experiment specifications in Benchpark.
      - Description
      - Listing in the docs
    * - benchpark list
-     - Lists all benchmarks and systems specified in Benchpark
+     - Lists all benchmarks, systems, and experiments specified in Benchpark
      -
    * - benchpark list systems
      - Lists all system specified in Benchpark
      - :doc:`system-list`
    * - benchmark list benchmarks
      - Lists all benchmarks specified in Benchpark
      - :doc:`benchmark-list`
+   * - benchmark list experiments
+     - Lists all experiments specified in Benchpark
+     - :doc:`benchmark-list`
+   * - benchmark list modifiers
+     - Lists all modifiers specified in Benchpark
+     - :doc:`modifiers`
    * - benchpark tags workspace
      - Lists all tags specified in Benchpark
      -
@@ -50,11 +52,8 @@ Search for available system and experiment specifications in Benchpark.
 
 Now that you know the existing benchmarks and systems, you can determine your necessary workflow in :doc:`benchpark-workflow`.
 
-------------
-Getting Help
-------------
 
-Benchpark help menu::
+Benchpark also has a help menu::
 
     $ benchpark --help
 

lib/benchpark/accounting.py

@@ -7,36 +7,58 @@
 
 import benchpark.paths
 
+exclude_exper = ["repo.yaml"]
+
 
 def benchpark_experiments():
     source_dir = benchpark.paths.benchpark_root
     experiments = []
-    experiments_dir = source_dir / "legacy" / "experiments"
-    for x in os.listdir(experiments_dir):
-        for y in os.listdir(experiments_dir / x):
-            experiments.append(f"{x}/{y}")
+    experiments_dir = source_dir / "experiments"
+    exclude_variants = ["Caliper"]
+    for x in sorted(os.listdir(experiments_dir)):
+        if x not in exclude_exper:
+            experiment_spec = benchpark.spec.ExperimentSpec(x)
+            conc = experiment_spec.concretize()
+            experiment_class = conc.experiment
+            for h in experiment_class.__dict__["helpers"]:
+                variant = str(h).split(".")[2]
+                if variant not in exclude_variants:
+                    experiments.append(f"{x}/{variant}")
     return experiments
 
 
 def benchpark_modifiers():
     source_dir = benchpark.paths.benchpark_root
     modifiers = []
-    for x in os.listdir(source_dir / "modifiers"):
-        modifiers.append(x)
-
-    modifiers += [
-        x for x in os.listdir(source_dir / "legacy" / "modifiers") if x not in modifiers
-    ]
+    exclude = ["modifier_repo.yaml"]
+    for x in sorted(os.listdir(source_dir / "modifiers")):
+        if x not in exclude:
+            modifiers.append(x)
 
     return modifiers
 
 
 def benchpark_systems():
     source_dir = benchpark.paths.benchpark_root
     systems = []
-    for x in os.listdir(source_dir / "legacy" / "systems"):
-        if not (
-            os.path.isfile(os.path.join(source_dir / "configs", x)) or x == "common"
-        ):
-            systems.append(x)
+    exclude = ["all_hardware_descriptions", "repo.yaml"]
+    for x in sorted(os.listdir(source_dir / "systems")):
+        if x not in exclude:
+            system_spec = benchpark.spec.SystemSpec(x)
+            system_class = system_spec.system_class
+            if hasattr(system_class, "id_to_resources"):
+                for c in system_class.id_to_resources.keys():
+                    systems.append(x + "/" + c)
+            else:
+                systems.append(x)
     return systems
+
+
+def benchpark_benchmarks():
+    source_dir = benchpark.paths.benchpark_root
+    benchmarks = []
+    experiments_dir = source_dir / "experiments"
+    for x in sorted(os.listdir(experiments_dir)):
+        if x not in exclude_exper:
+            benchmarks.append(f"{x}")
+    return benchmarks

lib/main.py

@@ -50,6 +50,7 @@
     benchpark_experiments,
     benchpark_modifiers,
     benchpark_systems,
+    benchpark_benchmarks,
 )
 
 
@@ -114,15 +115,6 @@ def benchpark_list(subparsers, actions_dict):
     actions_dict["list"] = benchpark_list_handler
 
 
-def benchpark_benchmarks():
-    source_dir = benchpark.paths.benchpark_root
-    benchmarks = []
-    experiments_dir = source_dir / "legacy" / "experiments"
-    for x in os.listdir(experiments_dir):
-        benchmarks.append(f"{x}")
-    return benchmarks
-
-
 def benchpark_get_tags():
     f = benchpark.paths.benchpark_root / "taxonomy.yaml"
     tags = []
@@ -152,29 +144,42 @@ def benchpark_list_handler(args):
     systems = benchpark_systems()
     modifiers = benchpark_modifiers()
 
+    try:
+        import llnl.util.tty.color as color
+
+        colors = True
+    except ImportError:
+        colors = False
+
+    def _print_helper(name, collection, colors=colors):
+        func = print
+        strs = ["", ""]
+        end = ""
+        if colors:
+            func = color.cprint
+            name = "@*b" + name + "@."
+            strs = ["@*r", "@*c"]
+            end = "@."
+
+        func(name)
+        for item in collection:
+            if "/" in item:
+                item = item.split("/")
+                func(f"    {strs[0]+item[0]+end+'/'+strs[1]+item[1]+end}")
+            else:
+                func(f"    {strs[0]+item+end}")
+
     if sublist is None:
-        print("Experiments:")
-        for experiment in experiments:
-            print(f"\t{experiment}")
-        print("Systems:")
-        for system in systems:
-            print(f"\t{system}")
+        _print_helper("Experiments:", experiments)
+        _print_helper("Systems:", systems)
     elif sublist == "benchmarks":
-        print("Benchmarks:")
-        for benchmark in benchmarks:
-            print(f"\t{benchmark}")
+        _print_helper("Benchmarks:", benchmarks)
     elif sublist == "experiments":
-        print("Experiments:")
-        for experiment in experiments:
-            print(f"\t{experiment}")
+        _print_helper("Experiments:", experiments)
     elif sublist == "systems":
-        print("Systems:")
-        for system in systems:
-            print(f"\t{system}")
+        _print_helper("Systems:", systems)
     elif sublist == "modifiers":
-        print("Modifiers:")
-        for modifier in modifiers:
-            print(f"\t{modifier}")
+        _print_helper("Modifiers:", modifiers)
     else:
         raise ValueError(
             f'Invalid benchpark list "{sublist}" - must choose [experiments], [systems], [modifiers] or leave empty'