Merge pull request #521 from linsword13/format-objs

Format Ramble's object files with `black`

lib/ramble/ramble/cmd/style.py

@@ -344,6 +344,11 @@ def filter_file(source, dest, output=False):
                     sys.stdout.write(line)
 
 
+def _split_file_list(file_list):
+    """Return a tuple of (primary_files, obj_files)"""
+    return [f for f in file_list if not is_object(f)], [f for f in file_list if is_object(f)]
+
+
 @tool("flake8")
 def run_flake8(flake8_cmd, file_list, args):
     temp = tempfile.mkdtemp()
@@ -352,8 +357,7 @@ def run_flake8(flake8_cmd, file_list, args):
         print_tool_header("flake8", file_list)
 
         # run flake8 on the temporary tree, once for core, once for apps
-        application_file_list = [f for f in file_list if is_object(f)]
-        primary_file_list = [f for f in file_list if not is_object(f)]
+        primary_file_list, application_file_list = _split_file_list(file_list)
 
         # filter files into a temporary directory with exemptions added.
         # TODO: DRY this duplication
@@ -424,13 +428,29 @@ def run_flake8(flake8_cmd, file_list, args):
 @tool("black")
 def run_black(black_cmd, file_list, args):
     print_tool_header("black", file_list)
-    black_args = ("--config", os.path.join(ramble.paths.prefix, "pyproject.toml"))
+    common_args = ("--config", os.path.join(ramble.paths.prefix, "pyproject.toml"))
     if not args.fix:
-        black_args += ("--check", "--diff")
-    black_args += tuple(file_list)
+        common_args += ("--check", "--diff")
+    primary_files, obj_files = _split_file_list(file_list)
+    output = ""
+    returncode = 0
+
+    # Operate on primary and object (apps and mods) files spearately with varying configs.
+    if primary_files:
+        output += black_cmd(
+            *(common_args + tuple(primary_files)), fail_on_error=False, output=str, error=str
+        )
+        returncode |= black_cmd.returncode
+
+    if obj_files:
+        output += black_cmd(
+            *(common_args + ("--config", "pyproject_objects.toml") + tuple(obj_files)),
+            fail_on_error=False,
+            output=str,
+            error=str,
+        )
+        returncode |= black_cmd.returncode
 
-    output = black_cmd(*black_args, fail_on_error=False, output=str, error=str)
-    returncode = black_cmd.returncode
     print_output(output, args)
     print_tool_result("black", returncode)
     return returncode
@@ -483,6 +503,7 @@ def prefix_relative(path):
             try:
                 base, untracked, list_all = arg_flags.pop(0)
                 file_list = changed_files(base, untracked, list_all)
+                break
             except ProcessError as e:
                 file_list = None
                 if not arg_flags:

pyproject_objects.toml

@@ -0,0 +1,8 @@
+# Config file for Ramble's object files (apps and mods)
+
+[tool.black]
+line-length = 79
+target-version = ["py36", "py37", "py38", "py39", "py310", "py311", "py312"]
+include = '''
+    \.pyi?$
+'''

var/ramble/repos/builtin.mock/applications/basic-inherited/application.py

@@ -14,11 +14,18 @@
 class BasicInherited(BaseBasic):
     name = "basic-inherited"
 
-    input_file('inherited_input', url='file:///tmp/inherited_file.log',
-               description='Again, not a file', extension='.log')
+    input_file(
+        "inherited_input",
+        url="file:///tmp/inherited_file.log",
+        description="Again, not a file",
+        extension=".log",
+    )
 
-    workload('test_wl3', executable='foo', input='inherited_input')
+    workload("test_wl3", executable="foo", input="inherited_input")
 
-    workload_variable('my_var', default='1.0',
-                      description='Shadowed Example var',
-                      workload='test_wl')
+    workload_variable(
+        "my_var",
+        default="1.0",
+        description="Shadowed Example var",
+        workload="test_wl",
+    )

var/ramble/repos/builtin.mock/applications/basic/application.py

@@ -12,30 +12,41 @@
 class Basic(ExecutableApplication):
     name = "basic"
 
-    executable('foo', 'bar', use_mpi=False)
-    executable('bar', 'baz', use_mpi=True)
-    executable('echo', 'echo "0.25 seconds"', use_mpi=False)
-
-    input_file('input', url='file:///tmp/test_file.log',
-               description='Not a file', extension='.log')
-
-    workload('test_wl', executable='foo', input='input')
-    workload('test_wl2', executable='bar', input='input')
-    workload('working_wl', executable='echo')
-
-    workload_variable('my_base_var', default='0.0',
-                      description='Example var',
-                      workload='test_wl')
-
-    workload_variable('my_var', default='1.0',
-                      description='Example var',
-                      workload='test_wl')
-
-    environment_variable('TEST_ENV', value="1",
-                         description="test var", workload="test_wl")
-
-    archive_pattern('{experiment_run_dir}/archive_test.*')
-
-    figure_of_merit('test_fom',
-                    fom_regex=r'(?P<test>[0-9]+\.[0-9]+).*seconds.*',
-                    group_name='test', units='s')
+    executable("foo", "bar", use_mpi=False)
+    executable("bar", "baz", use_mpi=True)
+    executable("echo", 'echo "0.25 seconds"', use_mpi=False)
+
+    input_file(
+        "input",
+        url="file:///tmp/test_file.log",
+        description="Not a file",
+        extension=".log",
+    )
+
+    workload("test_wl", executable="foo", input="input")
+    workload("test_wl2", executable="bar", input="input")
+    workload("working_wl", executable="echo")
+
+    workload_variable(
+        "my_base_var",
+        default="0.0",
+        description="Example var",
+        workload="test_wl",
+    )
+
+    workload_variable(
+        "my_var", default="1.0", description="Example var", workload="test_wl"
+    )
+
+    environment_variable(
+        "TEST_ENV", value="1", description="test var", workload="test_wl"
+    )
+
+    archive_pattern("{experiment_run_dir}/archive_test.*")
+
+    figure_of_merit(
+        "test_fom",
+        fom_regex=r"(?P<test>[0-9]+\.[0-9]+).*seconds.*",
+        group_name="test",
+        units="s",
+    )

var/ramble/repos/builtin.mock/applications/expanded_foms/application.py

@@ -12,23 +12,29 @@
 class ExpandedFoms(ExecutableApplication):
     name = "expanded-Foms"
 
-    executable('foo', template=['bar', 'echo "{my_var}"'], use_mpi=False)
+    executable("foo", template=["bar", 'echo "{my_var}"'], use_mpi=False)
 
-    input_file('input', url='file:///tmp/test_file.log',
-               description='Not a file', extension='.log')
+    input_file(
+        "input",
+        url="file:///tmp/test_file.log",
+        description="Not a file",
+        extension=".log",
+    )
 
-    workload('test_wl', executables=['foo'], input='input')
+    workload("test_wl", executables=["foo"], input="input")
 
-    workload_variable('my_var', default='1.0',
-                      description='Example var',
-                      workload='test_wl')
+    workload_variable(
+        "my_var", default="1.0", description="Example var", workload="test_wl"
+    )
 
-    archive_pattern('{experiment_run_dir}/archive_test.*')
+    archive_pattern("{experiment_run_dir}/archive_test.*")
 
-    figure_of_merit('test_fom {var}',
-                    fom_regex=r'Collect FOM (?P<var>\w+)\s=\s(?P<test>[0-9]+\.[0-9]+) seconds',
-                    log_file='{log_file}',
-                    group_name='test', units='s')
+    figure_of_merit(
+        "test_fom {var}",
+        fom_regex=r"Collect FOM (?P<var>\w+)\s=\s(?P<test>[0-9]+\.[0-9]+) seconds",
+        log_file="{log_file}",
+        group_name="test",
+        units="s",
+    )
 
-    success_criteria('Run', mode='string',
-                     match=r'Collect', file='{log_file}')
+    success_criteria("Run", mode="string", match=r"Collect", file="{log_file}")

var/ramble/repos/builtin.mock/applications/glob-patterns/application.py

@@ -12,46 +12,98 @@
 class GlobPatterns(ExecutableApplication):
     name = "glob-patterns"
 
-    executable('test', 'base test {test_var} {glob_var} {baz_var}', use_mpi=False)
-    executable('test-foo', 'test foo {test_var} {glob_var} {baz_var} {mod_var}', use_mpi=False)
-    executable('test-bar', 'test bar {test_var} {glob_var} {baz_var}', use_mpi=True)
-    executable('baz', 'baz {test_var} {glob_var} {baz_var}', use_mpi=True)
-
-    input_file('input', url='file:///tmp/test_file.log',
-               description='Not a file', extension='.log')
-    input_file('input-foo', url='file:///tmp/test_foo_file.log',
-               description='Not a file', extension='.log')
-    input_file('input-bar', url='file:///tmp/test_bar_file.log',
-               description='Not a file', extension='.log')
-    input_file('baz', url='file:///tmp/baz_file.log',
-               description='Not a file', extension='.log')
-
-    workload('test_one_exec', executables=['test'], inputs=['input'])
-    workload('test_three_exec', executables=['test*'], inputs=['input*'])
-    workload('one_baz_exec', executables=['baz'], inputs=['baz'])
-
-    environment_variable('env_var_test', 'set', description='Test env var',
-                         workloads=['test_one_exec'])
-    environment_variable('env_var_glob', 'set', description='Test env var',
-                         workloads=['test*'])
-    environment_variable('env_var_baz', 'set', description='Test env var',
-                         workloads=['one_baz_exec'])
-    environment_variable('env_var_mod', 'set', description='Env var to be modified',
-                         workloads=['test_three_exec'])
-
-    workload_variable('test_var', default='wl_var_test',
-                      description='Example var',
-                      workloads=['test_one_exec'])
-    workload_variable('glob_var', default='wl_var_glob',
-                      description='Example var',
-                      workloads=['test*'])
-    workload_variable('baz_var', default='wl_var_baz',
-                      description='Example var',
-                      workloads=['one_baz_exec'])
-    workload_variable('var_mod', default='wl_var_mod',
-                      description='Variable to be modified',
-                      workloads=['test_three_exec'])
-
-    figure_of_merit('test_fom',
-                    fom_regex=r'(?P<test>[0-9]+\.[0-9]+).*seconds.*',
-                    group_name='test', units='s')
+    executable(
+        "test", "base test {test_var} {glob_var} {baz_var}", use_mpi=False
+    )
+    executable(
+        "test-foo",
+        "test foo {test_var} {glob_var} {baz_var} {mod_var}",
+        use_mpi=False,
+    )
+    executable(
+        "test-bar", "test bar {test_var} {glob_var} {baz_var}", use_mpi=True
+    )
+    executable("baz", "baz {test_var} {glob_var} {baz_var}", use_mpi=True)
+
+    input_file(
+        "input",
+        url="file:///tmp/test_file.log",
+        description="Not a file",
+        extension=".log",
+    )
+    input_file(
+        "input-foo",
+        url="file:///tmp/test_foo_file.log",
+        description="Not a file",
+        extension=".log",
+    )
+    input_file(
+        "input-bar",
+        url="file:///tmp/test_bar_file.log",
+        description="Not a file",
+        extension=".log",
+    )
+    input_file(
+        "baz",
+        url="file:///tmp/baz_file.log",
+        description="Not a file",
+        extension=".log",
+    )
+
+    workload("test_one_exec", executables=["test"], inputs=["input"])
+    workload("test_three_exec", executables=["test*"], inputs=["input*"])
+    workload("one_baz_exec", executables=["baz"], inputs=["baz"])
+
+    environment_variable(
+        "env_var_test",
+        "set",
+        description="Test env var",
+        workloads=["test_one_exec"],
+    )
+    environment_variable(
+        "env_var_glob", "set", description="Test env var", workloads=["test*"]
+    )
+    environment_variable(
+        "env_var_baz",
+        "set",
+        description="Test env var",
+        workloads=["one_baz_exec"],
+    )
+    environment_variable(
+        "env_var_mod",
+        "set",
+        description="Env var to be modified",
+        workloads=["test_three_exec"],
+    )
+
+    workload_variable(
+        "test_var",
+        default="wl_var_test",
+        description="Example var",
+        workloads=["test_one_exec"],
+    )
+    workload_variable(
+        "glob_var",
+        default="wl_var_glob",
+        description="Example var",
+        workloads=["test*"],
+    )
+    workload_variable(
+        "baz_var",
+        default="wl_var_baz",
+        description="Example var",
+        workloads=["one_baz_exec"],
+    )
+    workload_variable(
+        "var_mod",
+        default="wl_var_mod",
+        description="Variable to be modified",
+        workloads=["test_three_exec"],
+    )
+
+    figure_of_merit(
+        "test_fom",
+        fom_regex=r"(?P<test>[0-9]+\.[0-9]+).*seconds.*",
+        group_name="test",
+        units="s",
+    )

var/ramble/repos/builtin.mock/applications/input-test/application.py

@@ -13,17 +13,30 @@
 class InputTest(ExecutableApplication):
     name = "input-test"
 
-    executable('test', 'echo "repo test"', use_mpi=False)
+    executable("test", 'echo "repo test"', use_mpi=False)
 
     cwd = os.getcwd()
-    input_file('test-input1', url=f'file://{cwd}/input1.tar.gz',
-               description='Test input')
-
-    input_file('test-input2', url=f'file://{cwd}/input2.tar.gz',
-               description='Test input')
-
-    input_file('test-input3', url=f'file://{cwd}/input3.txt',
-               expand=False,
-               description='Test input')
-
-    workload('test', executables=['test'], inputs=['test-input1', 'test-input2', 'test-input3'])
+    input_file(
+        "test-input1",
+        url=f"file://{cwd}/input1.tar.gz",
+        description="Test input",
+    )
+
+    input_file(
+        "test-input2",
+        url=f"file://{cwd}/input2.tar.gz",
+        description="Test input",
+    )
+
+    input_file(
+        "test-input3",
+        url=f"file://{cwd}/input3.txt",
+        expand=False,
+        description="Test input",
+    )
+
+    workload(
+        "test",
+        executables=["test"],
+        inputs=["test-input1", "test-input2", "test-input3"],
+    )