diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 151dd375657c..28479f236d15 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -86,7 +86,7 @@ jobs: - uses: ./.github/actions/ubuntu-apt-add-src - name: Run script run: | - source ./build/linux/ci-common.sh && bash test/renderdiff_tests.sh + source ./build/linux/ci-common.sh && bash test/renderdiff/test.sh - uses: actions/upload-artifact@v4 with: name: presubmit-renderdiff-result diff --git a/test/renderdiff/run.py b/test/renderdiff/run.py deleted file mode 100644 index 17a8a7acb47f..000000000000 --- a/test/renderdiff/run.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (C) 2024 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from utils import execute, ArgParseImpl - -from parse_test_json import parse_test_config_from_path -import sys -import os - -def run_test(gltf_viewer, pixel_test, output_dir, opengl_lib=None, vk_icd=None): - assert os.path.isdir(output_dir) - assert os.access(gltf_viewer, os.X_OK) - - for test in pixel_test.tests: - test_json_path = f'{output_dir}/{test.name}_simplified.json' - - with open(test_json_path, 'w') as f: - f.write(f'[{test.to_filament_format()}]') - - for backend in pixel_test.backends: - env = None - if backend == 'opengl' and opengl_lib and os.path.isdir(opengl_lib): - env = {'LD_LIBRARY_PATH': opengl_lib} - - for model in test.models: - model_path = pixel_test.models[model] - out_name = f'{test.name}_{model}_{backend}' - execute(f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless', - env=env, capture_output=False) - execute(f'mv -f {test.name}0.ppm {output_dir}/{out_name}.ppm', capture_output=False) - execute(f'mv -f {test.name}0.json {output_dir}/{test.name}.json', capture_output=False) - -if __name__ == "__main__": - parser = ArgParseImpl() - parser.add_argument('--test', help='Configuration of the test', required=True) - parser.add_argument('--gltf_viewer', help='Path to the gltf_viewer', required=True) - parser.add_argument('--output_dir', help='Output Directory', required=True) - parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)') - parser.add_argument('--vk_icd', help='Path to VK ICD file') - - args, _ = parser.parse_known_args(sys.argv[1:]) - test = parse_test_config_from_path(args.test) - run_test(args.gltf_viewer, test, args.output_dir, opengl_lib=args.opengl_lib, vk_icd=args.vk_icd) diff --git a/test/renderdiff/parse_test_json.py b/test/renderdiff/src/parse_test_json.py similarity index 98% rename from test/renderdiff/parse_test_json.py rename to test/renderdiff/src/parse_test_json.py index 2cc260ffec33..36e7e6521f61 100644 --- a/test/renderdiff/parse_test_json.py +++ b/test/renderdiff/src/parse_test_json.py @@ -63,7 +63,8 @@ def __init__(self, data, existing_models, presets): preset_models = [] if apply_presets: given_presets = {p.name: p for p in presets} - assert all((name in given_presets) for name in apply_presets) + assert all((name in given_presets) for name in apply_presets),\ + f'used preset {name} which is not in {given_presets}' for preset in apply_presets: rendering.update(given_presets[preset].rendering) preset_models += given_presets[preset].models diff --git a/test/renderdiff/src/run.py b/test/renderdiff/src/run.py new file mode 100644 index 000000000000..be90073709cf --- /dev/null +++ b/test/renderdiff/src/run.py @@ -0,0 +1,111 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os + +from utils import execute, ArgParseImpl +from parse_test_json import parse_test_config_from_path + +def important_print(msg): + lines = msg.split('\n') + max_len = max([len(l) for l in lines]) + print('-' * (max_len + 8)) + for line in lines: + diff = max_len - len(line) + information = f'--- {line} ' + (' ' * diff) + '---' + print(information) + print('-' * (max_len + 8)) + +def render_test(gltf_viewer, test_config, output_dir, + opengl_lib=None, vk_icd=None): + assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist" + assert os.access(gltf_viewer, os.X_OK) + + named_output_dir = os.path.join(output_dir, test_config.name) + execute(f'mkdir -p {named_output_dir}') + + results = [] + for test in test_config.tests: + test_json_path = f'{named_output_dir}/{test.name}.simplified.json' + + with open(test_json_path, 'w') as f: + f.write(f'[{test.to_filament_format()}]') + + for backend in test_config.backends: + env = None + if backend == 'opengl' and opengl_lib and os.path.isdir(opengl_lib): + env = {'LD_LIBRARY_PATH': opengl_lib} + + for model in test.models: + model_path = test_config.models[model] + out_name = f'{test.name}.{backend}.{model}' + test_desc = out_name + + important_print(f'Rendering {test_desc}') + + res, _ = execute(f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless', + env=env, capture_output=False) + + if res == 0: + execute(f'mv -f {test.name}0.ppm {named_output_dir}/{out_name}.ppm', capture_output=False) + execute(f'mv -f {test.name}0.json {named_output_dir}/{test.name}.json', capture_output=False) + else: + important_print(f'{test_desc} failed with error={res}') + print('') + + results.append((out_name, res)) + return results + +GOLDENS_DIR = 'renderdiff_goldens' + +# We pull the goldens from the filament-assets repo +def pull_goldens(output_dir): + assert os.path.isdir(output_dir), f"output directory {output_dir} does not exist" + golden_dir = os.path.join(output_dir, "golden") + assets_dir = os.path.join(output_dir, "filament-assets") + + if not os.path.exists(assets_dir): + execute('git clone --depth 1 git@github.com:google/filament-assets.git', cwd=output_dir) + else: + execute('git fetch', cwd=assets_dir) + execute('git checkout main ', cwd=assets_dir) + execute('git rebase', cwd=assets_dir) + + if os.path.exists(golden_dir): + execute('rm -f goldens/*', cwd=output_dir) + execute(f'cp filament-assets/{GOLDENS_DIR}/* goldens', cwd=output_dir) + +def push_goldens(output_dir, test_name, filter_func=lambda a:True): + for test in test_config.tests: + for backend in test_config.backends: + for model in test.models: + pass + +if __name__ == "__main__": + parser = ArgParseImpl() + parser.add_argument('--test', help='Configuration of the test', required=True) + parser.add_argument('--gltf_viewer', help='Path to the gltf_viewer', required=True) + parser.add_argument('--output_dir', help='Output Directory', required=True) + parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)') + parser.add_argument('--vk_icd', help='Path to VK ICD file') + + args, _ = parser.parse_known_args(sys.argv[1:]) + test = parse_test_config_from_path(args.test) + render_result = render_test(args.gltf_viewer, test, args.output_dir, opengl_lib=args.opengl_lib, vk_icd=args.vk_icd) + + failed = [f' {tname}' for tname, res in render_result if res != 0] + success_count = len(render_result) - len(failed ) + important_print(f'Successfully rendered {success_count} / {len(render_result)}' + + ('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else '')) diff --git a/test/renderdiff/utils.py b/test/renderdiff/src/utils.py similarity index 100% rename from test/renderdiff/utils.py rename to test/renderdiff/src/utils.py diff --git a/test/renderdiff/src/workflow_presubmit_msg.py b/test/renderdiff/src/workflow_presubmit_msg.py new file mode 100644 index 000000000000..06d8975364f1 --- /dev/null +++ b/test/renderdiff/src/workflow_presubmit_msg.py @@ -0,0 +1,46 @@ +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from utils import execute + +def get_last_commit(): + res, o = execute('git log -1') + commit, author, date, _, title, *desc = o.split('\n') + + desc = [l.strip() for l in desc[1:]] + if len(desc) > 0 and len(desc[0]) == 0: + desc = desc[1:] + + return ( + commit.split(' ')[1], + title.strip(), + desc) + +def sanitized_split(line, split_atom='\n'): + return list(filter(lambda x: len(x) > 0, map(lambda x: x.strip(), line.split(split_atom)))) + +RDIFF_UPDATE_GOLDEN_STR = 'RDIFF_UPDATE_GOLDEN' + +if __name__ == "__main__": + RE_STR = f'{RDIFF_UPDATE_GOLDEN_STR}(?:S)?=[\[]?([a-zA-Z0-9,\s]+)[\]]?' + + to_update = [] + commit, title, description = get_last_commit() + for line in description: + m = re.match(RE_STR, line) + if not m: + continue + + to_update += sanitize_split(m.group(1).replace(',', ' '), ' ') + print(','.join(to_update)) diff --git a/test/renderdiff_tests.sh b/test/renderdiff/test.sh similarity index 95% rename from test/renderdiff_tests.sh rename to test/renderdiff/test.sh index fc5b14fab048..7321a468ae04 100755 --- a/test/renderdiff_tests.sh +++ b/test/renderdiff/test.sh @@ -36,10 +36,8 @@ function prepare_mesa() { set -e && set -x && prepare_mesa && \ mkdir -p ${OUTPUT_DIR} && \ ./build.sh -X ${MESA_DIR} -p desktop debug gltf_viewer && \ - python3 ${RENDERDIFF_TEST_DIR}/run.py \ + python3 ${RENDERDIFF_TEST_DIR}/src/run.py \ --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \ --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \ --output_dir=${OUTPUT_DIR} \ --opengl_lib=${MESA_LIB_DIR} - -unset MESA_LIB_DIR diff --git a/test/renderdiff/tests/presubmit.json b/test/renderdiff/tests/presubmit.json index 78224d419ce9..761f043fc5da 100644 --- a/test/renderdiff/tests/presubmit.json +++ b/test/renderdiff/tests/presubmit.json @@ -1,10 +1,10 @@ { - "name": "PresubmitPixelTests", + "name": "presubmit", "backends": ["opengl"], "model_search_paths": ["third_party/models"], "presets": [ { - "name": "Standard", + "name": "base", "models": ["lucy", "DamagedHelmet"], "rendering": { "viewer.cameraFocusDistance": 0, @@ -16,7 +16,7 @@ { "name": "BloomFlare", "description": "Testing bloom and flare", - "apply_presets": ["Standard"], + "apply_presets": ["base"], "rendering": { "view.bloom.enabled": true, "view.bloom.lensFlare": true @@ -24,8 +24,8 @@ }, { "name": "MSAA", - "description": "Testing Multi-sample Anti-aliasing", - "apply_presets": ["Standard"], + "description": "Testing multisampling anti-aliasing", + "apply_presets": ["base"], "rendering": { "view.msaa.enabled": true } diff --git a/test/utils/get_mesa.sh b/test/utils/get_mesa.sh index 678753dadef9..b21481220971 100755 --- a/test/utils/get_mesa.sh +++ b/test/utils/get_mesa.sh @@ -14,15 +14,16 @@ #!/usr/bin/bash -set -e -set -x +set -xe CLANG_VERSION=`clang --version | head -n 1 | awk '{ print $4 }' | python3 -c "import sys; print(sys.stdin.read().split('.')[0])"` sudo apt-get -y build-dep mesa # Uninstall llvm-18 (and above) as they conflict with the mesa version we are compiling. +set +e sudo apt -y remove llvm-18 llvm-18-* +set -x sudo apt -y install clang-${CLANG_VERSION} \ libc++-${CLANG_VERSION}-dev \