From c43a9a665fdcd1cef23aa233c819587571b29443 Mon Sep 17 00:00:00 2001 From: Yan Pujante Date: Tue, 13 Feb 2024 17:03:30 -0800 Subject: [PATCH] Add suppport for external ports (#21316) --- ChangeLog.md | 2 + .../docs/compiling/Building-Projects.rst | 9 +++ test/other/ports/external.py | 62 ++++++++++++++++ test/other/ports/my_port.c | 3 + test/other/ports/my_port.h | 1 + test/other/ports/simple.py | 25 +++++++ test/other/test_external_ports.c | 33 +++++++++ test/other/test_external_ports_simple.c | 14 ++++ test/test_other.py | 35 +++++++-- tools/ports/__init__.py | 72 ++++++++++++------- 10 files changed, 225 insertions(+), 31 deletions(-) create mode 100644 test/other/ports/external.py create mode 100644 test/other/ports/my_port.c create mode 100644 test/other/ports/my_port.h create mode 100644 test/other/ports/simple.py create mode 100644 test/other/test_external_ports.c create mode 100644 test/other/test_external_ports_simple.c diff --git a/ChangeLog.md b/ChangeLog.md index 6581cbf87d4ef..a983f8de66b4f 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -42,6 +42,8 @@ See docs/process.md for more on how version tagging works. available via `--use-port=contrib.glfw3`: an emscripten port of glfw written in C++ with many features like support for multiple windows. (#21244 and #21276) +- Added concept of external ports which live outside emscripten and are + loaded on demand using the syntax `--use-port=/path/to/my_port.py` (#21316) 3.1.53 - 01/29/24 diff --git a/site/source/docs/compiling/Building-Projects.rst b/site/source/docs/compiling/Building-Projects.rst index eddd406aae396..cbc3d26c91b17 100644 --- a/site/source/docs/compiling/Building-Projects.rst +++ b/site/source/docs/compiling/Building-Projects.rst @@ -278,6 +278,15 @@ The simplest way to add a new port is to put it under the ``contrib`` directory. * Make sure the port is open source and has a suitable license. * Read the ``README.md`` file under ``tools/ports/contrib`` which contains more information. +External ports +-------------- + +Emscripten also supports external ports (ports that are not part of the +distribution). In order to use such a port, you simply provide its path: +``--use-port=/path/to/my_port.py`` + +.. note:: Be aware that if you are working on the code of a port, the port API + used by emscripten is not 100% stable and could change between versions. Build system issues =================== diff --git a/test/other/ports/external.py b/test/other/ports/external.py new file mode 100644 index 0000000000000..0ca8eabc8b0f6 --- /dev/null +++ b/test/other/ports/external.py @@ -0,0 +1,62 @@ +# Copyright 2024 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import os +from typing import Dict, Optional + +OPTIONS = { + 'value1': 'Value for define TEST_VALUE_1', + 'value2': 'Value for define TEST_VALUE_2', + 'dependency': 'A dependency' +} + +# user options (from --use-port) +opts: Dict[str, Optional[str]] = { + 'value1': None, + 'value2': None, + 'dependency': None +} + +deps = [] + + +def get_lib_name(settings): + return 'lib_external.a' + + +def get(ports, settings, shared): + # for simplicity in testing, the source is in the same folder as the port and not fetched as a tarball + source_dir = os.path.dirname(__file__) + + def create(final): + ports.install_headers(source_dir) + print(f'about to build {source_dir}') + ports.build_port(source_dir, final, 'external') + + return [shared.cache.get_lib(get_lib_name(settings), create, what='port')] + + +def clear(ports, settings, shared): + shared.cache.erase_lib(get_lib_name(settings)) + + +def process_args(ports): + args = ['-isystem', ports.get_include_dir('external')] + if opts['value1']: + args.append(f'-DTEST_VALUE_1={opts["value1"]}') + if opts['value2']: + args.append(f'-DTEST_VALUE_2={opts["value2"]}') + if opts['dependency']: + args.append(f'-DTEST_DEPENDENCY_{opts["dependency"].upper()}') + return args + + +def process_dependencies(settings): + if opts['dependency']: + deps.append(opts['dependency']) + + +def handle_options(options): + opts.update(options) diff --git a/test/other/ports/my_port.c b/test/other/ports/my_port.c new file mode 100644 index 0000000000000..b54c9d5f30469 --- /dev/null +++ b/test/other/ports/my_port.c @@ -0,0 +1,3 @@ +int my_port_fn(int value) { + return value; +} diff --git a/test/other/ports/my_port.h b/test/other/ports/my_port.h new file mode 100644 index 0000000000000..2d8cc29b6b31a --- /dev/null +++ b/test/other/ports/my_port.h @@ -0,0 +1 @@ +int my_port_fn(int); diff --git a/test/other/ports/simple.py b/test/other/ports/simple.py new file mode 100644 index 0000000000000..e952c8014b8e8 --- /dev/null +++ b/test/other/ports/simple.py @@ -0,0 +1,25 @@ +# Copyright 2024 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import os + + +def get_lib_name(settings): + return 'lib_simple.a' + + +def get(ports, settings, shared): + # for simplicity in testing, the source is in the same folder as the port and not fetched as a tarball + source_dir = os.path.dirname(__file__) + + def create(final): + ports.install_headers(source_dir) + ports.build_port(source_dir, final, 'simple') + + return [shared.cache.get_lib(get_lib_name(settings), create, what='port')] + + +def clear(ports, settings, shared): + shared.cache.erase_lib(get_lib_name(settings)) diff --git a/test/other/test_external_ports.c b/test/other/test_external_ports.c new file mode 100644 index 0000000000000..3ef5f1787dd49 --- /dev/null +++ b/test/other/test_external_ports.c @@ -0,0 +1,33 @@ +/* + * Copyright 2024 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include +#include +#include + +#ifdef TEST_DEPENDENCY_SDL2 +#include +#endif + +// TEST_VALUE_1 and TEST_VALUE_2 are defined via port options +#ifndef TEST_VALUE_1 +#define TEST_VALUE_1 0 +#endif +#ifndef TEST_VALUE_2 +#define TEST_VALUE_2 0 +#endif + +int main() { + assert(my_port_fn(99) == 99); // check that we can call a function from my_port.h + printf("value1=%d&value2=%d\n", TEST_VALUE_1, TEST_VALUE_2); +#ifdef TEST_DEPENDENCY_SDL2 + SDL_version version; + SDL_VERSION(&version); + printf("sdl2=%d\n", version.major); +#endif + return 0; +} diff --git a/test/other/test_external_ports_simple.c b/test/other/test_external_ports_simple.c new file mode 100644 index 0000000000000..6c3fa366bd89d --- /dev/null +++ b/test/other/test_external_ports_simple.c @@ -0,0 +1,14 @@ +/* + * Copyright 2024 The Emscripten Authors. All rights reserved. + * Emscripten is available under two separate licenses, the MIT license and the + * University of Illinois/NCSA Open Source License. Both these licenses can be + * found in the LICENSE file. + */ + +#include +#include + +int main() { + assert(my_port_fn(99) == 99); // check that we can call a function from my_port.h + return 0; +} diff --git a/test/test_other.py b/test/test_other.py index 65b9a5ce5367d..77fce4d6e3795 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -2393,6 +2393,31 @@ def test_contrib_ports(self): # with a different contrib port when there is another one self.emcc(test_file('other/test_contrib_ports.cpp'), ['--use-port=contrib.glfw3']) + @crossplatform + def test_external_ports_simple(self): + if config.FROZEN_CACHE: + self.skipTest("test doesn't work with frozen cache") + simple_port_path = test_file("other/ports/simple.py") + self.do_runf('other/test_external_ports_simple.c', emcc_args=[f'--use-port={simple_port_path}']) + + @crossplatform + def test_external_ports(self): + if config.FROZEN_CACHE: + self.skipTest("test doesn't work with frozen cache") + external_port_path = test_file("other/ports/external.py") + # testing no option + self.do_runf('other/test_external_ports.c', 'value1=0&value2=0\n', emcc_args=[f'--use-port={external_port_path}']) + # testing 1 option + self.do_runf('other/test_external_ports.c', 'value1=12&value2=0\n', emcc_args=[f'--use-port={external_port_path}:value1=12']) + # testing 2 options + self.do_runf('other/test_external_ports.c', 'value1=12&value2=36\n', emcc_args=[f'--use-port={external_port_path}:value1=12:value2=36']) + # testing dependency + self.do_runf('other/test_external_ports.c', 'sdl2=2\n', emcc_args=[f'--use-port={external_port_path}:dependency=sdl2']) + # testing invalid dependency + stderr = self.expect_fail([EMCC, test_file('other/test_external_ports.c'), f'--use-port={external_port_path}:dependency=invalid', '-o', 'a4.out.js']) + self.assertFalse(os.path.exists('a4.out.js')) + self.assertContained('Unknown dependency `invalid` for port `external`', stderr) + def test_link_memcpy(self): # memcpy can show up *after* optimizations, so after our opportunity to link in libc, so it must be special-cased create_file('main.c', r''' @@ -14528,16 +14553,16 @@ def test_js_preprocess_pre_post(self): def test_use_port_errors(self, compiler): stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=invalid', '-o', 'out.js']) self.assertFalse(os.path.exists('out.js')) - self.assertContained('Error with --use-port=invalid | invalid port name: invalid', stderr) + self.assertContained('Error with `--use-port=invalid` | invalid port name: `invalid`', stderr) stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=sdl2:opt1=v1', '-o', 'out.js']) self.assertFalse(os.path.exists('out.js')) - self.assertContained('Error with --use-port=sdl2:opt1=v1 | no options available for port sdl2', stderr) + self.assertContained('Error with `--use-port=sdl2:opt1=v1` | no options available for port `sdl2`', stderr) stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=sdl2_image:format=jpg', '-o', 'out.js']) self.assertFalse(os.path.exists('out.js')) - self.assertContained('Error with --use-port=sdl2_image:format=jpg | format is not supported', stderr) + self.assertContained('Error with `--use-port=sdl2_image:format=jpg` | `format` is not supported', stderr) stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=sdl2_image:formats', '-o', 'out.js']) self.assertFalse(os.path.exists('out.js')) - self.assertContained('Error with --use-port=sdl2_image:formats | formats is missing a value', stderr) + self.assertContained('Error with `--use-port=sdl2_image:formats` | `formats` is missing a value', stderr) stderr = self.expect_fail([compiler, test_file('hello_world.c'), '--use-port=sdl2_image:formats=jpg:formats=png', '-o', 'out.js']) self.assertFalse(os.path.exists('out.js')) - self.assertContained('Error with --use-port=sdl2_image:formats=jpg:formats=png | duplicate option formats', stderr) + self.assertContained('Error with `--use-port=sdl2_image:formats=jpg:formats=png` | duplicate option `formats`', stderr) diff --git a/tools/ports/__init__.py b/tools/ports/__init__.py index dd84aebb3cdc0..18ce4dd995f29 100644 --- a/tools/ports/__init__.py +++ b/tools/ports/__init__.py @@ -8,6 +8,8 @@ import os import shutil import glob +import importlib.util +import sys from typing import Set from tools import cache from tools import config @@ -33,8 +35,7 @@ logger = logging.getLogger('ports') -def load_port(name): - port = __import__(name, globals(), level=1, fromlist=[None]) +def init_port(name, port): ports.append(port) port.is_contrib = name.startswith('contrib.') port.name = name @@ -60,9 +61,29 @@ def load_port(name): for variant, extra_settings in port.variants.items(): if variant in port_variants: - utils.exit_with_error('duplicate port variant: %s' % variant) + utils.exit_with_error('duplicate port variant: `%s`' % variant) port_variants[variant] = (port.name, extra_settings) + validate_port(port) + + +def load_port_by_name(name): + port = __import__(name, globals(), level=1, fromlist=[None]) + init_port(name, port) + + +def load_port_by_path(path): + name = os.path.splitext(os.path.basename(path))[0] + if name in ports_by_name: + utils.exit_with_error(f'port path [`{path}`] is invalid: duplicate port name `{name}`') + module_name = f'tools.ports.{name}' + spec = importlib.util.spec_from_file_location(module_name, path) + port = importlib.util.module_from_spec(spec) + sys.modules[module_name] = port + spec.loader.exec_module(port) + init_port(name, port) + return name + def validate_port(port): expected_attrs = ['get', 'clear', 'show'] @@ -74,30 +95,20 @@ def validate_port(port): assert hasattr(port, a), 'port %s is missing %s' % (port, a) -def validate_ports(): - for port in ports: - validate_port(port) - for dep in port.deps: - if dep not in ports_by_name: - utils.exit_with_error('unknown dependency in port: %s' % dep) - - @ToolchainProfiler.profile() def read_ports(): for filename in os.listdir(ports_dir): if not filename.endswith('.py') or filename == '__init__.py': continue filename = os.path.splitext(filename)[0] - load_port(filename) + load_port_by_name(filename) contrib_dir = os.path.join(ports_dir, 'contrib') for filename in os.listdir(contrib_dir): if not filename.endswith('.py') or filename == '__init__.py': continue filename = os.path.splitext(filename)[0] - load_port('contrib.' + filename) - - validate_ports() + load_port_by_name('contrib.' + filename) def get_all_files_under(dirname): @@ -386,6 +397,8 @@ def resolve_dependencies(port_set, settings): def add_deps(node): node.process_dependencies(settings) for d in node.deps: + if d not in ports_by_name: + utils.exit_with_error(f'Unknown dependency `{d}` for port `{node.name}`') dep = ports_by_name[d] if dep not in port_set: port_set.add(dep) @@ -396,31 +409,38 @@ def add_deps(node): def handle_use_port_error(arg, message): - utils.exit_with_error(f'Error with --use-port={arg} | {message}') + utils.exit_with_error(f'Error with `--use-port={arg}` | {message}') def handle_use_port_arg(settings, arg): - args = arg.split(':', 1) - name, options = args[0], None - if len(args) == 2: - options = args[1] - if name not in ports_by_name: - handle_use_port_error(arg, f'invalid port name: {name}') + # Ignore ':' in first or second char of string since we could be dealing with a windows drive separator + pos = arg.find(':', 2) + if pos != -1: + name, options = arg[:pos], arg[pos + 1:] + else: + name, options = arg, None + if name.endswith('.py'): + port_file_path = name + if not os.path.isfile(port_file_path): + handle_use_port_error(arg, f'not a valid port path: {port_file_path}') + name = load_port_by_path(port_file_path) + elif name not in ports_by_name: + handle_use_port_error(arg, f'invalid port name: `{name}`') ports_needed.add(name) if options: port = ports_by_name[name] if not hasattr(port, 'handle_options'): - handle_use_port_error(arg, f'no options available for port {name}') + handle_use_port_error(arg, f'no options available for port `{name}`') else: options_dict = {} for name_value in options.split(':'): nv = name_value.split('=', 1) if len(nv) != 2: - handle_use_port_error(arg, f'{name_value} is missing a value') + handle_use_port_error(arg, f'`{name_value}` is missing a value') if nv[0] not in port.OPTIONS: - handle_use_port_error(arg, f'{nv[0]} is not supported; available options are {port.OPTIONS}') + handle_use_port_error(arg, f'`{nv[0]}` is not supported; available options are {port.OPTIONS}') if nv[0] in options_dict: - handle_use_port_error(arg, f'duplicate option {nv[0]}') + handle_use_port_error(arg, f'duplicate option `{nv[0]}`') options_dict[nv[0]] = nv[1] port.handle_options(options_dict)