From 499285f4328a5226586a2b04ed08a023faa50ffe Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 2 Feb 2023 10:10:54 -0600 Subject: [PATCH 01/25] Fix typos in documentation --- src/cjdk/__main__.py | 4 ++-- src/cjdk/_index.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cjdk/__main__.py b/src/cjdk/__main__.py index c49dd62..edc8632 100644 --- a/src/cjdk/__main__.py +++ b/src/cjdk/__main__.py @@ -71,7 +71,7 @@ def cache_jdk(ctx): Download and extract the requested JDK if it is not already cached. Usually there is no need to invoke this command on its own, but it may be - useful if you want any potentil JDK download to happen at a controlled + useful if you want any potential JDK download to happen at a controlled point in time. See 'cjdk --help' for the common options used to specify the JDK and how it @@ -108,7 +108,7 @@ def exec(ctx, prog, args): """ Run PROG with the environment variables set for the requested JDK. - The JDK is download if not already cached. + The JDK is downloaded if not already cached. See 'cjdk --help' for the common options used to specify the JDK and how it is obtained. diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index 1a10198..98c404d 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -56,7 +56,7 @@ def resolve_jdk_version(index, conf: Configuration): Find in index the exact JDK version for the given configuration. Arguments: - index -- The JDK index (dested dict) + index -- The JDK index (nested dict) """ jdks = available_jdks(index, conf) versions = [i[1] for i in jdks if i[0] == conf.vendor] From 68e3fffc010081aafd1074809f4600b30f72a857 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 2 Feb 2023 11:18:21 -0600 Subject: [PATCH 02/25] Add function to return all matching versions And simplify the existing _match_version function by delegating to the new _match_versions. Technically this is slower time-complexity-wise, but it's also more Pythonic, with performance unlikely to matter for the number of available JDKs in the index. --- src/cjdk/_index.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index 98c404d..5be0cd5 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -109,7 +109,8 @@ def _read_index(path): return json.load(infile) -def _match_version(vendor, candidates, requested): +def _match_versions(vendor, candidates, requested): + # Find all candidates compatible with the request is_graal = "graalvm" in vendor.lower() normreq = _normalize_version(requested, remove_prefix_1=not is_graal) normcands = {} @@ -126,14 +127,19 @@ def _match_version(vendor, candidates, requested): continue # Skip any non-numeric versions (not expected) normcands[normcand] = candidate - # Find the newest candidate compatible with the request - for normcand in sorted(normcands.keys(), reverse=True): - if _is_version_compatible_with_spec(normcand, normreq): - return normcands[normcand] - if normcand > normreq: - continue - break - raise LookupError(f"No matching version for '{vendor}:{requested}'") + return { + k: v for k, v in normcands.items() + if _is_version_compatible_with_spec(k, normreq) + } + + +def _match_version(vendor, candidates, requested): + matched = _match_versions(vendor, candidates, requested) + + if len(matched) == 0: + raise LookupError(f"No matching version for '{vendor}:{requested}'") + + return matched[max(matched)] _VER_SEPS = re.compile(r"[.-]") From 4cd63010f0e608d28ac3f6d8dc501bcb63bde4ad Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 2 Feb 2023 11:20:22 -0600 Subject: [PATCH 03/25] Add a command to list matching JDKs It's helpful to be able to query the index to know what is available. --- src/cjdk/__init__.py | 3 ++- src/cjdk/__main__.py | 12 ++++++++++++ src/cjdk/_api.py | 45 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/cjdk/__init__.py b/src/cjdk/__init__.py index 1773436..3be31e1 100644 --- a/src/cjdk/__init__.py +++ b/src/cjdk/__init__.py @@ -2,11 +2,12 @@ # Copyright 2022 Board of Regents of the University of Wisconsin System # SPDX-License-Identifier: MIT -from ._api import cache_file, cache_jdk, cache_package, java_env, java_home +from ._api import cache_file, cache_jdk, cache_package, java_env, java_home, list_jdks from ._version import __version__ as __version__ __all__ = [ "cache_file", + "list_jdks", "cache_jdk", "cache_package", "java_env", diff --git a/src/cjdk/__main__.py b/src/cjdk/__main__.py index edc8632..18f1945 100644 --- a/src/cjdk/__main__.py +++ b/src/cjdk/__main__.py @@ -64,6 +64,17 @@ def main(ctx, jdk, cache_dir, index_url, index_ttl, os, arch, progress): ) +@click.command(short_help="List available JDKs matching criteria.") +@click.pass_context +def list_jdks(ctx): + """ + Output a list of JDKs matching the given criteria. + + See 'cjdk --help' for the common options used to specify the JDK. + """ + _api.list_jdks(**ctx.obj) + + @click.command(short_help="Ensure the requested JDK is cached.") @click.pass_context def cache_jdk(ctx): @@ -223,6 +234,7 @@ def cache_package(ctx, url, name, sha1, sha256, sha512): main.add_command(java_home) main.add_command(exec) +main.add_command(list_jdks) main.add_command(cache_jdk) main.add_command(cache_file) main.add_command(cache_package) diff --git a/src/cjdk/_api.py b/src/cjdk/_api.py index ba7f9da..541e164 100644 --- a/src/cjdk/_api.py +++ b/src/cjdk/_api.py @@ -6,9 +6,10 @@ import os from contextlib import contextmanager -from . import _conf, _install, _jdk +from . import _conf, _install, _index, _jdk __all__ = [ + "list_jdks", "cache_jdk", "java_env", "java_home", @@ -17,6 +18,48 @@ ] +def list_jdks(*, vendor=None, version=None, **kwargs): + """ + Output a list of JDKs matching the given criteria. + + Parameters + ---------- + vendor : str, optional + JDK vendor name, such as "adoptium". + version : str, optional + JDK version expression, such as "17+". + + Other Parameters + ---------------- + jdk : str, optional + JDK vendor and version, such as "adoptium:17+". Cannot be specified + together with `vendor` or `version`. + cache_dir : pathlib.Path or str, optional + Override the root cache directory. + index_url : str, optional + Alternative URL for the JDK index. + os : str, optional + Operating system for the JDK (default: current operating system). + arch : str, optional + CPU architecture for the JDK (default: current architecture). + + Returns + ------- + None + """ + conf = _conf.configure(vendor=vendor, version=version, **kwargs) + # TODO: Eliminate code duplication with _index.resolve_jdk_version. + jdks = _index.available_jdks(_index.jdk_index(conf), conf) + versions = [i[1] for i in jdks if i[0] == conf.vendor] + if not versions: + raise KeyError( + f"No {conf.vendor} JDK is available for {conf.os}-{conf.arch}" + ) + matched = _index._match_versions(conf.vendor, versions, conf.version) + print(f"[{conf.vendor}]") + print(os.linesep.join([v for _, v in sorted(matched.items())])) + + def cache_jdk(*, vendor=None, version=None, **kwargs): """ Download and extract the given JDK if it is not already cached. From ee3b519b638fd7153207989d57df16186fc45a40 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 2 Feb 2023 11:55:09 -0600 Subject: [PATCH 04/25] Add a command to list available vendors --- src/cjdk/__init__.py | 3 ++- src/cjdk/__main__.py | 10 ++++++++++ src/cjdk/_api.py | 29 +++++++++++++++++++++++++++++ src/cjdk/_index.py | 12 ++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/cjdk/__init__.py b/src/cjdk/__init__.py index 3be31e1..179e924 100644 --- a/src/cjdk/__init__.py +++ b/src/cjdk/__init__.py @@ -2,11 +2,12 @@ # Copyright 2022 Board of Regents of the University of Wisconsin System # SPDX-License-Identifier: MIT -from ._api import cache_file, cache_jdk, cache_package, java_env, java_home, list_jdks +from ._api import cache_file, cache_jdk, cache_package, java_env, java_home, list_jdks, list_vendors from ._version import __version__ as __version__ __all__ = [ "cache_file", + "list_vendors", "list_jdks", "cache_jdk", "cache_package", diff --git a/src/cjdk/__main__.py b/src/cjdk/__main__.py index 18f1945..0e3469d 100644 --- a/src/cjdk/__main__.py +++ b/src/cjdk/__main__.py @@ -64,6 +64,15 @@ def main(ctx, jdk, cache_dir, index_url, index_ttl, os, arch, progress): ) +@click.command(short_help="List available JDK vendors.") +@click.pass_context +def list_vendors(ctx): + """ + Output the list of available vendors. + """ + _api.list_vendors(**ctx.obj) + + @click.command(short_help="List available JDKs matching criteria.") @click.pass_context def list_jdks(ctx): @@ -234,6 +243,7 @@ def cache_package(ctx, url, name, sha1, sha256, sha512): main.add_command(java_home) main.add_command(exec) +main.add_command(list_vendors) main.add_command(list_jdks) main.add_command(cache_jdk) main.add_command(cache_file) diff --git a/src/cjdk/_api.py b/src/cjdk/_api.py index 541e164..6d0c8f7 100644 --- a/src/cjdk/_api.py +++ b/src/cjdk/_api.py @@ -9,6 +9,7 @@ from . import _conf, _install, _index, _jdk __all__ = [ + "list_vendors", "list_jdks", "cache_jdk", "java_env", @@ -18,6 +19,34 @@ ] +def list_vendors(**kwargs): + """ + Output the list of available vendors. + + Parameters + ---------- + None + + Other Parameters + ---------------- + index_url : str, optional + Alternative URL for the JDK index. + + Returns + ------- + None + """ + conf = _conf.configure(**kwargs) + index = _index.jdk_index(conf) + vendors = { + vendor.replace("jdk@", "") + for osys in index + for arch in index[osys] + for vendor in index[osys][arch].keys() + } + print(os.linesep.join(sorted(vendors))) + + def list_jdks(*, vendor=None, version=None, **kwargs): """ Output a list of JDKs matching the given criteria. diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index 5be0cd5..ce8dc88 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -29,6 +29,18 @@ def jdk_index(conf: Configuration): return _read_index(_cached_index(conf)) +def available_vendors(index): + """ + Find in index the available JDK vendors. + + A set of strings is returned. + + Arguments: + index -- The JDK index (nested dict) + """ + return set(index[os][arch] for os in index for arch in index[os]) + + def available_jdks(index, conf: Configuration): """ Find in index the available JDK vendor-version combinations. From e94730e10b4a3c98131359495a7bc2848bb916e5 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 06:43:40 -0500 Subject: [PATCH 05/25] Rename the listing commands * list-jdks -> ls * list-vendors -> ls-vendors --- src/cjdk/__main__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cjdk/__main__.py b/src/cjdk/__main__.py index 0e3469d..d8e6ee2 100644 --- a/src/cjdk/__main__.py +++ b/src/cjdk/__main__.py @@ -66,7 +66,7 @@ def main(ctx, jdk, cache_dir, index_url, index_ttl, os, arch, progress): @click.command(short_help="List available JDK vendors.") @click.pass_context -def list_vendors(ctx): +def ls_vendors(ctx): """ Output the list of available vendors. """ @@ -75,7 +75,7 @@ def list_vendors(ctx): @click.command(short_help="List available JDKs matching criteria.") @click.pass_context -def list_jdks(ctx): +def ls(ctx): """ Output a list of JDKs matching the given criteria. @@ -243,8 +243,8 @@ def cache_package(ctx, url, name, sha1, sha256, sha512): main.add_command(java_home) main.add_command(exec) -main.add_command(list_vendors) -main.add_command(list_jdks) +main.add_command(ls_vendors) +main.add_command(ls) main.add_command(cache_jdk) main.add_command(cache_file) main.add_command(cache_package) From fb15d4bf7e7573aa80ae83251ee1591820b70f67 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 06:54:32 -0500 Subject: [PATCH 06/25] Rename cache-jdk command to cache --- src/cjdk/__main__.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/cjdk/__main__.py b/src/cjdk/__main__.py index d8e6ee2..53d87a2 100644 --- a/src/cjdk/__main__.py +++ b/src/cjdk/__main__.py @@ -86,7 +86,7 @@ def ls(ctx): @click.command(short_help="Ensure the requested JDK is cached.") @click.pass_context -def cache_jdk(ctx): +def cache(ctx): """ Download and extract the requested JDK if it is not already cached. @@ -100,6 +100,15 @@ def cache_jdk(ctx): _api.cache_jdk(**ctx.obj) +@click.command(hidden=True) +@click.pass_context +def cache_jdk(ctx): + """ + Deprecated. Use cache function instead. + """ + _api.cache_jdk(**ctx.obj) + + @click.command( short_help="Print the Java home directory for the requested JDK." ) @@ -241,14 +250,17 @@ def cache_package(ctx, url, name, sha1, sha256, sha512): ) +# Register current commands. main.add_command(java_home) main.add_command(exec) main.add_command(ls_vendors) main.add_command(ls) -main.add_command(cache_jdk) +main.add_command(cache) main.add_command(cache_file) main.add_command(cache_package) +# Register hidden/deprecated commands, for backwards compatibility. +main.add_command(cache_jdk) if __name__ == "__main__": main() From b0a3f9c8c4930cebc2088c56d2c6556d18cc07f0 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 07:22:40 -0500 Subject: [PATCH 07/25] Add type alias for the JDK index dict And add some type hints around adjacent functions, too. And rename _cached_index function to _cached_index_path because it returns a Path, not an Index dict. --- src/cjdk/_cache.py | 8 ++++---- src/cjdk/_index.py | 30 +++++++++++++++++++----------- src/cjdk/_install.py | 6 ++++-- tests/test_index.py | 6 +++--- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/cjdk/_cache.py b/src/cjdk/_cache.py index 458be91..1e9bc59 100644 --- a/src/cjdk/_cache.py +++ b/src/cjdk/_cache.py @@ -59,7 +59,7 @@ def atomic_file( ttl, timeout_for_fetch_elsewhere=10, timeout_for_read_elsewhere=2.5, -): +) -> Path: """ Retrieve cached file for key, fetching with fetchfunc if necessary. @@ -150,7 +150,7 @@ def permanent_directory( return keydir -def _file_exists_and_is_fresh(file, ttl): +def _file_exists_and_is_fresh(file, ttl) -> bool: if not file.is_file(): return False now = time.time() @@ -185,11 +185,11 @@ def _create_key_tmpdir(cache_dir, key): shutil.rmtree(tmpdir) -def _key_directory(cache_dir, key): +def _key_directory(cache_dir: Path, key) -> Path: return cache_dir / "v0" / Path(*key) -def _key_tmpdir(cache_dir, key): +def _key_tmpdir(cache_dir: Path, key) -> Path: return cache_dir / "v0" / Path("fetching", *key) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index ce8dc88..7907b51 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -6,6 +6,8 @@ import json import re import warnings +from pathlib import Path +from typing import Dict, List, Tuple from . import _install from ._conf import Configuration @@ -22,14 +24,21 @@ _INDEX_FILENAME = "jdk-index.json" -def jdk_index(conf: Configuration): +# Type alias declarations. +Versions = Dict[str, str] # key = version, value = archive URL +Vendors = Dict[str, Versions] # key = vendor name +Arches = Dict[str, Vendors] # key = arch name +Index = Dict[str, Arches] # key = os name + + +def jdk_index(conf: Configuration) -> Index: """ Get the JDK index, from cache if possible. """ - return _read_index(_cached_index(conf)) + return _read_index(_cached_index_path(conf)) -def available_vendors(index): +def available_vendors(index: Index): """ Find in index the available JDK vendors. @@ -41,7 +50,7 @@ def available_vendors(index): return set(index[os][arch] for os in index for arch in index[os]) -def available_jdks(index, conf: Configuration): +def available_jdks(index: Index, conf: Configuration) -> Tuple[str, str]: """ Find in index the available JDK vendor-version combinations. @@ -51,8 +60,7 @@ def available_jdks(index, conf: Configuration): index -- The JDK index (nested dict) """ try: - # jdks is dict: vendor -> (version -> url) - jdks = index[conf.os][conf.arch] + jdks: Vendors = index[conf.os][conf.arch] except KeyError: return [] @@ -63,7 +71,7 @@ def available_jdks(index, conf: Configuration): ) -def resolve_jdk_version(index, conf: Configuration): +def resolve_jdk_version(index: Index, conf: Configuration): """ Find in index the exact JDK version for the given configuration. @@ -84,7 +92,7 @@ def resolve_jdk_version(index, conf: Configuration): return matched -def jdk_url(index, conf: Configuration): +def jdk_url(index: Index, conf: Configuration): """ Find in index the URL for the JDK binary for the given vendor and version. @@ -97,7 +105,7 @@ def jdk_url(index, conf: Configuration): return index[conf.os][conf.arch][f"jdk@{conf.vendor}"][matched] -def _cached_index(conf: Configuration): +def _cached_index_path(conf: Configuration) -> Path: def check_index(path): # Ensure valid JSON. _read_index(path) @@ -116,12 +124,12 @@ def check_index(path): ) -def _read_index(path): +def _read_index(path: Path) -> Index: with open(path, encoding="ascii") as infile: return json.load(infile) -def _match_versions(vendor, candidates, requested): +def _match_versions(vendor, candidates: List[str], requested): # Find all candidates compatible with the request is_graal = "graalvm" in vendor.lower() normreq = _normalize_version(requested, remove_prefix_1=not is_graal) diff --git a/src/cjdk/_install.py b/src/cjdk/_install.py index 412a4d9..de9b816 100644 --- a/src/cjdk/_install.py +++ b/src/cjdk/_install.py @@ -4,6 +4,8 @@ import sys +from pathlib import Path + from . import _cache, _download __all__ = [ @@ -12,7 +14,7 @@ ] -def install_file(prefix, name, url, filename, conf, *, ttl, checkfunc=None): +def install_file(prefix, name, url, filename, conf, *, ttl, checkfunc=None) -> Path: def fetch(dest): _print_progress_header(conf, name) _download.download_file( @@ -33,7 +35,7 @@ def fetch(dest): ) -def install_dir(prefix, name, url, conf, *, checkfunc=None): +def install_dir(prefix, name, url, conf, *, checkfunc=None) -> Path: def fetch(destdir): _print_progress_header(conf, name) _download.download_and_extract( diff --git a/tests/test_index.py b/tests/test_index.py index 1ab089a..9e564dd 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -89,7 +89,7 @@ def test_jdk_url(): ) -def test_cached_index(tmp_path): +def test_cached_index_path(tmp_path): with mock_server.start( endpoint="/index.json", data={"hello": "world"} ) as server: @@ -101,7 +101,7 @@ def test_cached_index(tmp_path): / _cache._key_for_url(url) / _index._INDEX_FILENAME ) - path = _index._cached_index( + path = _index._cached_index_path( configure( index_url=url, cache_dir=tmp_path, @@ -116,7 +116,7 @@ def test_cached_index(tmp_path): # Second time should return same data without fetching. assert server.request_count() == 1 - path2 = _index._cached_index( + path2 = _index._cached_index_path( configure( index_url=url, cache_dir=tmp_path, From 3da11f4f2296e6975b212b2ea6a6bb6c19a836d8 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 07:58:31 -0500 Subject: [PATCH 08/25] Eliminate resolve_jdk_version code duplication --- src/cjdk/_api.py | 7 +------ src/cjdk/_index.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/cjdk/_api.py b/src/cjdk/_api.py index 6d0c8f7..d9e109a 100644 --- a/src/cjdk/_api.py +++ b/src/cjdk/_api.py @@ -77,13 +77,8 @@ def list_jdks(*, vendor=None, version=None, **kwargs): None """ conf = _conf.configure(vendor=vendor, version=version, **kwargs) - # TODO: Eliminate code duplication with _index.resolve_jdk_version. jdks = _index.available_jdks(_index.jdk_index(conf), conf) - versions = [i[1] for i in jdks if i[0] == conf.vendor] - if not versions: - raise KeyError( - f"No {conf.vendor} JDK is available for {conf.os}-{conf.arch}" - ) + versions = _index._get_versions(jdks, conf) matched = _index._match_versions(conf.vendor, versions, conf.version) print(f"[{conf.vendor}]") print(os.linesep.join([v for _, v in sorted(matched.items())])) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index 7907b51..6b989ac 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -79,17 +79,8 @@ def resolve_jdk_version(index: Index, conf: Configuration): index -- The JDK index (nested dict) """ jdks = available_jdks(index, conf) - versions = [i[1] for i in jdks if i[0] == conf.vendor] - if not versions: - raise KeyError( - f"No {conf.vendor} JDK is available for {conf.os}-{conf.arch}" - ) - matched = _match_version(conf.vendor, versions, conf.version) - if not matched: - raise KeyError( - f"No JDK matching version {conf.version} for {conf.os}-{conf.arch}-{conf.vendor}" - ) - return matched + versions = _get_versions(jdks, conf) + return _match_version(conf.vendor, versions, conf.version) def jdk_url(index: Index, conf: Configuration): @@ -129,6 +120,15 @@ def _read_index(path: Path) -> Index: return json.load(infile) +def _get_versions(jdks: Tuple[str, str], conf) -> List[str]: + versions = [i[1] for i in jdks if i[0] == conf.vendor] + if not versions: + raise KeyError( + f"No {conf.vendor} JDK is available for {conf.os}-{conf.arch}" + ) + return versions + + def _match_versions(vendor, candidates: List[str], requested): # Find all candidates compatible with the request is_graal = "graalvm" in vendor.lower() From cfc613e9da90ea9934c8b28ef8331186f1f9b7b1 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 07:59:08 -0500 Subject: [PATCH 09/25] Constrain ls to already-cached JDKs by default And let `ls --available` list all matches from the index. --- src/cjdk/__main__.py | 11 ++++++++--- src/cjdk/_api.py | 24 +++++++++++++++++++++--- src/cjdk/_index.py | 14 ++++++++------ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/cjdk/__main__.py b/src/cjdk/__main__.py index 53d87a2..9595423 100644 --- a/src/cjdk/__main__.py +++ b/src/cjdk/__main__.py @@ -73,15 +73,20 @@ def ls_vendors(ctx): _api.list_vendors(**ctx.obj) -@click.command(short_help="List available JDKs matching criteria.") +@click.command(short_help="List cached or available JDKs matching criteria.") @click.pass_context -def ls(ctx): +@click.option( + "--cached/--available", + default=True, + help="Show only already-cached JDKs, or show all available JDKs from the index (default cached only).", +) +def ls(ctx, cached: bool = False): """ Output a list of JDKs matching the given criteria. See 'cjdk --help' for the common options used to specify the JDK. """ - _api.list_jdks(**ctx.obj) + _api.list_jdks(**ctx.obj, cached_only=cached) @click.command(short_help="Ensure the requested JDK is cached.") diff --git a/src/cjdk/_api.py b/src/cjdk/_api.py index d9e109a..a39cd10 100644 --- a/src/cjdk/_api.py +++ b/src/cjdk/_api.py @@ -6,7 +6,7 @@ import os from contextlib import contextmanager -from . import _conf, _install, _index, _jdk +from . import _cache, _conf, _install, _index, _jdk __all__ = [ "list_vendors", @@ -47,7 +47,7 @@ def list_vendors(**kwargs): print(os.linesep.join(sorted(vendors))) -def list_jdks(*, vendor=None, version=None, **kwargs): +def list_jdks(*, vendor=None, version=None, cached_only=True, **kwargs): """ Output a list of JDKs matching the given criteria. @@ -57,6 +57,9 @@ def list_jdks(*, vendor=None, version=None, **kwargs): JDK vendor name, such as "adoptium". version : str, optional JDK version expression, such as "17+". + cached_only : bool, optional + If True, list only already-cached JDKs. + If False, list all matching JDKs in the index. Other Parameters ---------------- @@ -77,9 +80,24 @@ def list_jdks(*, vendor=None, version=None, **kwargs): None """ conf = _conf.configure(vendor=vendor, version=version, **kwargs) - jdks = _index.available_jdks(_index.jdk_index(conf), conf) + index = _index.jdk_index(conf) + jdks = _index.available_jdks(index, conf) versions = _index._get_versions(jdks, conf) matched = _index._match_versions(conf.vendor, versions, conf.version) + + if cached_only: + # Filter matches by existing key directories. + def is_cached(v): + url = _index.jdk_url(index, conf, v) + key = (_jdk._JDK_KEY_PREFIX, _cache._key_for_url(url)) + keydir = _cache._key_directory(conf.cache_dir, key) + return keydir.exists() + matched = { + k: v + for k, v in matched.items() + if is_cached(v) + } + print(f"[{conf.vendor}]") print(os.linesep.join([v for _, v in sorted(matched.items())])) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index 6b989ac..da3632c 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -71,7 +71,7 @@ def available_jdks(index: Index, conf: Configuration) -> Tuple[str, str]: ) -def resolve_jdk_version(index: Index, conf: Configuration): +def resolve_jdk_version(index: Index, conf: Configuration) -> str: """ Find in index the exact JDK version for the given configuration. @@ -83,7 +83,7 @@ def resolve_jdk_version(index: Index, conf: Configuration): return _match_version(conf.vendor, versions, conf.version) -def jdk_url(index: Index, conf: Configuration): +def jdk_url(index: Index, conf: Configuration, exact_version: str = None) -> str: """ Find in index the URL for the JDK binary for the given vendor and version. @@ -91,9 +91,11 @@ def jdk_url(index: Index, conf: Configuration): Arguments: index -- The JDK index (nested dict) + exact_version (optional) -- The JDK version, or None to resolve it from the configuration """ - matched = resolve_jdk_version(index, conf) - return index[conf.os][conf.arch][f"jdk@{conf.vendor}"][matched] + if exact_version is None: + exact_version = resolve_jdk_version(index, conf) + return index[conf.os][conf.arch][f"jdk@{conf.vendor}"][exact_version] def _cached_index_path(conf: Configuration) -> Path: @@ -129,7 +131,7 @@ def _get_versions(jdks: Tuple[str, str], conf) -> List[str]: return versions -def _match_versions(vendor, candidates: List[str], requested): +def _match_versions(vendor, candidates: List[str], requested) -> Dict[Tuple[int], str]: # Find all candidates compatible with the request is_graal = "graalvm" in vendor.lower() normreq = _normalize_version(requested, remove_prefix_1=not is_graal) @@ -153,7 +155,7 @@ def _match_versions(vendor, candidates: List[str], requested): } -def _match_version(vendor, candidates, requested): +def _match_version(vendor, candidates: List[str], requested) -> str: matched = _match_versions(vendor, candidates, requested) if len(matched) == 0: From 5461c0f794b8454aae436f64e3cced3c0ca67ad3 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 10:45:11 -0500 Subject: [PATCH 10/25] Merge the ibm-semuru-openj9-java<##> vendors --- src/cjdk/_index.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index da3632c..265e25e 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -119,7 +119,32 @@ def check_index(path): def _read_index(path: Path) -> Index: with open(path, encoding="ascii") as infile: - return json.load(infile) + index = json.load(infile) + + # Post-process the index to normalize the data. + # Some "vendors" include the major Java version, + # so let's merge such entries. In particular: + # + # * ibm-semuru-openj9-java<##> + # * graalvm-java<##> + # + # However: while the graalvm vendors follow this pattern, the version + # numbers for graalvm are *not* JDK versions, but rather GraalVM versions, + # which merely strongly resemble JDK version strings. For example, + # graalvm-java17 version 22.3.3 bundles OpenJDK 17.0.8, but + # unfortunately there is no way to know this from the index alone. + + pattern = re.compile('^(jdk@ibm-semeru.*)-java\\d+$') + for os, arches in index.items(): + for arch, vendors in arches.items(): + for vendor, versions in vendors.copy().items(): + if not vendor.startswith("jdk@graalvm") and (m := pattern.match(vendor)): + true_vendor = m.group(1) + if not true_vendor in index[os][arch]: + index[os][arch][true_vendor] = {} + index[os][arch][true_vendor].update(versions) + + return index def _get_versions(jdks: Tuple[str, str], conf) -> List[str]: From b84b13cb1b671aea93af58914d83877260d7158d Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 13:20:09 -0500 Subject: [PATCH 11/25] Ruff up the code --- src/cjdk/__init__.py | 10 +++++++++- src/cjdk/_api.py | 11 ++++------- src/cjdk/_index.py | 34 ++++++++++++++++++++-------------- src/cjdk/_install.py | 5 +++-- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/src/cjdk/__init__.py b/src/cjdk/__init__.py index 179e924..6d2696a 100644 --- a/src/cjdk/__init__.py +++ b/src/cjdk/__init__.py @@ -2,7 +2,15 @@ # Copyright 2022 Board of Regents of the University of Wisconsin System # SPDX-License-Identifier: MIT -from ._api import cache_file, cache_jdk, cache_package, java_env, java_home, list_jdks, list_vendors +from ._api import ( + cache_file, + cache_jdk, + cache_package, + java_env, + java_home, + list_jdks, + list_vendors, +) from ._version import __version__ as __version__ __all__ = [ diff --git a/src/cjdk/_api.py b/src/cjdk/_api.py index a39cd10..423538e 100644 --- a/src/cjdk/_api.py +++ b/src/cjdk/_api.py @@ -6,7 +6,7 @@ import os from contextlib import contextmanager -from . import _cache, _conf, _install, _index, _jdk +from . import _cache, _conf, _index, _install, _jdk __all__ = [ "list_vendors", @@ -42,7 +42,7 @@ def list_vendors(**kwargs): vendor.replace("jdk@", "") for osys in index for arch in index[osys] - for vendor in index[osys][arch].keys() + for vendor in index[osys][arch] } print(os.linesep.join(sorted(vendors))) @@ -92,11 +92,8 @@ def is_cached(v): key = (_jdk._JDK_KEY_PREFIX, _cache._key_for_url(url)) keydir = _cache._key_directory(conf.cache_dir, key) return keydir.exists() - matched = { - k: v - for k, v in matched.items() - if is_cached(v) - } + + matched = {k: v for k, v in matched.items() if is_cached(v)} print(f"[{conf.vendor}]") print(os.linesep.join([v for _, v in sorted(matched.items())])) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index 265e25e..0a95a7b 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -7,7 +7,6 @@ import re import warnings from pathlib import Path -from typing import Dict, List, Tuple from . import _install from ._conf import Configuration @@ -25,10 +24,10 @@ # Type alias declarations. -Versions = Dict[str, str] # key = version, value = archive URL -Vendors = Dict[str, Versions] # key = vendor name -Arches = Dict[str, Vendors] # key = arch name -Index = Dict[str, Arches] # key = os name +Versions = dict[str, str] # key = version, value = archive URL +Vendors = dict[str, Versions] # key = vendor name +Arches = dict[str, Vendors] # key = arch name +Index = dict[str, Arches] # key = os name def jdk_index(conf: Configuration) -> Index: @@ -50,7 +49,7 @@ def available_vendors(index: Index): return set(index[os][arch] for os in index for arch in index[os]) -def available_jdks(index: Index, conf: Configuration) -> Tuple[str, str]: +def available_jdks(index: Index, conf: Configuration) -> tuple[str, str]: """ Find in index the available JDK vendor-version combinations. @@ -83,7 +82,9 @@ def resolve_jdk_version(index: Index, conf: Configuration) -> str: return _match_version(conf.vendor, versions, conf.version) -def jdk_url(index: Index, conf: Configuration, exact_version: str = None) -> str: +def jdk_url( + index: Index, conf: Configuration, exact_version: str = None +) -> str: """ Find in index the URL for the JDK binary for the given vendor and version. @@ -134,20 +135,22 @@ def _read_index(path: Path) -> Index: # graalvm-java17 version 22.3.3 bundles OpenJDK 17.0.8, but # unfortunately there is no way to know this from the index alone. - pattern = re.compile('^(jdk@ibm-semeru.*)-java\\d+$') + pattern = re.compile("^(jdk@ibm-semeru.*)-java\\d+$") for os, arches in index.items(): for arch, vendors in arches.items(): for vendor, versions in vendors.copy().items(): - if not vendor.startswith("jdk@graalvm") and (m := pattern.match(vendor)): + if not vendor.startswith("jdk@graalvm") and ( + m := pattern.match(vendor) + ): true_vendor = m.group(1) - if not true_vendor in index[os][arch]: + if true_vendor not in index[os][arch]: index[os][arch][true_vendor] = {} index[os][arch][true_vendor].update(versions) return index -def _get_versions(jdks: Tuple[str, str], conf) -> List[str]: +def _get_versions(jdks: tuple[str, str], conf) -> list[str]: versions = [i[1] for i in jdks if i[0] == conf.vendor] if not versions: raise KeyError( @@ -156,7 +159,9 @@ def _get_versions(jdks: Tuple[str, str], conf) -> List[str]: return versions -def _match_versions(vendor, candidates: List[str], requested) -> Dict[Tuple[int], str]: +def _match_versions( + vendor, candidates: list[str], requested +) -> dict[tuple[int], str]: # Find all candidates compatible with the request is_graal = "graalvm" in vendor.lower() normreq = _normalize_version(requested, remove_prefix_1=not is_graal) @@ -175,12 +180,13 @@ def _match_versions(vendor, candidates: List[str], requested) -> Dict[Tuple[int] normcands[normcand] = candidate return { - k: v for k, v in normcands.items() + k: v + for k, v in normcands.items() if _is_version_compatible_with_spec(k, normreq) } -def _match_version(vendor, candidates: List[str], requested) -> str: +def _match_version(vendor, candidates: list[str], requested) -> str: matched = _match_versions(vendor, candidates, requested) if len(matched) == 0: diff --git a/src/cjdk/_install.py b/src/cjdk/_install.py index de9b816..f8fdfa2 100644 --- a/src/cjdk/_install.py +++ b/src/cjdk/_install.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: MIT import sys - from pathlib import Path from . import _cache, _download @@ -14,7 +13,9 @@ ] -def install_file(prefix, name, url, filename, conf, *, ttl, checkfunc=None) -> Path: +def install_file( + prefix, name, url, filename, conf, *, ttl, checkfunc=None +) -> Path: def fetch(dest): _print_progress_header(conf, name) _download.download_file( From 98a2befe82badb47ff299d4452a9fd40812fccbc Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 21:11:25 -0500 Subject: [PATCH 12/25] Guard against weird index structures --- src/cjdk/_index.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index 0a95a7b..3d2cc0f 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -136,8 +136,14 @@ def _read_index(path: Path) -> Index: # unfortunately there is no way to know this from the index alone. pattern = re.compile("^(jdk@ibm-semeru.*)-java\\d+$") + if not hasattr(index, "items"): + return index for os, arches in index.items(): + if not hasattr(arches, "items"): + continue for arch, vendors in arches.items(): + if not hasattr(vendors, "items"): + continue for vendor, versions in vendors.copy().items(): if not vendor.startswith("jdk@graalvm") and ( m := pattern.match(vendor) From d38d3def29154151d0025be4caf751aa4ebf7778 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 21:42:53 -0500 Subject: [PATCH 13/25] Split out helper methods for ls and ls-vendors So that the computed data structures are consumable, rather than only printed. --- src/cjdk/_api.py | 63 +++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/src/cjdk/_api.py b/src/cjdk/_api.py index 423538e..afa8c8f 100644 --- a/src/cjdk/_api.py +++ b/src/cjdk/_api.py @@ -36,15 +36,7 @@ def list_vendors(**kwargs): ------- None """ - conf = _conf.configure(**kwargs) - index = _index.jdk_index(conf) - vendors = { - vendor.replace("jdk@", "") - for osys in index - for arch in index[osys] - for vendor in index[osys][arch] - } - print(os.linesep.join(sorted(vendors))) + print(os.linesep.join(sorted(_get_vendors(**kwargs)))) def list_jdks(*, vendor=None, version=None, cached_only=True, **kwargs): @@ -79,24 +71,10 @@ def list_jdks(*, vendor=None, version=None, cached_only=True, **kwargs): ------- None """ - conf = _conf.configure(vendor=vendor, version=version, **kwargs) - index = _index.jdk_index(conf) - jdks = _index.available_jdks(index, conf) - versions = _index._get_versions(jdks, conf) - matched = _index._match_versions(conf.vendor, versions, conf.version) - - if cached_only: - # Filter matches by existing key directories. - def is_cached(v): - url = _index.jdk_url(index, conf, v) - key = (_jdk._JDK_KEY_PREFIX, _cache._key_for_url(url)) - keydir = _cache._key_directory(conf.cache_dir, key) - return keydir.exists() - - matched = {k: v for k, v in matched.items() if is_cached(v)} - - print(f"[{conf.vendor}]") - print(os.linesep.join([v for _, v in sorted(matched.items())])) + jdks_list = _get_jdks( + vendor=vendor, version=version, cached_only=cached_only, **kwargs + ) + print(os.linesep.join(jdks_list)) def cache_jdk(*, vendor=None, version=None, **kwargs): @@ -278,6 +256,37 @@ def cache_package(name, url, **kwargs): ) +def _get_vendors(**kwargs): + conf = _conf.configure(**kwargs) + index = _index.jdk_index(conf) + return { + vendor.replace("jdk@", "") + for osys in index + for arch in index[osys] + for vendor in index[osys][arch] + } + + +def _get_jdks(*, vendor=None, version=None, cached_only=True, **kwargs): + conf = _conf.configure(vendor=vendor, version=version, **kwargs) + index = _index.jdk_index(conf) + jdks = _index.available_jdks(index, conf) + versions = _index._get_versions(jdks, conf) + matched = _index._match_versions(conf.vendor, versions, conf.version) + + if cached_only: + # Filter matches by existing key directories. + def is_cached(v): + url = _index.jdk_url(index, conf, v) + key = (_jdk._JDK_KEY_PREFIX, _cache._key_for_url(url)) + keydir = _cache._key_directory(conf.cache_dir, key) + return keydir.exists() + + matched = {k: v for k, v in matched.items() if is_cached(v)} + + return [f"{conf.vendor}:{v}" for k, v in sorted(matched.items())] + + def _make_hash_checker(hashes): checks = [ (hashes.pop("sha1", None), hashlib.sha1), From 5377c92796457025dc1cae5f146293b0ced5c26b Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 21:43:36 -0500 Subject: [PATCH 14/25] Let ls operate on all vendors by default --- src/cjdk/_api.py | 20 +++++++++++++++++++- src/cjdk/_conf.py | 7 ++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/cjdk/_api.py b/src/cjdk/_api.py index afa8c8f..4ebdab8 100644 --- a/src/cjdk/_api.py +++ b/src/cjdk/_api.py @@ -268,7 +268,25 @@ def _get_vendors(**kwargs): def _get_jdks(*, vendor=None, version=None, cached_only=True, **kwargs): - conf = _conf.configure(vendor=vendor, version=version, **kwargs) + conf = _conf.configure( + vendor=vendor, + version=version, + fallback_to_default_vendor=False, + **kwargs, + ) + if conf.vendor is None: + # Search across all vendors. + kwargs.pop("jdk", None) # It was already parsed. + return [ + jdk + for v in sorted(_get_vendors()) + for jdk in _get_jdks( + vendor=v, + version=conf.version, + cached_only=cached_only, + **kwargs, + ) + ] index = _index.jdk_index(conf) jdks = _index.available_jdks(index, conf) versions = _index._get_versions(jdks, conf) diff --git a/src/cjdk/_conf.py b/src/cjdk/_conf.py index 499e56d..df2b35d 100644 --- a/src/cjdk/_conf.py +++ b/src/cjdk/_conf.py @@ -39,10 +39,15 @@ def configure(**kwargs): raise ValueError("Cannot specify jdk= together with version=") kwargs["vendor"], kwargs["version"] = _parse_vendor_version(jdk) + default_vendor = ( + _default_vendor() + if kwargs.pop("fallback_to_default_vendor", True) + else None + ) conf = Configuration( os=_canonicalize_os(kwargs.pop("os", None)), arch=_canonicalize_arch(kwargs.pop("arch", None)), - vendor=kwargs.pop("vendor", None) or _default_vendor(), + vendor=kwargs.pop("vendor", None) or default_vendor, version=kwargs.pop("version", "") or "", cache_dir=kwargs.pop("cache_dir", None) or _default_cachedir(), index_url=kwargs.pop("index_url", None) or _default_index_url(), From d53ba0818f71c73867532ce9c30a362e27249b29 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Thu, 1 Aug 2024 23:13:25 -0500 Subject: [PATCH 15/25] Make the version comparison more robust This patch is kind of horrible, but it gets the job done correctly for more challenging version strings like 21.0.1+12_openj9-0.42.0. --- src/cjdk/_api.py | 23 ++++++++++++++++++++++- src/cjdk/_index.py | 22 +++++++++++++--------- tests/test_index.py | 4 ++-- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/cjdk/_api.py b/src/cjdk/_api.py index 4ebdab8..4b6b129 100644 --- a/src/cjdk/_api.py +++ b/src/cjdk/_api.py @@ -302,7 +302,28 @@ def is_cached(v): matched = {k: v for k, v in matched.items() if is_cached(v)} - return [f"{conf.vendor}:{v}" for k, v in sorted(matched.items())] + class VersionElement: + def __init__(self, value): + self.value = value + self.is_int = isinstance(value, int) + + def __eq__(self, other): + if self.is_int and other.is_int: + return self.value == other.value + return str(self.value) == str(other.value) + + def __lt__(self, other): + if self.is_int and other.is_int: + return self.value < other.value + return str(self.value) < str(other.value) + + def version_key(version_tuple): + return tuple(VersionElement(elem) for elem in version_tuple[0]) + + return [ + f"{conf.vendor}:{v}" + for k, v in sorted(matched.items(), key=version_key) + ] def _make_hash_checker(hashes): diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index 3d2cc0f..ed8b9b8 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -201,16 +201,16 @@ def _match_version(vendor, candidates: list[str], requested) -> str: return matched[max(matched)] -_VER_SEPS = re.compile(r"[.-]") +_VER_SEPS = re.compile(r"[.+_-]") def _normalize_version(ver, *, remove_prefix_1=False): # Normalize requested version and candidates: # - Split at dots and dashes (so we don't distinguish between '.' and '-') - # - Convert elements to integers (so that we require numeric elements and - # compare them numerically) + # - Try to convert elements to integers (so that we can compare elements + # numerically where feasible) # - If remove_prefix_1 and first element is 1, remove it (so JDK 1.8 == 8) - # - Return as tuple of ints (so that we compare lexicographically) + # - Return as a tuple (so that we compare element by element) # - Trailing zero elements are NOT removed, so, e.g., 11 < 11.0 (for the # most part, the index uses versions with the same number of elements # within a given vendor; versions like "11" are outliers) @@ -219,11 +219,8 @@ def _normalize_version(ver, *, remove_prefix_1=False): if is_plus: ver = ver[:-1] if ver: - norm = re.split(_VER_SEPS, ver) - try: - norm = tuple(int(e) for e in norm) - except ValueError as err: - raise ValueError(f"Invalid version string: {ver}") from err + norm = tuple(re.split(_VER_SEPS, ver)) + norm = tuple(_intify(e) for e in norm) else: norm = () plus = ("+",) if is_plus else () @@ -232,6 +229,13 @@ def _normalize_version(ver, *, remove_prefix_1=False): return norm + plus +def _intify(s: str): + try: + return int(s) + except ValueError: + return s + + def _is_version_compatible_with_spec(version, spec): assert "+" not in version is_plus = spec and spec[-1] == "+" diff --git a/tests/test_index.py b/tests/test_index.py index 9e564dd..9daa21c 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -160,8 +160,8 @@ def test_normalize_version(): assert f("1", remove_prefix_1=True) == () assert f("1.8", remove_prefix_1=True) == (8,) assert f("1.8.0", remove_prefix_1=True) == (8, 0) - with pytest.raises(ValueError): - f("1.8u300", remove_prefix_1=True) + assert f("1.8u300", remove_prefix_1=True) == ("8u300",) + assert f("21.0.1+12_openj9-0.42.0") == (21, 0, 1, 12, "openj9", 0, 42, 0) def test_is_version_compatible_with_spec(): From 4c3ed9c149235ac4c0998b156d56c84f23f6101f Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 7 Aug 2024 19:25:21 -0500 Subject: [PATCH 16/25] Remove unused available_vendors function --- src/cjdk/_index.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index ed8b9b8..ae26249 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -37,18 +37,6 @@ def jdk_index(conf: Configuration) -> Index: return _read_index(_cached_index_path(conf)) -def available_vendors(index: Index): - """ - Find in index the available JDK vendors. - - A set of strings is returned. - - Arguments: - index -- The JDK index (nested dict) - """ - return set(index[os][arch] for os in index for arch in index[os]) - - def available_jdks(index: Index, conf: Configuration) -> tuple[str, str]: """ Find in index the available JDK vendor-version combinations. From 0f60b553bb7f8029c165592dc97d62901e7fd3e1 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 7 Aug 2024 19:43:15 -0500 Subject: [PATCH 17/25] Move index postprocessing to helper function --- src/cjdk/_index.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index ae26249..db23b9f 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -110,18 +110,25 @@ def _read_index(path: Path) -> Index: with open(path, encoding="ascii") as infile: index = json.load(infile) - # Post-process the index to normalize the data. - # Some "vendors" include the major Java version, - # so let's merge such entries. In particular: - # - # * ibm-semuru-openj9-java<##> - # * graalvm-java<##> - # - # However: while the graalvm vendors follow this pattern, the version - # numbers for graalvm are *not* JDK versions, but rather GraalVM versions, - # which merely strongly resemble JDK version strings. For example, - # graalvm-java17 version 22.3.3 bundles OpenJDK 17.0.8, but - # unfortunately there is no way to know this from the index alone. + return _postprocess_index(index) + + +def _postprocess_index(index: Index) -> Index: + """ + Post-process the index to normalize the data. + + Some "vendors" include the major Java version, + so let's merge such entries. In particular: + + * ibm-semuru-openj9-java<##> + * graalvm-java<##> + + However: while the graalvm vendors follow this pattern, the version + numbers for graalvm are *not* JDK versions, but rather GraalVM versions, + which merely strongly resemble JDK version strings. For example, + graalvm-java17 version 22.3.3 bundles OpenJDK 17.0.8, but + unfortunately there is no way to know this from the index alone. + """ pattern = re.compile("^(jdk@ibm-semeru.*)-java\\d+$") if not hasattr(index, "items"): From b67016567fa00bfd76aaefbe229b6a8ca8c7f138 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 7 Aug 2024 19:49:11 -0500 Subject: [PATCH 18/25] Add tests for new functions and parameters _api: - _get_vendors - _get_jdks _conf: - configure: fallback_to_default_vendor=False _index: - jdk_url: exact_version=non-None - _postprocess_index - _match_versions --- tests/test_api.py | 35 +++++++++++++++++++++ tests/test_conf.py | 4 +++ tests/test_index.py | 75 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) diff --git a/tests/test_api.py b/tests/test_api.py index 87912b9..6fbcb39 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -152,3 +152,38 @@ def test_env_var_set(): with f("CJDK_TEST_ENV_VAR", "testvalue"): assert os.environ["CJDK_TEST_ENV_VAR"] == "testvalue" assert "CJDK_TEST_ENV_VAR" not in os.environ + + +def test_get_vendors(): + vendors = _api._get_vendors() + assert vendors is not None + assert "adoptium" in vendors + assert "corretto" in vendors + assert "graalvm" in vendors + assert "ibm-semeru-openj9" in vendors + assert "java-oracle" in vendors + assert "liberica" in vendors + assert "temurin" in vendors + assert "zulu" in vendors + + +def test_get_jdks(): + jdks = _api._get_jdks(cached_only=False) + assert jdks is not None + assert "adoptium:1.21.0.4" in jdks + assert "corretto:21.0.4.7.1" in jdks + assert "graalvm-community:21.0.2" in jdks + assert "graalvm-java17:22.3.3" in jdks + assert "graalvm-java21:21.0.2" in jdks + assert "liberica:22.0.2" in jdks + assert "temurin:1.21.0.4" in jdks + assert "zulu:8.0.362" in jdks + + cached_jdks = _api._get_jdks() + assert cached_jdks is not None + assert len(cached_jdks) < len(jdks) + + zulu_jdks = _api._get_jdks(vendor="zulu", cached_only=False) + assert zulu_jdks is not None + assert len(set(zulu_jdks)) > 175 + assert all(jdk.startswith("zulu:") for jdk in zulu_jdks) diff --git a/tests/test_conf.py b/tests/test_conf.py index 0c45a7a..8779ef3 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -30,6 +30,10 @@ def test_configure(): assert conf.vendor == _conf._default_vendor() assert not conf.version + conf = f(jdk=":", fallback_to_default_vendor=False) + assert conf.vendor is None + assert not conf.version + conf = f(cache_dir="abc") assert conf.cache_dir == Path("abc") diff --git a/tests/test_index.py b/tests/test_index.py index 9daa21c..1650f2d 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -87,6 +87,25 @@ def test_jdk_url(): ) == "tgz+https://example.com/a/b/c.tgz" ) + assert ( + _index.jdk_url( + index, + configure( + os="linux", arch="amd64", vendor="adoptium", version="17" + ), + ) + == "tgz+https://example.com/a/b/c.tgz" + ) + assert ( + _index.jdk_url( + index, + configure( + os="linux", arch="amd64", vendor="adoptium", version="11" + ), + exact_version="17.0.1", + ) + == "tgz+https://example.com/a/b/c.tgz" + ) def test_cached_index_path(tmp_path): @@ -140,6 +159,62 @@ def test_read_index(tmp_path): assert _index._read_index(path) == data +def test_postprocess_index(): + index = { + "linux": { + "amd64": { + "jdk@ibm-semeru-openj9-java11": { + "11.0.21+9_openj9-0.41.0": "a", + "11.0.22+7_openj9-0.43.0": "b", + "11.0.23+9_openj9-0.44.0": "c", + }, + "jdk@ibm-semeru-openj9-java17": { + "17.0.1+12_openj9-0.29.1": "d", + "17.0.2+8_openj9-0.30.0": "e", + "17.0.3+7_openj9-0.32.0": "f", + }, + "jdk@ibm-semeru-openj9-java21": { + "21.0.1+12_openj9-0.42.0": "g", + "21.0.2+13_openj9-0.43.0": "h", + }, + "jdk@not-semeru": {"8.0.252": "i"}, + } + } + } + pp_index = _index._postprocess_index(index) + assert pp_index is index + assert "jdk@ibm-semeru-openj9" in index["linux"]["amd64"] + assert len(index["linux"]["amd64"]["jdk@ibm-semeru-openj9"]) == 8 + + +def test_match_versions(): + f = _index._match_versions + assert f("adoptium", ["10", "11.0", "11.1", "1.12.0"], "11") == { + (11, 0): "11.0", + (11, 1): "11.1", + } + assert f("adoptium", ["10", "11.0", "11.1", "1.12.0"], "12") == { + (12, 0): "1.12.0" + } + assert f("graalvm", ["10", "11.0", "11.1", "1.12.0"], "11") == { + (11, 0): "11.0", + (11, 1): "11.1", + } + assert f("graalvm", ["10", "11.0", "11.1", "1.12.0"], "1") == { + (1, 12, 0): "1.12.0" + } + assert f("temurin", ["11.0", "17.0", "18.0"], "") == { + (11, 0): "11.0", + (17, 0): "17.0", + (18, 0): "18.0", + } + assert f("temurin", ["11.0", "17.0", "18.0"], "17+") == { + (17, 0): "17.0", + (18, 0): "18.0", + } + assert f("temurin", ["11.0", "17.0", "18.0"], "19+") == {} + + def test_match_version(): f = _index._match_version assert f("adoptium", ["10", "11.0", "11.1", "1.12.0"], "11") == "11.1" From ac6b76dca1a91a87b0d962edeccf8abef9f48718 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 7 Aug 2024 19:57:51 -0500 Subject: [PATCH 19/25] Update CLI documentation * Add ls and ls-vendors commands. * Rename cache-jdk command to cache. --- docs/cli.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index e4794e9..bf24a62 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -24,6 +24,26 @@ More details about the choices and defaults for [`VENDOR`](./vendors.md), [`VERSION`](./versions.md), and [`--cache_dir`](./cachedir.md) are available on separate pages. +## Querying the JDK index + +### `ls` + +```{command-output} cjdk ls --help +``` + +```{eval-rst} +.. versionadded:: 0.4.0 +``` + +### `ls-vendors` + +```{command-output} cjdk ls-vendors --help +``` + +```{eval-rst} +.. versionadded:: 0.4.0 +``` + ## Working with cached JDKs ### `exec` @@ -53,13 +73,13 @@ $ cjdk --jdk temurin-jre:17.0.3 java-home (The output will depend on your operating system and configuration; the example shown was on macOS.) -### `cache-jdk` +### `cache` -```{command-output} cjdk cache-jdk --help +```{command-output} cjdk cache --help ``` ```{eval-rst} -.. versionadded:: 0.2.0 +.. versionadded:: 0.4.0 ``` ## Caching arbitrary files and packages From 33bfbc3762c965760e9637d4faaf7f29d345dc19 Mon Sep 17 00:00:00 2001 From: Curtis Rueden Date: Wed, 7 Aug 2024 19:58:19 -0500 Subject: [PATCH 20/25] Update the changelog --- docs/changelog.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 7e58b78..298e398 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -10,7 +10,15 @@ See also the section on [versioning](versioning-scheme). ## [Unreleased] -- No notable changes yet. +### Added + +- Python API functions `list_jdks()` and `list_vendors()`. +- Command line commands `ls` and `ls-vendors`. +- Light postprocessing of vendor names, notably `ibm-semeru-openj9`. + +### Changed + +- Command line command `cache-jdk` renamed to `cache`. ## [0.3.0] - 2022-07-09 From f73ad236df19da166855f885e6960a3e8c559696 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 9 Aug 2024 11:09:33 -0500 Subject: [PATCH 21/25] CI: Don't cancel other OS tests on failure --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b967aa..609c532 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,7 @@ jobs: test: strategy: + fail-fast: false matrix: runner: [ubuntu-latest, macos-latest, windows-latest] name: test-${{ matrix.runner }} From 0273f7b593e5c46bb169fc40712795c0ff384eb7 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 9 Aug 2024 14:35:48 -0500 Subject: [PATCH 22/25] Move error generation to the right place It's not an error for the versions to be empty unless we are trying to resolve to a single version. --- src/cjdk/_index.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/cjdk/_index.py b/src/cjdk/_index.py index db23b9f..b0fb0b2 100644 --- a/src/cjdk/_index.py +++ b/src/cjdk/_index.py @@ -67,6 +67,10 @@ def resolve_jdk_version(index: Index, conf: Configuration) -> str: """ jdks = available_jdks(index, conf) versions = _get_versions(jdks, conf) + if not versions: + raise KeyError( + f"No {conf.vendor} JDK is available for {conf.os}-{conf.arch}" + ) return _match_version(conf.vendor, versions, conf.version) @@ -152,12 +156,7 @@ def _postprocess_index(index: Index) -> Index: def _get_versions(jdks: tuple[str, str], conf) -> list[str]: - versions = [i[1] for i in jdks if i[0] == conf.vendor] - if not versions: - raise KeyError( - f"No {conf.vendor} JDK is available for {conf.os}-{conf.arch}" - ) - return versions + return [i[1] for i in jdks if i[0] == conf.vendor] def _match_versions( From e6a6f28d7141073fc88da67655fb39ab47e6ee20 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 9 Aug 2024 14:36:41 -0500 Subject: [PATCH 23/25] Edit test for non-Linux (Better if we avoid remote-dependent unit tests, or course, but I'll leave that for another time :) --- tests/test_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 6fbcb39..f2bb84a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -173,7 +173,6 @@ def test_get_jdks(): assert "adoptium:1.21.0.4" in jdks assert "corretto:21.0.4.7.1" in jdks assert "graalvm-community:21.0.2" in jdks - assert "graalvm-java17:22.3.3" in jdks assert "graalvm-java21:21.0.2" in jdks assert "liberica:22.0.2" in jdks assert "temurin:1.21.0.4" in jdks @@ -185,5 +184,5 @@ def test_get_jdks(): zulu_jdks = _api._get_jdks(vendor="zulu", cached_only=False) assert zulu_jdks is not None - assert len(set(zulu_jdks)) > 175 + assert len(set(zulu_jdks)) assert all(jdk.startswith("zulu:") for jdk in zulu_jdks) From fd8a044cc0a85cc33a385b305d80584f257486ba Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 9 Aug 2024 16:43:26 -0500 Subject: [PATCH 24/25] Return, do not print, in Python API Also avoid printing an empty line when the result is empty. --- src/cjdk/__init__.py | 4 ++-- src/cjdk/__main__.py | 14 +++++++++----- src/cjdk/_api.py | 23 ++++++++++++----------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/cjdk/__init__.py b/src/cjdk/__init__.py index 6d2696a..4d4a54c 100644 --- a/src/cjdk/__init__.py +++ b/src/cjdk/__init__.py @@ -15,10 +15,10 @@ __all__ = [ "cache_file", - "list_vendors", - "list_jdks", "cache_jdk", "cache_package", "java_env", "java_home", + "list_jdks", + "list_vendors", ] diff --git a/src/cjdk/__main__.py b/src/cjdk/__main__.py index 9595423..0aab32c 100644 --- a/src/cjdk/__main__.py +++ b/src/cjdk/__main__.py @@ -68,9 +68,11 @@ def main(ctx, jdk, cache_dir, index_url, index_ttl, os, arch, progress): @click.pass_context def ls_vendors(ctx): """ - Output the list of available vendors. + Print the list of available JDK vendors. """ - _api.list_vendors(**ctx.obj) + vendors = _api.list_vendors(**ctx.obj) + if vendors: + print("\n".join(vendors)) @click.command(short_help="List cached or available JDKs matching criteria.") @@ -82,11 +84,13 @@ def ls_vendors(ctx): ) def ls(ctx, cached: bool = False): """ - Output a list of JDKs matching the given criteria. + Print the list of JDKs matching the given criteria. - See 'cjdk --help' for the common options used to specify the JDK. + See 'cjdk --help' for the common options used to specify the criteria. """ - _api.list_jdks(**ctx.obj, cached_only=cached) + jdks = _api.list_jdks(**ctx.obj, cached_only=cached) + if jdks: + print("\n".join(jdks)) @click.command(short_help="Ensure the requested JDK is cached.") diff --git a/src/cjdk/_api.py b/src/cjdk/_api.py index 4b6b129..02875b1 100644 --- a/src/cjdk/_api.py +++ b/src/cjdk/_api.py @@ -9,19 +9,19 @@ from . import _cache, _conf, _index, _install, _jdk __all__ = [ - "list_vendors", - "list_jdks", + "cache_file", "cache_jdk", + "cache_package", "java_env", "java_home", - "cache_file", - "cache_package", + "list_jdks", + "list_vendors", ] def list_vendors(**kwargs): """ - Output the list of available vendors. + Return the list of available JDK vendors. Parameters ---------- @@ -34,14 +34,15 @@ def list_vendors(**kwargs): Returns ------- - None + list[str] + The available JDK vendors. """ - print(os.linesep.join(sorted(_get_vendors(**kwargs)))) + return sorted(_get_vendors(**kwargs)) def list_jdks(*, vendor=None, version=None, cached_only=True, **kwargs): """ - Output a list of JDKs matching the given criteria. + Return the list of JDKs matching the given criteria. Parameters ---------- @@ -69,12 +70,12 @@ def list_jdks(*, vendor=None, version=None, cached_only=True, **kwargs): Returns ------- - None + list[str] + JDKs (vendor:version) matching the criteria. """ - jdks_list = _get_jdks( + return _get_jdks( vendor=vendor, version=version, cached_only=cached_only, **kwargs ) - print(os.linesep.join(jdks_list)) def cache_jdk(*, vendor=None, version=None, **kwargs): From 0d35cf50b1db5b0f1cc380518610107db8e18f12 Mon Sep 17 00:00:00 2001 From: "Mark A. Tsuchida" Date: Fri, 9 Aug 2024 16:52:49 -0500 Subject: [PATCH 25/25] Add new Python APIs to docs --- docs/api.md | 12 ++++++++++++ docs/cli.md | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index c85e52f..5cf66ad 100644 --- a/docs/api.md +++ b/docs/api.md @@ -6,6 +6,18 @@ SPDX-License-Identifier: MIT # Python API +## Querying the JDK index + +```{eval-rst} +.. autofunction:: cjdk.list_vendors +.. versionadded:: 0.4.0 +``` + +```{eval-rst} +.. autofunction:: cjdk.list_jdks +.. versionadded:: 0.4.0 +``` + ## Working with cached JDKs ```{eval-rst} diff --git a/docs/cli.md b/docs/cli.md index bf24a62..135df44 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -79,7 +79,8 @@ shown was on macOS.) ``` ```{eval-rst} -.. versionadded:: 0.4.0 +.. versionchanged:: 0.4.0 + Renamed from ``cache-jdk``. ``` ## Caching arbitrary files and packages