From 98e321e79578823c5d3ff4aee2826b1812a2f03f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 8 Oct 2023 13:04:06 -0700 Subject: [PATCH 1/2] sage --package list: Sort output, add switches --{in,ex}clude-dependencies, handle --exclude at the end --- bootstrap | 8 ++-- build/bin/sage-package | 2 +- build/sage_bootstrap/app.py | 4 +- build/sage_bootstrap/cmdline.py | 28 ++++++++++--- build/sage_bootstrap/expand_class.py | 61 +++++++++++++++++++++++----- build/sage_bootstrap/package.py | 40 ++++++++++++++++++ src/doc/bootstrap | 10 ++--- 7 files changed, 125 insertions(+), 28 deletions(-) diff --git a/bootstrap b/bootstrap index 54d0a156239..82e42789250 100755 --- a/bootstrap +++ b/bootstrap @@ -47,12 +47,12 @@ bootstrap () { done spkg_configures="" # initialize SAGE_ENABLE... options for standard packages - for pkgname in $(sage-package list :standard: | sort); do + for pkgname in $(sage-package list :standard:); do spkg_configures="$spkg_configures AS_VAR_SET_IF([SAGE_ENABLE_$pkgname], [], [AS_VAR_SET([SAGE_ENABLE_$pkgname], [yes])])" done # --enable-SPKG options - for pkgname in $(sage-package list :optional: :experimental: | sort); do + for pkgname in $(sage-package list :optional: :experimental:); do # Issue #29629: Temporary solution for Sage 9.1: Do not provide # --enable-SPKG options for installing pip packages if [ ! -f build/pkgs/$pkgname/requirements.txt ]; then @@ -71,7 +71,7 @@ SAGE_SPKG_ENABLE([$pkgname], [$pkgtype], [$(grep -v ^= build/pkgs/$pkgname/SPKG. esac fi done - for pkgname in $(sage-package list --has-file spkg-configure.m4 | sort); do + for pkgname in $(sage-package list --has-file spkg-configure.m4); do echo "m4_sinclude([build/pkgs/$pkgname/spkg-configure.m4])" config="SAGE_SPKG_CONFIGURE_$(echo ${pkgname} | tr '[a-z]' '[A-Z]')" if grep -q SAGE_PYTHON_PACKAGE_CHECK build/pkgs/$pkgname/spkg-configure.m4; then @@ -86,7 +86,7 @@ $config" $spkg_configures $spkg_configures_python EOF - for pkgname in $(sage-package list | sort); do + for pkgname in $(sage-package list); do DIR=build/pkgs/$pkgname pkgtype="$(cat $DIR/type)" if test -f "$DIR/requirements.txt"; then diff --git a/build/bin/sage-package b/build/bin/sage-package index ccb3ad194a7..02a07140875 100755 --- a/build/bin/sage-package +++ b/build/bin/sage-package @@ -13,7 +13,7 @@ # # * Print a list of all available packages # -# $ sage-package list | sort +# $ sage-package list # 4ti2 # arb # autotools diff --git a/build/sage_bootstrap/app.py b/build/sage_bootstrap/app.py index 22fb252410b..8c17541a9b7 100644 --- a/build/sage_bootstrap/app.py +++ b/build/sage_bootstrap/app.py @@ -53,7 +53,7 @@ def list_cls(self, *package_classes, **filters): """ Print a list of all available packages - $ sage --package list | sort + $ sage --package list 4ti2 arb autotools @@ -63,7 +63,7 @@ def list_cls(self, *package_classes, **filters): $ sage -package list --has-file=spkg-configure.m4 :experimental: perl_term_readline_gnu - $ sage -package list --has-file=spkg-configure.m4 --has-file=distros/debian.txt | sort + $ sage -package list --has-file=spkg-configure.m4 --has-file=distros/debian.txt arb boost_cropped brial diff --git a/build/sage_bootstrap/cmdline.py b/build/sage_bootstrap/cmdline.py index ca89b4fedac..c62ca6d3a01 100644 --- a/build/sage_bootstrap/cmdline.py +++ b/build/sage_bootstrap/cmdline.py @@ -64,18 +64,18 @@ epilog_list = \ """ -Print a list of packages known to Sage +Print a list of packages known to Sage (sorted alphabetically) EXAMPLE: - $ sage --package list | sort + $ sage --package list 4ti2 arb autotools [...] zlib - $ sage --package list :standard: | sort + $ sage --package list :standard: arb backports_ssl_match_hostname [...] @@ -214,7 +214,7 @@ def make_parser(): help='Print a list of packages known to Sage') parser_list.add_argument( 'package_class', metavar='[package_name|:package_type:]', - type=str, default=[':all:'], nargs='*', + type=str, default=[':all-or-nothing:'], nargs='*', help=('package name or designator for all packages of a given type ' '(one of :all:, :standard:, :optional:, and :experimental:); ' 'default: :all:')) @@ -227,8 +227,15 @@ def make_parser(): help=('only include packages that do not have this file in their metadata directory ' '(examples: huge, patches, huge|has_nonfree_dependencies)')) parser_list.add_argument( - '--exclude', action='append', default=[], metavar='PACKAGE_NAME', + '--exclude', nargs='*', action='append', default=[], metavar='PACKAGE_NAME', help='exclude package from list') + parser_list.add_argument( + '--include-dependencies', action='store_true', + help='include (ordinary) dependencies of the packages recursively') + parser_list.add_argument( + '--exclude-dependencies', action='store_true', + help='exclude (ordinary) dependencies of the packages recursively') + parser_name = subparsers.add_parser( 'name', epilog=epilog_name, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -353,7 +360,16 @@ def run(): if args.subcommand == 'config': app.config() elif args.subcommand == 'list': - app.list_cls(*args.package_class, has_files=args.has_files, no_files=args.no_files, exclude=args.exclude) + if args.package_class == [':all-or-nothing:']: + if args.include_dependencies or args.exclude_dependencies: + args.package_class = [] + else: + args.package_class = [':all:'] + app.list_cls(*args.package_class, + has_files=args.has_files, no_files=args.no_files, + exclude=args.exclude, + include_dependencies=args.include_dependencies, + exclude_dependencies=args.exclude_dependencies) elif args.subcommand == 'name': app.name(args.tarball_filename) elif args.subcommand == 'tarball': diff --git a/build/sage_bootstrap/expand_class.py b/build/sage_bootstrap/expand_class.py index 10ec907d0fa..fb2c88f6e97 100644 --- a/build/sage_bootstrap/expand_class.py +++ b/build/sage_bootstrap/expand_class.py @@ -21,17 +21,19 @@ class PackageClass(object): - def __init__(self, *package_names_or_classes, **filters): - self.names = [] + def __init__(self, *package_names_or_classes, exclude=(), + include_dependencies=False, exclude_dependencies=False, + **filters): + self.__names = set() filenames = filters.pop('has_files', []) no_filenames = filters.pop('no_files', []) - excluded = filters.pop('exclude', []) + excluded = [] + for package_names in exclude: + excluded.extend(package_names) if filters: raise ValueError('filter not supported') def included_in_filter(pkg): - if pkg.name in excluded: - return False if not all(any(pkg.has_file(filename) for filename in filename_disjunction.split('|')) for filename_disjunction in filenames): @@ -55,19 +57,58 @@ def included_in_filter(pkg): 'which must be one of ' ':all:, :standard:, :optional:, or :experimental:' 'got {}'.format(package_name_or_class)) - self.names.append(package_name_or_class) + self.__names.add(package_name_or_class) + + def include_recursive_dependencies(names, package_name): + if package_name in names: + return + try: + pkg = Package(package_name) + except FileNotFoundError: + # Silently ignore unknown packages, + # substitutions such as $(BLAS) $(PYTHON), + # and optional dependencies of the form $(find-string ...). + return + names.add(package_name) + for dependency in pkg.dependencies: + include_recursive_dependencies(names, dependency) + + if include_dependencies: + package_names = set() + for name in self.__names: + include_recursive_dependencies(package_names, name) + self.__names = package_names + + def exclude_recursive_dependencies(names, package_name): + try: + pkg = Package(package_name) + except FileNotFoundError: + return + for dependency in pkg.dependencies: + names.discard(dependency) + exclude_recursive_dependencies(names, dependency) + + if exclude_dependencies: + for name in list(self.__names): + exclude_recursive_dependencies(self.__names, name) + + self.__names.difference_update(excluded) + + @property + def names(self): + return sorted(self.__names) def _init_all(self, predicate): - self.names.extend(pkg.name for pkg in Package.all() if predicate(pkg)) + self.__names.update(pkg.name for pkg in Package.all() if predicate(pkg)) def _init_standard(self, predicate): - self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'standard' and predicate(pkg)) + self.__names.update(pkg.name for pkg in Package.all() if pkg.type == 'standard' and predicate(pkg)) def _init_optional(self, predicate): - self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'optional' and predicate(pkg)) + self.__names.update(pkg.name for pkg in Package.all() if pkg.type == 'optional' and predicate(pkg)) def _init_experimental(self, predicate): - self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'experimental' and predicate(pkg)) + self.__names.update(pkg.name for pkg in Package.all() if pkg.type == 'experimental' and predicate(pkg)) def apply(self, function, *args, **kwds): for package_name in self.names: diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py index 15b342223c4..8f8f1477261 100644 --- a/build/sage_bootstrap/package.py +++ b/build/sage_bootstrap/package.py @@ -47,6 +47,7 @@ def __init__(self, package_name): self._init_version() self._init_type() self._init_install_requires() + self._init_dependencies() def __repr__(self): return 'Package {0}'.format(self.name) @@ -245,6 +246,28 @@ def distribution_name(self): return part return None + @property + def dependencies(self): + """ + Return a list of strings, the package names of the (ordinary) dependencies + """ + # after a '|', we have order-only dependencies + return self.__dependencies.partition('|')[0].strip().split() + + @property + def dependencies_order_only(self): + """ + Return a list of strings, the package names of the order-only dependencies + """ + return self.__dependencies.partition('|')[2].strip().split() + self.__dependencies_order_only.strip().split() + + @property + def dependencies_check(self): + """ + Return a list of strings, the package names of the check dependencies + """ + return self.__dependencies_order_only.strip().split() + def __eq__(self, other): return self.tarball == other.tarball @@ -335,3 +358,20 @@ def _init_install_requires(self): self.__install_requires = f.read().strip() except IOError: self.__install_requires = None + + def _init_dependencies(self): + try: + with open(os.path.join(self.path, 'dependencies')) as f: + self.__dependencies = f.readline().strip() + except IOError: + self.__dependencies = '' + try: + with open(os.path.join(self.path, 'dependencies_check')) as f: + self.__dependencies_check = f.readline().strip() + except IOError: + self.__dependencies_check = '' + try: + with open(os.path.join(self.path, 'dependencies_order_only')) as f: + self.__dependencies_order_only = f.readline() + except IOError: + self.__dependencies_order_only = '' diff --git a/src/doc/bootstrap b/src/doc/bootstrap index be2168af064..fde83e4fd1b 100755 --- a/src/doc/bootstrap +++ b/src/doc/bootstrap @@ -100,7 +100,7 @@ Sage depends. It installs them automatically if it does not find equivalent system packages. EOF -for PKG_BASE in $(sage-package list --has-file SPKG.rst :standard: | grep -v '^sagemath_' | sort); do +for PKG_BASE in $(sage-package list --has-file SPKG.rst :standard: | grep -v '^sagemath_'); do echo "* :ref:\`spkg_$PKG_BASE\`" done >> "$OUTPUT_INDEX" cat >> "$OUTPUT_INDEX" <> "$OUTPUT_INDEX" cat >> "$OUTPUT_INDEX" <> "$OUTPUT_INDEX" cat >> "$OUTPUT_INDEX" <> "$OUTPUT_INDEX" @@ -203,7 +203,7 @@ Packages are in alphabetical order. :maxdepth: 1 EOF -for PKG_BASE in $(sage-package list --has-file SPKG.rst | sort); do +for PKG_BASE in $(sage-package list --has-file SPKG.rst); do PKG_SCRIPTS=build/pkgs/$PKG_BASE # Instead of just copying, we may want to call # a version of sage-spkg-info to format extra information. From bc7d33eeffd72b269b1f4ae7acbc0bbe3670d40e Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Fri, 6 Oct 2023 15:23:30 -0700 Subject: [PATCH 2/2] build/sage_bootstrap/cmdline.py: Explain default --- build/sage_bootstrap/cmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/sage_bootstrap/cmdline.py b/build/sage_bootstrap/cmdline.py index c62ca6d3a01..3ea6b9b7cb8 100644 --- a/build/sage_bootstrap/cmdline.py +++ b/build/sage_bootstrap/cmdline.py @@ -217,7 +217,7 @@ def make_parser(): type=str, default=[':all-or-nothing:'], nargs='*', help=('package name or designator for all packages of a given type ' '(one of :all:, :standard:, :optional:, and :experimental:); ' - 'default: :all:')) + 'default: :all: (or nothing when --include-dependencies or --exclude-dependencies is given')) parser_list.add_argument( '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files', help=('only include packages that have this file in their metadata directory '