diff --git a/build-aux/flatpak-pip-generator.py b/build-aux/flatpak-pip-generator.py deleted file mode 100755 index e93a9ec2..00000000 --- a/build-aux/flatpak-pip-generator.py +++ /dev/null @@ -1,494 +0,0 @@ -#!/usr/bin/env python3 - -__license__ = 'MIT' - -import argparse -import json -import hashlib -import os -import re -import shutil -import subprocess -import sys -import tempfile -import urllib.request - -from collections import OrderedDict -from typing import Dict - -try: - import requirements -except ImportError: - exit('Requirements modules is not installed. Run "pip install requirements-parser"') - -parser = argparse.ArgumentParser() -parser.add_argument('packages', nargs='*') -parser.add_argument('--python2', action='store_true', - help='Look for a Python 2 package') -parser.add_argument('--cleanup', choices=['scripts', 'all'], - help='Select what to clean up after build') -parser.add_argument('--requirements-file', '-r', - help='Specify requirements.txt file') -parser.add_argument('--build-only', action='store_const', - dest='cleanup', const='all', - help='Clean up all files after build') -parser.add_argument('--build-isolation', action='store_true', - default=False, - help=( - 'Do not disable build isolation. ' - 'Mostly useful on pip that does\'t ' - 'support the feature.' - )) -parser.add_argument('--ignore-installed', - type=lambda s: s.split(','), - default='', - help='Comma-separated list of package names for which pip ' - 'should ignore already installed packages. Useful when ' - 'the package is installed in the SDK but not in the ' - 'runtime.') -parser.add_argument('--checker-data', action='store_true', - help='Include x-checker-data in output for the "Flatpak External Data Checker"') -parser.add_argument('--output', '-o', - help='Specify output file name') -parser.add_argument('--runtime', - help='Specify a flatpak to run pip inside of a sandbox, ensures python version compatibility') -parser.add_argument('--yaml', action='store_true', - help='Use YAML as output format instead of JSON') -parser.add_argument('--ignore-errors', action='store_true', - help='Ignore errors when downloading packages') -parser.add_argument('--ignore-pkg', nargs='*', - help='Ignore a package when generating the manifest. Can only be used with a requirements file') -opts = parser.parse_args() - -if opts.yaml: - try: - import yaml - except ImportError: - exit('PyYAML modules is not installed. Run "pip install PyYAML"') - - -def get_pypi_url(name: str, filename: str) -> str: - url = 'https://pypi.org/pypi/{}/json'.format(name) - print('Extracting download url for', name) - with urllib.request.urlopen(url) as response: - body = json.loads(response.read().decode('utf-8')) - for release in body['releases'].values(): - for source in release: - if source['filename'] == filename: - return source['url'] - raise Exception('Failed to extract url from {}'.format(url)) - - -def get_tar_package_url_pypi(name: str, version: str) -> str: - url = 'https://pypi.org/pypi/{}/{}/json'.format(name, version) - with urllib.request.urlopen(url) as response: - body = json.loads(response.read().decode('utf-8')) - for ext in ['bz2', 'gz', 'xz', 'zip']: - for source in body['urls']: - if source['url'].endswith(ext): - return source['url'] - err = 'Failed to get {}-{} source from {}'.format(name, version, url) - raise Exception(err) - - -def get_package_name(filename: str) -> str: - if filename.endswith(('bz2', 'gz', 'xz', 'zip')): - segments = filename.split('-') - if len(segments) == 2: - return segments[0] - return '-'.join(segments[:len(segments) - 1]) - elif filename.endswith('whl'): - segments = filename.split('-') - if len(segments) == 5: - return segments[0] - candidate = segments[:len(segments) - 4] - # Some packages list the version number twice - # e.g. PyQt5-5.15.0-5.15.0-cp35.cp36.cp37.cp38-abi3-manylinux2014_x86_64.whl - if candidate[-1] == segments[len(segments) - 4]: - return '-'.join(candidate[:-1]) - return '-'.join(candidate) - else: - raise Exception( - 'Downloaded filename: {} does not end with bz2, gz, xz, zip, or whl'.format(filename) - ) - - -def get_file_version(filename: str) -> str: - name = get_package_name(filename) - segments = filename.split(name + '-') - version = segments[1].split('-')[0] - for ext in ['tar.gz', 'whl', 'tar.xz', 'tar.gz', 'tar.bz2', 'zip']: - version = version.replace('.' + ext, '') - return version - - -def get_file_hash(filename: str) -> str: - sha = hashlib.sha256() - print('Generating hash for', filename.split('/')[-1]) - with open(filename, 'rb') as f: - while True: - data = f.read(1024 * 1024 * 32) - if not data: - break - sha.update(data) - return sha.hexdigest() - - -def download_tar_pypi(url: str, tempdir: str) -> None: - with urllib.request.urlopen(url) as response: - file_path = os.path.join(tempdir, url.split('/')[-1]) - with open(file_path, 'x+b') as tar_file: - shutil.copyfileobj(response, tar_file) - - -def parse_continuation_lines(fin): - for line in fin: - line = line.rstrip('\n') - while line.endswith('\\'): - try: - line = line[:-1] + next(fin).rstrip('\n') - except StopIteration: - exit('Requirements have a wrong number of line continuation characters "\\"') - yield line - - -def fprint(string: str) -> None: - separator = '=' * 72 # Same as `flatpak-builder` - print(separator) - print(string) - print(separator) - - -packages = [] -if opts.requirements_file: - requirements_file_input = os.path.expanduser(opts.requirements_file) - try: - with open(requirements_file_input, 'r') as req_file: - reqs = parse_continuation_lines(req_file) - reqs_as_str = '\n'.join([r.split('--hash')[0] for r in reqs]) - reqs_list_raw = reqs_as_str.splitlines() - py_version_regex = re.compile(r';.*python_version .+$') # Remove when pip-generator can handle python_version - reqs_list = [py_version_regex.sub('', p) for p in reqs_list_raw] - if opts.ignore_pkg: - reqs_new = '\n'.join(i for i in reqs_list if i not in opts.ignore_pkg) - else: - reqs_new = reqs_as_str - packages = list(requirements.parse(reqs_new)) - with tempfile.NamedTemporaryFile('w', delete=False, prefix='requirements.') as req_file: - req_file.write(reqs_new) - requirements_file_output = req_file.name - except FileNotFoundError as err: - print(err) - sys.exit(1) - -elif opts.packages: - packages = list(requirements.parse('\n'.join(opts.packages))) - with tempfile.NamedTemporaryFile('w', delete=False, prefix='requirements.') as req_file: - req_file.write('\n'.join(opts.packages)) - requirements_file_output = req_file.name -else: - if not len(sys.argv) > 1: - exit('Please specifiy either packages or requirements file argument') - else: - exit('This option can only be used with requirements file') - -for i in packages: - if i["name"].lower().startswith("pyqt"): - print("PyQt packages are not supported by flapak-pip-generator") - print("However, there is a BaseApp for PyQt available, that you should use") - print("Visit https://github.com/flathub/com.riverbankcomputing.PyQt.BaseApp for more information") - sys.exit(0) - -with open(requirements_file_output, 'r') as req_file: - use_hash = '--hash=' in req_file.read() - -python_version = '2' if opts.python2 else '3' -if opts.python2: - pip_executable = 'pip2' -else: - pip_executable = 'pip3' - -if opts.runtime: - flatpak_cmd = [ - 'flatpak', - '--devel', - '--share=network', - '--filesystem=/tmp', - '--command={}'.format(pip_executable), - 'run', - opts.runtime - ] - if opts.requirements_file: - if os.path.exists(requirements_file_output): - prefix = os.path.realpath(requirements_file_output) - flag = '--filesystem={}'.format(prefix) - flatpak_cmd.insert(1,flag) -else: - flatpak_cmd = [pip_executable] - -output_path = '' - -if opts.output: - output_path = os.path.dirname(opts.output) - output_package = os.path.basename(opts.output) -elif opts.requirements_file: - output_package = 'python{}-{}'.format( - python_version, - os.path.basename(opts.requirements_file).replace('.txt', ''), - ) -elif len(packages) == 1: - output_package = 'python{}-{}'.format( - python_version, packages[0].name, - ) -else: - output_package = 'python{}-modules'.format(python_version) -if opts.yaml: - output_filename = os.path.join(output_path, output_package) + '.yaml' -else: - output_filename = os.path.join(output_path, output_package) + '.json' - -modules = [] -vcs_modules = [] -sources = {} - -tempdir_prefix = 'pip-generator-{}'.format(output_package) -with tempfile.TemporaryDirectory(prefix=tempdir_prefix) as tempdir: - pip_download = flatpak_cmd + [ - 'download', - '--exists-action=i', - '--dest', - tempdir, - '-r', - requirements_file_output - ] - if use_hash: - pip_download.append('--require-hashes') - - fprint('Downloading sources') - cmd = ' '.join(pip_download) - print('Running: "{}"'.format(cmd)) - try: - subprocess.run(pip_download, check=True) - os.remove(requirements_file_output) - except subprocess.CalledProcessError: - os.remove(requirements_file_output) - print('Failed to download') - print('Please fix the module manually in the generated file') - if not opts.ignore_errors: - print('Ignore the error by passing --ignore-errors') - raise - - try: - os.remove(requirements_file_output) - except FileNotFoundError: - pass - - fprint('Downloading arch independent packages') - for filename in os.listdir(tempdir): - if not filename.endswith(('bz2', 'any.whl', 'gz', 'xz', 'zip')): - version = get_file_version(filename) - name = get_package_name(filename) - url = get_tar_package_url_pypi(name, version) - print('Deleting', filename) - try: - os.remove(os.path.join(tempdir, filename)) - except FileNotFoundError: - pass - print('Downloading {}'.format(url)) - download_tar_pypi(url, tempdir) - - files = {get_package_name(f): [] for f in os.listdir(tempdir)} - - for filename in os.listdir(tempdir): - name = get_package_name(filename) - files[name].append(filename) - - # Delete redundant sources, for vcs sources - for name in files: - if len(files[name]) > 1: - zip_source = False - for f in files[name]: - if f.endswith('.zip'): - zip_source = True - if zip_source: - for f in files[name]: - if not f.endswith('.zip'): - try: - os.remove(os.path.join(tempdir, f)) - except FileNotFoundError: - pass - - vcs_packages = { - x.name: {'vcs': x.vcs, 'revision': x.revision, 'uri': x.uri} - for x in packages - if x.vcs - } - - fprint('Obtaining hashes and urls') - for filename in os.listdir(tempdir): - name = get_package_name(filename) - sha256 = get_file_hash(os.path.join(tempdir, filename)) - - if name in vcs_packages: - uri = vcs_packages[name]['uri'] - revision = vcs_packages[name]['revision'] - vcs = vcs_packages[name]['vcs'] - url = 'https://' + uri.split('://', 1)[1] - s = 'commit' - if vcs == 'svn': - s = 'revision' - source = OrderedDict([ - ('type', vcs), - ('url', url), - (s, revision), - ]) - is_vcs = True - else: - url = get_pypi_url(name, filename) - source = OrderedDict([ - ('type', 'file'), - ('url', url), - ('sha256', sha256)]) - if opts.checker_data: - source['x-checker-data'] = { - 'type': 'pypi', - 'name': name} - if url.endswith(".whl"): - source['x-checker-data']['packagetype'] = 'bdist_wheel' - is_vcs = False - sources[name] = {'source': source, 'vcs': is_vcs} - -# Python3 packages that come as part of org.freedesktop.Sdk. -system_packages = ['cython', 'easy_install', 'mako', 'markdown', 'meson', 'pip', 'pygments', 'setuptools', 'six', 'wheel'] - -fprint('Generating dependencies') -for package in packages: - - if package.name is None: - print('Warning: skipping invalid requirement specification {} because it is missing a name'.format(package.line), file=sys.stderr) - print('Append #egg= to the end of the requirement line to fix', file=sys.stderr) - continue - elif package.name.casefold() in system_packages: - print(f"{package.name} is in system_packages. Skipping.") - continue - - if len(package.extras) > 0: - extras = '[' + ','.join(extra for extra in package.extras) + ']' - else: - extras = '' - - version_list = [x[0] + x[1] for x in package.specs] - version = ','.join(version_list) - - if package.vcs: - revision = '' - if package.revision: - revision = '@' + package.revision - pkg = package.uri + revision + '#egg=' + package.name - else: - pkg = package.name + extras + version - - dependencies = [] - # Downloads the package again to list dependencies - - tempdir_prefix = 'pip-generator-{}'.format(package.name) - with tempfile.TemporaryDirectory(prefix='{}-{}'.format(tempdir_prefix, package.name)) as tempdir: - pip_download = flatpak_cmd + [ - 'download', - '--exists-action=i', - '--dest', - tempdir, - ] - try: - print('Generating dependencies for {}'.format(package.name)) - subprocess.run(pip_download + [pkg], check=True, stdout=subprocess.DEVNULL) - for filename in sorted(os.listdir(tempdir)): - dep_name = get_package_name(filename) - if dep_name.casefold() in system_packages: - continue - dependencies.append(dep_name) - - except subprocess.CalledProcessError: - print('Failed to download {}'.format(package.name)) - - is_vcs = True if package.vcs else False - package_sources = [] - for dependency in dependencies: - if dependency in sources: - source = sources[dependency] - elif dependency.replace('_', '-') in sources: - source = sources[dependency.replace('_', '-')] - else: - continue - - if not (not source['vcs'] or is_vcs): - continue - - package_sources.append(source['source']) - - if package.vcs: - name_for_pip = '.' - else: - name_for_pip = pkg - - module_name = 'python{}-{}'.format(python_version, package.name) - - pip_command = [ - pip_executable, - 'install', - '--verbose', - '--exists-action=i', - '--no-index', - '--find-links="file://${PWD}"', - '--prefix=${FLATPAK_DEST}', - '"{}"'.format(name_for_pip) - ] - if package.name in opts.ignore_installed: - pip_command.append('--ignore-installed') - if not opts.build_isolation: - pip_command.append('--no-build-isolation') - - module = OrderedDict([ - ('name', module_name), - ('buildsystem', 'simple'), - ('build-commands', [' '.join(pip_command)]), - ('sources', package_sources), - ]) - if opts.cleanup == 'all': - module['cleanup'] = ['*'] - elif opts.cleanup == 'scripts': - module['cleanup'] = ['/bin', '/share/man/man1'] - - if package.vcs: - vcs_modules.append(module) - else: - modules.append(module) - -modules = vcs_modules + modules -if len(modules) == 1: - pypi_module = modules[0] -else: - pypi_module = { - 'name': output_package, - 'buildsystem': 'simple', - 'build-commands': [], - 'modules': modules, - } - -print() -with open(output_filename, 'w') as output: - if opts.yaml: - class OrderedDumper(yaml.Dumper): - def increase_indent(self, flow=False, indentless=False): - return super(OrderedDumper, self).increase_indent(flow, False) - - def dict_representer(dumper, data): - return dumper.represent_dict(data.items()) - - OrderedDumper.add_representer(OrderedDict, dict_representer) - - output.write("# Generated with flatpak-pip-generator " + " ".join(sys.argv[1:]) + "\n") - yaml.dump(pypi_module, output, Dumper=OrderedDumper) - else: - output.write(json.dumps(pypi_module, indent=4)) - print('Output saved to {}'.format(output_filename)) diff --git a/build-aux/python3-caldav.json b/build-aux/python3-caldav.json new file mode 100644 index 00000000..de47a58a --- /dev/null +++ b/build-aux/python3-caldav.json @@ -0,0 +1,106 @@ +{ + "name": "python3-package-installation", + "buildsystem": "simple", + "build-commands": [ + "pip3 install --verbose --exists-action=i --no-index --find-links=\"file://${PWD}\" --prefix=${FLATPAK_DEST} --no-build-isolation caldav certifi charset-normalizer icalendar idna lxml python-dateutil pytz recurring-ical-events requests six tzlocal urllib3 vobject x-wr-timezone" + ], + "sources": [ + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/f1/e0/0a3762eea7b1d4d855639f7eaee3a0006169fd79d86ae0c660ae5d1d4c60/caldav-1.3.9-py3-none-any.whl", + "sha256": "7c38f1def110809bd50acd3d648f6115b33b8f5395c457215b3b1147c70564d9" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl", + "sha256": "dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/e4/a6/7ee57823d46331ddc37dd00749c95b0edec2c79b15fc0d6e6efb532e89ac/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "sha256": "f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "only-arches": [ + "aarch64" + ] + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/40/26/f35951c45070edc957ba40a5b1db3cf60a9dbb1b350c2d5bef03e01e61de/charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sha256": "753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "only-arches": [ + "x86_64" + ] + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/fb/89/badc6427111cffabb6a462bf447cfff5e9e4c856527ddc030c11020b6cc5/icalendar-5.0.12-py3-none-any.whl", + "sha256": "d873bb859df9c6d0e597b16d247436e0f83f7ac1b90a06429b8393fe8afeba40" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/e5/3e/741d8c82801c347547f8a2a06aa57dbb1992be9e948df2ea0eda2c8b79e8/idna-3.7-py3-none-any.whl", + "sha256": "82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/6e/27/5775154626e0898a82dba7a7ee8bb503c78f64d33e56c3a031bd1bf4e336/lxml-5.2.1-cp311-cp311-manylinux_2_28_aarch64.whl", + "sha256": "3a6b45da02336895da82b9d472cd274b22dc27a5cea1d4b793874eead23dd14f", + "only-arches": [ + "aarch64" + ] + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/be/c3/1765e019344d3f042dfe750eb9a424c0ea2fd43deb6b2ac176b5603a436e/lxml-5.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", + "sha256": "200e63525948e325d6a13a76ba2911f927ad399ef64f57898cf7c74e69b71095", + "only-arches": [ + "x86_64" + ] + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", + "sha256": "a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/9c/3d/a121f284241f08268b21359bd425f7d4825cffc5ac5cd0e1b3d82ffd2b10/pytz-2024.1-py2.py3-none-any.whl", + "sha256": "328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/8a/3c/c1e8d2fb47dfb091d2552ca8bee98aefa7593db3bc713a2d40826547f6ef/recurring_ical_events-2.2.1-py3-none-any.whl", + "sha256": "9e8e0390e7cfe2e7425690e6b858eed635bf7560b44cb52260cd3466fec9cec5" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", + "sha256": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", + "sha256": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/97/3f/c4c51c55ff8487f2e6d0e618dba917e3c3ee2caae6cf0fbb59c9b1876f2e/tzlocal-5.2-py3-none-any.whl", + "sha256": "49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/a2/73/a68704750a7679d0b6d3ad7aa8d4da8e14e151ae82e6fee774e6e0d05ec8/urllib3-2.2.1-py3-none-any.whl", + "sha256": "450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/a2/f2/ea094c009f962bd2fda9851bd54cd32b20721c9228842df2eefc1122aa40/vobject-0.9.7-py2.py3-none-any.whl", + "sha256": "67ebec81ee39fc60b7355ce077f850d5f13d99d08b110fa1abcfdbb516205e20" + }, + { + "type": "file", + "url": "https://files.pythonhosted.org/packages/9d/c6/53227e391c641b891e173b0454f137a21cb969dd58b5171e487e4da7e87e/x_wr_timezone-0.0.7-py3-none-any.whl", + "sha256": "0b5e16f677c8f51ce41087a0b3d4f786c5fdcf78af4f8a75d4d960107dcb6d3a" + } + ] +} \ No newline at end of file diff --git a/build-aux/python3-modules.json b/build-aux/python3-modules.json deleted file mode 100644 index 62466219..00000000 --- a/build-aux/python3-modules.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "name": "python3-modules", - "buildsystem": "simple", - "build-commands": [], - "modules": [ - { - "name": "python3-caldav", - "buildsystem": "simple", - "build-commands": [ - "pip3 install --verbose --exists-action=i --no-index --find-links=\"file://${PWD}\" --prefix=${FLATPAK_DEST} \"caldav\" --no-build-isolation" - ], - "sources": [ - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/f1/e0/0a3762eea7b1d4d855639f7eaee3a0006169fd79d86ae0c660ae5d1d4c60/caldav-1.3.9-py3-none-any.whl", - "sha256": "7c38f1def110809bd50acd3d648f6115b33b8f5395c457215b3b1147c70564d9" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/ba/06/a07f096c664aeb9f01624f858c3add0a4e913d6c96257acb4fce61e7de14/certifi-2024.2.2-py3-none-any.whl", - "sha256": "dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/63/09/c1bc53dab74b1816a00d8d030de5bf98f724c52c1635e07681d312f20be8/charset-normalizer-3.3.2.tar.gz", - "sha256": "f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/56/df/da62437403ceafea8e5b6a03ca08d4c574eb4d13eec6b5dc7018200696e5/icalendar-5.0.11-py3-none-any.whl", - "sha256": "81864971ac43a1b7d0a555dc1b667836ce59fc719a7f845a96f2f03205fb83b9" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl", - "sha256": "c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/2b/b4/bbccb250adbee490553b6a52712c46c20ea1ba533a643f1424b27ffc6845/lxml-5.1.0.tar.gz", - "sha256": "3eea6ed6e6c918e468e693c41ef07f3c3acc310b70ddd9cc72d9ef84bc9564ca" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/36/7a/87837f39d0296e723bb9b62bbb257d0355c7f6128853c78955f57342a56d/python_dateutil-2.8.2-py2.py3-none-any.whl", - "sha256": "961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/9c/3d/a121f284241f08268b21359bd425f7d4825cffc5ac5cd0e1b3d82ffd2b10/pytz-2024.1-py2.py3-none-any.whl", - "sha256": "328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/50/2e/840094f0635bab147fb429ab07352c9e9de142aeec82672ee3c6bdd7befa/recurring_ical_events-2.1.2-py3-none-any.whl", - "sha256": "b510d6d46e54381df1d6b35fcf2a727b71ecc510c85701b15c57312a5f172eff" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", - "sha256": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/97/3f/c4c51c55ff8487f2e6d0e618dba917e3c3ee2caae6cf0fbb59c9b1876f2e/tzlocal-5.2-py3-none-any.whl", - "sha256": "49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/88/75/311454fd3317aefe18415f04568edc20218453b709c63c58b9292c71be17/urllib3-2.2.0-py3-none-any.whl", - "sha256": "ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/da/ce/27c48c0e39cc69ffe7f6e3751734f6073539bf18a0cfe564e973a3709a52/vobject-0.9.6.1.tar.gz", - "sha256": "96512aec74b90abb71f6b53898dd7fe47300cc940104c4f79148f0671f790101" - }, - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/f0/e0/bb85ef1170f3d7fc6b5d14452426e14e0f6ec2e0650bb882895cbc8c43db/x_wr_timezone-0.0.6-py3-none-any.whl", - "sha256": "26d98b5f5ae190b68df8b9b9856c4867389956f5b295e0e14f632d7341b60f67" - } - ] - }, - { - "name": "python3-lxml", - "buildsystem": "simple", - "build-commands": [ - "pip3 install --verbose --exists-action=i --no-index --find-links=\"file://${PWD}\" --prefix=${FLATPAK_DEST} \"lxml\" --ignore-installed --no-build-isolation" - ], - "sources": [ - { - "type": "file", - "url": "https://files.pythonhosted.org/packages/2b/b4/bbccb250adbee490553b6a52712c46c20ea1ba533a643f1424b27ffc6845/lxml-5.1.0.tar.gz", - "sha256": "3eea6ed6e6c918e468e693c41ef07f3c3acc310b70ddd9cc72d9ef84bc9564ca" - } - ] - } - ] -} \ No newline at end of file diff --git a/build-aux/req2flatpak.py b/build-aux/req2flatpak.py new file mode 100755 index 00000000..5982c044 --- /dev/null +++ b/build-aux/req2flatpak.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python3 + +# req2flatpak is MIT-licensed. +# +# Copyright 2022 johannesjh +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +req2flatpak converts python package requirements to a flatpak build module. + +The req2flatpak script takes python package requirements as input, e.g., as +``requirements.txt`` file. It allows to specify the target platform’s +python version and architecture. The script outputs an automatically +generated ``flatpak-builder`` build module. The build module, if included +into a flatpak-builder build manifest, will install the python packages +using pip. +""" + +import argparse +import json +import logging +import pathlib +import re +import shelve +import sys +import urllib.request +from contextlib import nullcontext, suppress +from dataclasses import asdict, dataclass, field +from itertools import product +from typing import ( + Any, + Dict, + FrozenSet, + Generator, + Hashable, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + Union, +) +from urllib.parse import urlparse + +import pkg_resources + +logger = logging.getLogger(__name__) + +try: + import yaml +except ImportError: + yaml = None # type: ignore + +# ============================================================================= +# Helper functions / semi vendored code +# ============================================================================= + +try: + # added with py 3.8 + from functools import cached_property # type: ignore[attr-defined] +except ImportError: + # Inspired by the implementation in the standard library + # pylint: disable=invalid-name,too-few-public-methods + class cached_property: # type: ignore[no-redef] + """A property-like wrapper that caches the value.""" + + def __init__(self, func): + """Init.""" + self.func = func + self.attrname = None + self.__doc__ = func.__doc__ + + def __set_name__(self, owner, name): + """Set name of the attribute this instance is assigned to.""" + self.attrname = name + + def __get__(self, instance, owner=None): + """Return value if this is set as an attribute.""" + if not self.attrname: + raise TypeError("cached_property must be used as an attribute.") + _None = object() + cache = instance.__dict__ + val = cache.get(self.attrname, _None) + if val is _None: + val = self.func(instance) + cache[self.attrname] = val + return val + + +class InvalidWheelFilename(Exception): + """An invalid wheel filename was found, users should refer to PEP 427.""" + + +try: + # use packaging.tags functionality if available + from packaging.utils import parse_wheel_filename + + def tags_from_wheel_filename(filename: str) -> Set[str]: + """ + Parses a wheel filename into a list of compatible platform tags. + + Implemented using functionality from ``packaging.utils.parse_wheel_filename``. + """ + _, _, _, tags = parse_wheel_filename(filename) + return {str(tag) for tag in tags} + +except ModuleNotFoundError: + # fall back to a local implementation + # that is heavily inspired by / almost vendored from the `packaging` package: + def tags_from_wheel_filename(filename: str) -> Set[str]: + """ + Parses a wheel filename into a list of compatible platform tags. + + Implemented as (semi-)vendored functionality in req2flatpak. + """ + Tag = Tuple[str, str, str] + + # the following code is based on packaging.tags.parse_tag, + # it is needed for the parse_wheel_filename function: + def parse_tag(tag: str) -> FrozenSet[Tag]: + tags: Set[Tag] = set() + interpreters, abis, platforms = tag.split("-") + for interpreter in interpreters.split("."): + for abi in abis.split("."): + for platform_ in platforms.split("."): + tags.add((interpreter, abi, platform_)) + return frozenset(tags) + + # the following code is based on packaging.utils.parse_wheel_filename: + # pylint: disable=redefined-outer-name + def parse_wheel_filename( + wheel_filename: str, + ) -> Iterable[Tag]: + if not wheel_filename.endswith(".whl"): + raise InvalidWheelFilename( + "Error parsing wheel filename: " + "Invalid wheel filename (extension must be '.whl'): " + f"{wheel_filename}" + ) + wheel_filename = wheel_filename[:-4] + dashes = wheel_filename.count("-") + if dashes not in (4, 5): + raise InvalidWheelFilename( + "Error parsing wheel filename: " + "Invalid wheel filename (wrong number of parts): " + f"{wheel_filename}" + ) + parts = wheel_filename.split("-", dashes - 2) + return parse_tag(parts[-1]) + + return {"-".join(tag_tuple) for tag_tuple in parse_wheel_filename(filename)} + + +# ============================================================================= +# Data Structures +# ============================================================================= + + +@dataclass(frozen=True) +class Platform: + """Represents a target platform for python package installations.""" + + python_version: List[str] + """A list of python version numbers, similar to ``platform.python_version_tuple()``.""" + + python_tags: List[str] + """A list of platform tags, similar to ``packaging.tags.sys_tags()``.""" + + +@dataclass(frozen=True) +class Requirement: + """Represents a python package requirement.""" + + package: str + """A python package name.""" + + version: str + """The exact version of the package.""" + + +@dataclass(frozen=True) +class Download(Requirement): + """Represents a python package download.""" + + filename: str + url: str + sha256: str + + @cached_property + def is_wheel(self): + """True if this download is a wheel.""" + return self.filename.endswith(".whl") + + @cached_property + def is_sdist(self): + """True if this download is a source distribution.""" + return not self.is_wheel and not self.filename.endswith(".egg") + + @cached_property + def tags(self) -> Optional[Set[str]]: + """Returns a list of tags that this download is compatible for.""" + # https://packaging.pypa.io/en/latest/utils.html#packaging.utils.parse_wheel_filename + # https://packaging.pypa.io/en/latest/utils.html#packaging.utils.parse_sdist_filename + if not self.is_sdist and self.filename.endswith(".whl"): + return tags_from_wheel_filename(self.filename) + return None + + @cached_property + def arch(self) -> Optional[str]: + """Returns a wheel's target architecture, and None for sdists.""" + if not self.is_sdist and self.is_wheel and self.tags: + if any(tag.endswith("x86_64") for tag in self.tags): + return "x86_64" + if any(tag.endswith("aarch64") for tag in self.tags): + return "aarch64" + return None + + def __lt__(self, other): + """Makes this class sortable.""" + # Note: Implementing __lt__ is sufficient to make a class sortable, + # see, e.g., https://stackoverflow.com/a/7152796 + + def sort_keys(download: Download) -> Tuple[str, str, str]: + """A tuple of package, version and architecture is used as key for sorting.""" + return download.package, download.version, download.arch or "" + + return sort_keys(self) < sort_keys(other) + + +@dataclass(frozen=True) +class Release(Requirement): + """Represents a package release as name, version, and downloads.""" + + package: str + version: str + downloads: List[Download] = field(default_factory=list) + + +# ============================================================================= +# Operations +# ============================================================================= + + +class PlatformFactory: + """Provides methods for creating platform objects.""" + + @staticmethod + def _get_current_python_version() -> List[str]: + # pylint: disable=import-outside-toplevel + import platform + + return list(platform.python_version_tuple()) + + @staticmethod + def _get_current_python_tags() -> List[str]: + try: + # pylint: disable=import-outside-toplevel + import packaging.tags + + tags = [str(tag) for tag in packaging.tags.sys_tags()] + return tags + except ModuleNotFoundError as e: + logger.warning( + 'Error trying to import the "packaging" package.', exc_info=e + ) + return [] + + @classmethod + def from_current_interpreter(cls) -> Platform: + """ + Returns a platform object that describes the current interpreter and system. + + This method requires the ``packaging`` package to be installed + because functionality from this package is used to generate the list of + tags that are supported by the current platform. + + The platform object returned by this method obviously depends on the + specific python interpreter and system architecture used to run the + req2flatpak script. The reason why is that this method reads platform + properties from the current interpreter and system. + """ + return Platform( + python_version=cls._get_current_python_version(), + python_tags=cls._get_current_python_tags(), + ) + + @classmethod + def from_string(cls, platform_string: str) -> Optional[Platform]: + """ + Returns a platform object by parsing a platform string. + + :param platform_string: A string specifying python version and system + architecture. The string format is + "{python_version}-{system_architecture}". + For example: "cp39-x86_64" or "cp310-aarch64". + Acceptable values are the same as in + :py:meth:`~req2flatpak.PlatformFactory.from_python_version_and_arch`. + """ + try: + _, minor, arch = re.match( # type: ignore[union-attr] + r"^(?:py|cp)?(\d)(\d+)-(.*)$", platform_string + ).groups() + return cls.from_python_version_and_arch(minor_version=int(minor), arch=arch) + except AttributeError: + logger.warning("Could not parse platform string %s", platform_string) + return None + + @classmethod + def from_python_version_and_arch( + cls, minor_version: Optional[int] = None, arch="x86_64" + ) -> Platform: + """ + Returns a platform object that roughly describes a cpython installation on linux. + + The tags in the platform object are a rough approximation, trying to match what + `packaging.tags.sys_tags` would return if invoked on a linux system with cpython. + No guarantees are made about how closely this approximation matches a real system. + + :param minor_version: the python 3 minor version, specified as int. + Defaults to the current python version. + :param arch: either "x86_64" or "aarch64". + """ + if not minor_version: + minor_version = int(cls._get_current_python_version()[1]) + assert arch in ["x86_64", "aarch64"] + return Platform( + python_version=["3", str(minor_version)], + python_tags=list( + cls._cp3_linux_tags(minor_version=minor_version, arch=arch) + ), + ) + + @classmethod + def _cp3_linux_tags( + cls, minor_version: Optional[int] = None, arch="x86_64" + ) -> Generator[str, None, None]: + """Yields python platform tags for cpython3 on linux.""" + # pylint: disable=too-many-branches + + assert minor_version is not None + assert arch in ["x86_64", "aarch64"] + + def seq(start: int, end: int) -> Iterable[int]: + """Returns a range of numbers, from start to end, in steps of +/- 1.""" + step = 1 if start < end else -1 + return range(start, end + step, step) + + cache = set() + + def dedup(obj: Hashable): + if obj in cache: + return None + cache.add(obj) + return obj + + platforms = [f"manylinux_2_{v}" for v in seq(35, 17)] + ["manylinux2014"] + if arch == "x86_64": + platforms += ( + [f"manylinux_2_{v}" for v in seq(16, 12)] + + ["manylinux2010"] + + [f"manylinux_2_{v}" for v in seq(11, 5)] + + ["manylinux1"] + ) + platforms += ["linux"] + platform_tags = [f"{platform}_{arch}" for platform in platforms] + + # current cpython version, all abis, all platforms: + for py in [f"cp3{minor_version}"]: + for abi in [f"cp3{minor_version}", "abi3", "none"]: + for platform in platform_tags: + yield dedup(f"{py}-{abi}-{platform}") + + # older cpython versions, abi3, all platforms: + for py in [f"cp3{v}" for v in seq(minor_version - 1, 2)]: + for abi in ["abi3"]: + for platform in platform_tags: + yield dedup(f"{py}-{abi}-{platform}") + + # current python version, abi=none, all platforms: + for py in [f"py3{minor_version}"]: + for abi in ["none"]: + for platform in platform_tags: + yield dedup(f"{py}-{abi}-{platform}") + + # current major python version (py3), abi=none, all platforms: + for py in ["py3"]: + for abi in ["none"]: + for platform in platform_tags: + yield dedup(f"{py}-{abi}-{platform}") + + # older python versions, abi=none, all platforms: + for py in [f"py3{v}" for v in seq(minor_version - 1, 0)]: + for abi in ["none"]: + for platform in platform_tags: + yield dedup(f"{py}-{abi}-{platform}") + + # current python version, abi=none, platform=any + yield f"py3{minor_version}-none-any" + + # current major python version, abi=none, platform=any + yield "py3-none-any" + + # older python versions, abi=none, platform=any + for py in [f"py3{v}" for v in seq(minor_version - 1, 0)]: + yield f"{py}-none-any" + + +class RequirementsParser: + """ + Parses requirements.txt files in a very simple way. + + This methods expects all versions to be pinned, and it does not + resolve dependencies. + """ + + # based on: https://stackoverflow.com/a/59971236 + # using functionality from pkg_resources.parse_requirements + + @classmethod + def parse_string(cls, requirements_txt: str) -> List[Requirement]: + """Parses requirements.txt string content into a list of Requirement objects.""" + + def validate_requirement(req: pkg_resources.Requirement) -> None: + assert ( + len(req.specs) == 1 + ), "Error parsing requirements: A single version number must be specified." + assert ( + req.specs[0][0] == "==" + ), "Error parsing requirements: The exact version must specified as 'package==version'." + + def make_requirement(req: pkg_resources.Requirement) -> Requirement: + validate_requirement(req) + return Requirement(package=req.project_name, version=req.specs[0][1]) + + reqs = pkg_resources.parse_requirements(requirements_txt) + return [make_requirement(req) for req in reqs] + + @classmethod + def parse_file(cls, file) -> List[Requirement]: + """Parses a requirements.txt file into a list of Requirement objects.""" + if hasattr(file, "read"): + req_txt = file.read() + else: + req_txt = pathlib.Path(file).read_text(encoding="utf-8") + return cls.parse_string(req_txt) + + +# Cache typealias +# This is meant for caching responses when querying package information. +# A cache can either be a dict for in-memory caching, or a shelve.Shelf +Cache = Union[dict, shelve.Shelf] + + +class PypiClient: + """Queries package information from the PyPi package index.""" + + cache: Cache = {} + """A dict-like object for caching responses from PyPi.""" + + @classmethod + def _query_from_cache(cls, url) -> Optional[str]: + try: + return cls.cache[url] + except KeyError: + return None + + @classmethod + def _query_from_pypi(cls, url) -> str: + # url scheme might be `file:/` + if not urlparse(url).scheme == "https": + raise ValueError("URL scheme not `https`.") + with urllib.request.urlopen(url) as response: # nosec: B310 + json_string = response.read().decode("utf-8") + cls.cache[url] = json_string + return json_string + + @classmethod + def _query(cls, url) -> str: + return cls._query_from_cache(url) or cls._query_from_pypi(url) + + @classmethod + def get_release(cls, req: Requirement) -> Release: + """Queries pypi regarding available releases for this requirement.""" + url = f"https://pypi.org/pypi/{req.package}/{req.version}/json" + json_string = cls._query(url) + json_dict = json.loads(json_string) + return Release( + package=req.package, + version=req.version, + downloads=[ + Download( + package=req.package, + version=req.version, + filename=url["filename"], + url=url["url"], + sha256=url["digests"]["sha256"], + ) + for url in json_dict["urls"] + ], + ) + + @classmethod + def get_releases(cls, reqs: Iterable[Requirement]) -> List[Release]: + """Queries pypi regarding available releases for these requirements.""" + return [cls.get_release(req) for req in reqs] + + +class DownloadChooser: + """ + Provides methods for choosing package downloads. + + This class implements logic for filtering wheel and sdist downloads + that are compatible with a given target platform. + """ + + @classmethod + def matches(cls, download: Download, platform_tag: str) -> bool: + """Returns whether a download is compatible with a target platform tag.""" + if download.is_sdist: + return True + + return platform_tag in (download.tags or []) + + @classmethod + def downloads( + cls, + release: Release, + platform: Platform, + wheels_only=False, + sdists_only=False, + ) -> Iterator[Download]: + """ + Yields suitable downloads for a specific platform. + + The order of downloads matches the order of platform tags, i.e., + preferred downloads are returned first. + """ + cache = set() + for platform_tag, download in product(platform.python_tags, release.downloads): + if download in cache: + continue + if wheels_only and not download.is_wheel: + continue + if sdists_only and not download.is_sdist: + continue + if cls.matches(download, platform_tag): + cache.add(download) + yield download + + @classmethod + def wheel( + cls, + release: Release, + platform: Platform, + ) -> Optional[Download]: + """Returns the preferred wheel download for this release.""" + try: + return next(cls.downloads(release, platform, wheels_only=True)) + except StopIteration: + return None + + @classmethod + def sdist(cls, release: Release) -> Optional[Download]: + """Returns the source package download for this release.""" + try: + return next(filter(lambda d: d.is_sdist, release.downloads)) + except StopIteration: + return None + + @classmethod + def wheel_or_sdist(cls, release: Release, platform: Platform) -> Optional[Download]: + """Returns a wheel or an sdist for this release, in this order of preference.""" + return cls.wheel(release, platform) or cls.sdist(release) + + @classmethod + def sdist_or_wheel(cls, release: Release, platform: Platform) -> Optional[Download]: + """Returns an sdist or a wheel for this release, in this order of preference.""" + return cls.sdist(release) or cls.wheel(release, platform) + + +class FlatpakGenerator: + """Provides methods for generating a flatpak-builder build module.""" + + @staticmethod + def build_module( + requirements: Iterable[Requirement], + downloads: Iterable[Download], + module_name="python3-package-installation", + pip_install_template: str = "pip3 install --verbose --exists-action=i " + '--no-index --find-links="file://${PWD}" ' + "--prefix=${FLATPAK_DEST} --no-build-isolation ", + ) -> dict: + """Generates a build module for inclusion in a flatpak-builder build manifest.""" + + def source(download: Download) -> dict: + source: Dict[str, Any] = { + "type": "file", + "url": download.url, + "sha256": download.sha256, + } + if download.arch: + source["only-arches"] = [download.arch] + return source + + def sources(downloads: Iterable[Download]) -> List[dict]: + return [source(download) for download in sorted(downloads)] + + return { + "name": module_name, + "buildsystem": "simple", + "build-commands": [ + pip_install_template + " ".join([req.package for req in requirements]) + ], + "sources": sources(downloads), + } + + @classmethod + def build_module_as_str(cls, *args, **kwargs) -> str: + """ + Generate JSON build module for inclusion in a flatpak-builder build manifest. + + The args and kwargs are the same as in + :py:meth:`~req2flatpak.FlatpakGenerator.build_module` + """ + return json.dumps(cls.build_module(*args, **kwargs), indent=4) + + @classmethod + def build_module_as_yaml_str(cls, *args, **kwargs) -> str: + """ + Generate YAML build module for inclusion in a flatpak-builder build manifest. + + The args and kwargs are the same as in + :py:meth:`~req2flatpak.FlatpakGenerator.build_module` + """ + # optional dependency, not imported at top + if not yaml: + raise ImportError( + "Package `pyyaml` has to be installed for the yaml format." + ) + + return yaml.dump( + cls.build_module(*args, **kwargs), default_flow_style=False, sort_keys=False + ) + + +# ============================================================================= +# CLI commandline interface +# ============================================================================= + + +def cli_parser() -> argparse.ArgumentParser: + """Returns the req2flatpak commandline interface parser.""" + parser = argparse.ArgumentParser( + description="req2flatpak generates a flatpak-builder build module for installing required python packages." + ) + parser.add_argument( + "--requirements", + nargs="*", + help="One or more requirements can be specified as commandline arguments, e.g., 'pandas==1.4.4'.", + ) + parser.add_argument( + "--requirements-file", + "-r", + nargs="?", + type=argparse.FileType("r"), + help="Requirements can be read from a specified requirements.txt file.", + ) + parser.add_argument( + "--target-platforms", + "-t", + nargs="+", + help="Target platforms can be specified as, e.g., '39-x86_64' or '310-aarch64'.", + ) + parser.add_argument( + "--cache", + action="store_true", + default=False, + help="Uses a persistent cache when querying pypi.", + ) + parser.add_argument( + "--yaml", + action="store_true", + help="Write YAML instead of the default JSON. Needs the 'pyyaml' package.", + ) + + parser.add_argument( + "--outfile", + "-o", + nargs="?", + type=argparse.FileType("w"), + default=sys.stdout, + help=""" + By default, writes JSON but specify a '.yaml' extension and YAML + will be written instead, provided you have the 'pyyaml' package. + """, + ) + parser.add_argument( + "--platform-info", + action="store_true", + default=False, + help="Prints information about the current platform.", + ) + parser.add_argument( + "--installed-packages", + action="store_true", + default=False, + help="Prints installed packages in requirements.txt format.", + ) + return parser + + +def main(): # pylint: disable=too-many-branches + """Main function that provides req2flatpak's commandline interface.""" + + # process commandline arguments + parser = cli_parser() + options = parser.parse_args() + + # stream output to a file or to stdout + if hasattr(options.outfile, "write"): + output_stream = options.outfile + if pathlib.Path(output_stream.name).suffix.casefold() in (".yaml", ".yml"): + options.yaml = True + else: + output_stream = sys.stdout + + if options.yaml and not yaml: + parser.error( + "Outputing YAML requires 'pyyaml' package: try 'pip install pyyaml'" + ) + + # print platform info if requested, and exit + if options.platform_info: + info = asdict(PlatformFactory.from_current_interpreter()) + if options.yaml: + yaml.dump(info, output_stream, default_flow_style=False, sort_keys=False) + else: + json.dump(info, output_stream, indent=4) + parser.exit() + + # print installed packages if requested, and exit + if options.installed_packages: + # pylint: disable=not-an-iterable + pkgs = {p.key: p.version for p in pkg_resources.working_set} + for pkg, version in pkgs.items(): + print(f"{pkg}=={version}", file=output_stream) + parser.exit() + + # parse requirements + requirements = [] + with suppress(AttributeError): + if options.requirements: + requirements += RequirementsParser.parse_string( + "\n".join(options.requirements) + ) + if options.requirements_file: + requirements += RequirementsParser.parse_file(options.requirements_file) + if not requirements: + parser.error( + "Error parsing requirements: At least one requirement must be specified." + ) + + # parse target platforms + if not options.target_platforms: + parser.error( + "Error parsing target platforms. " + "Missing commandline argument, at least one target platform must " + "be specified as, e.g., '39-x86_64' or '310-aarch64'." + ) + platforms = [ + PlatformFactory.from_string(platform) for platform in options.target_platforms + ] + if not platforms: + parser.error( + "Error parsing target platforms. " + "At least one target platform must be specified " + "as, e.g., '39-x86_64' or '310-aarch64'." + ) + + # query released downloads from PyPi, optionally using a shelve.Shelf to cache responses: + with ( + shelve.open("pypi_cache") if options.cache else nullcontext() # nosec: B301 + ) as cache: + PypiClient.cache = cache or {} + releases = PypiClient.get_releases(requirements) + + # choose suitable downloads for the target platforms + downloads = { + DownloadChooser.wheel_or_sdist(release, platform) + for release in releases + for platform in platforms + if platform + } + + # generate flatpak-builder build module + if options.yaml: + try: + name = pathlib.Path(__file__).name + except NameError: + name = "req2flatpak" + output_stream.write(f"# Generated by {name} {' '.join(sys.argv[1:])}\n") + output_stream.write( + FlatpakGenerator.build_module_as_yaml_str(requirements, downloads) + ) + else: + # write json + output_stream.write( + FlatpakGenerator.build_module_as_str(requirements, downloads) + ) + + +if __name__ == "__main__": + main() diff --git a/build-aux/requirements.txt b/build-aux/requirements.txt new file mode 100644 index 00000000..35d31515 --- /dev/null +++ b/build-aux/requirements.txt @@ -0,0 +1,15 @@ +caldav==1.3.9 +certifi==2024.2.2 +charset-normalizer==3.3.2 +icalendar==5.0.12 +idna==3.7 +lxml==5.2.1 +python-dateutil==2.9.0.post0 +pytz==2024.1 +recurring-ical-events==2.2.1 +requests==2.31.0 +six==1.16.0 +tzlocal==5.2 +urllib3==2.2.1 +vobject==0.9.7 +x-wr-timezone==0.0.7 diff --git a/build-aux/update_python_deps.sh b/build-aux/update_python_deps.sh index 721d4d08..56656c06 100755 --- a/build-aux/update_python_deps.sh +++ b/build-aux/update_python_deps.sh @@ -1,7 +1,3 @@ #!/usr/bin/bash -DEPS="caldav lxml pip" -IGNORE_INSTALLED="lxml,pip" -RUNTIME="org.gnome.Sdk//46" - -build-aux/flatpak-pip-generator.py --ignore-installed=$IGNORE_INSTALLED --output build-aux/python3-modules $DEPS \ No newline at end of file +build-aux/req2flatpak.py --requirements-file requirements.txt --target-platforms 311-x86_64 311-aarch64 > manifest.json diff --git a/io.github.mrvladus.List.Devel.json b/io.github.mrvladus.List.Devel.json index a8f372eb..4e1863fd 100644 --- a/io.github.mrvladus.List.Devel.json +++ b/io.github.mrvladus.List.Devel.json @@ -67,7 +67,7 @@ } ] }, - "build-aux/python3-modules.json", + "build-aux/python3-caldav.json", { "name": "errands", "buildsystem": "meson",