From d17ad1042d9fcc6bd252762132726f62e284aa9b Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Tue, 16 Nov 2021 10:44:42 -0800 Subject: [PATCH 001/401] Add tag parameter to workflow inputs --- lib/galaxy/workflow/modules.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index d00780a2e821..97930d0183ee 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -734,6 +734,7 @@ def get_runtime_inputs(self, connections=None): input_param = DataToolParameter(None, data_src, self.trans) return dict(input=input_param) + def get_inputs(self): parameter_def = self._parse_state_into_dict() optional = parameter_def["optional"] @@ -752,6 +753,7 @@ class InputDataCollectionModule(InputModule): def get_inputs(self): parameter_def = self._parse_state_into_dict() collection_type = parameter_def["collection_type"] + tag = parameter_def["tag"] optional = parameter_def["optional"] collection_type_source = dict(name="collection_type", label="Collection type", type="text", value=collection_type) collection_type_source["options"] = [ @@ -760,10 +762,13 @@ def get_inputs(self): {"value": "list:paired", "label": "List of Dataset Pairs"}, ] input_collection_type = TextToolParameter(None, collection_type_source) + tag_source = dict(name="tag", label="Tag filter", type="text", value=tag, help="Tags to automatically filter inputs") + input_tag = TextToolParameter(None, tag_source) inputs = {} inputs["collection_type"] = input_collection_type inputs["optional"] = optional_param(optional) inputs["format"] = format_param(self.trans, parameter_def.get("format")) + inputs["tag"] = input_tag return inputs def get_runtime_inputs(self, **kwds): @@ -803,6 +808,11 @@ def _parse_state_into_dict(self): else: collection_type = self.default_collection_type state_as_dict["collection_type"] = collection_type + if "tag" in inputs: + tag = inputs["tag"] + else: + tag = None + state_as_dict["tag"] = tag return state_as_dict From 43fbc360e75f670a35cfc9e3e2a99a7b8c54cb26 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Wed, 17 Nov 2021 12:06:30 +0530 Subject: [PATCH 002/401] Add interface definitions and test case for abstract vault --- lib/galaxy/security/vault.py | 69 ++++++++++++++++++++++++++++++++ test/unit/security/test_vault.py | 6 +++ 2 files changed, 75 insertions(+) create mode 100644 lib/galaxy/security/vault.py create mode 100644 test/unit/security/test_vault.py diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py new file mode 100644 index 000000000000..04040866aa70 --- /dev/null +++ b/lib/galaxy/security/vault.py @@ -0,0 +1,69 @@ +from abc import ABC +import os +import yaml + + +class UnknownVaultTypeException(Exception): + pass + + +class Vault(ABC): + + def read_secret(self, path: str) -> dict: + pass + + def write_secret(self, path: str, value: dict) -> None: + pass + + +class HashicorpVault(Vault): + pass + + +class DatabaseVault(Vault): + pass + + +class CustosVault(Vault): + pass + + +class UserVaultWrapper(Vault): + + def __init__(self, vault: Vault, user): + self.vault = vault + self.user = user + + def read_secret(self, path: str) -> dict: + return self.vault.read_secret(f"user/{self.user.id}/{path}") + + def write_secret(self, path: str, value: dict) -> None: + return self.vault.write_secret(f"user/{self.user.id}/{path}", value) + + +class VaultFactory(object): + + @staticmethod + def load_vault_config(vault_conf_yml): + if os.path.exists(vault_conf_yml): + with open(vault_conf_yml) as f: + return yaml.safe_load(f) + return None + + @staticmethod + def from_vault_type(vault_type, cfg): + if vault_type == "hashicorp": + return HashicorpVault(cfg) + elif vault_type == "database": + return DatabaseVault(cfg) + elif vault_type == "custos": + return CustosVault(cfg) + else: + raise UnknownVaultTypeException(f"Unknown vault type: {vault_type}") + + @staticmethod + def from_app_config(config): + vault_config = VaultFactory.load_vault_config(config.vault_config_file) + if vault_config: + return VaultFactory.from_vault_type(vault_config.get('type'), vault_config) + return None diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py new file mode 100644 index 000000000000..10894d25e209 --- /dev/null +++ b/test/unit/security/test_vault.py @@ -0,0 +1,6 @@ +class VaultTestBase: + + def test_read_write_secret(self): + self.assertIsNone(self.vault.read_secret("my/test/secret"), "Vault secret should initially be empty") + self.vault.write_secret("my/test/secret", {"value": "hello world"}) + self.assertEqual(self.vault.read_secret("my/test/secret"), {"value": "hello world"}) From 1a26b450fc5510fa99b4b7434ca4c16805b1a3e1 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Wed, 17 Nov 2021 12:07:42 +0530 Subject: [PATCH 003/401] Add initial implementation and tests for hashicorp vault --- lib/galaxy/security/vault.py | 22 ++++++++++++++++++- .../fixtures/vault_conf_hashicorp.yaml | 3 +++ test/unit/security/test_vault.py | 22 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 test/unit/security/fixtures/vault_conf_hashicorp.yaml diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index 04040866aa70..59aaaf3b802d 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -1,6 +1,10 @@ from abc import ABC import os import yaml +try: + import hvac +except ImportError: + hvac = None class UnknownVaultTypeException(Exception): @@ -17,7 +21,23 @@ def write_secret(self, path: str, value: dict) -> None: class HashicorpVault(Vault): - pass + + def __init__(self, config): + if not hvac: + raise UnknownVaultTypeException("Hashicorp vault library 'hvac' is not available. Make sure hvac is installed.") + self.vault_address = config.get('vault_address') + self.vault_token = config.get('vault_token') + self.client = hvac.Client(url=self.vault_address, token=self.vault_token) + + def read_secret(self, path: str) -> dict: + try: + response = self.client.secrets.kv.read_secret_version(path=path) + return response['data']['data'] + except hvac.exceptions.InvalidPath: + return None + + def write_secret(self, path: str, value: dict) -> None: + self.client.secrets.kv.v2.create_or_update_secret(path=path, secret=value) class DatabaseVault(Vault): diff --git a/test/unit/security/fixtures/vault_conf_hashicorp.yaml b/test/unit/security/fixtures/vault_conf_hashicorp.yaml new file mode 100644 index 000000000000..43ef3c9ffbc5 --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_hashicorp.yaml @@ -0,0 +1,3 @@ +type: hashicorp +vault_address: http://localhost:8200 +vault_token: galaxy_test_token diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py index 10894d25e209..49904ca2e8de 100644 --- a/test/unit/security/test_vault.py +++ b/test/unit/security/test_vault.py @@ -1,6 +1,28 @@ +import os +import unittest + +from galaxy.security import vault + + class VaultTestBase: def test_read_write_secret(self): self.assertIsNone(self.vault.read_secret("my/test/secret"), "Vault secret should initially be empty") self.vault.write_secret("my/test/secret", {"value": "hello world"}) self.assertEqual(self.vault.read_secret("my/test/secret"), {"value": "hello world"}) + + +VAULT_CONF_HASHICORP = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_hashicorp.yaml") + + +class MockAppConfig: + + def __init__(self, vault_conf_file): + self.vault_config_file = vault_conf_file + + +class TestHashicorpVault(VaultTestBase, unittest.TestCase): + + def setUp(self) -> None: + config = MockAppConfig(vault_conf_file=VAULT_CONF_HASHICORP) + self.vault = vault.VaultFactory.from_app_config(config) From f207b23626a6a91a25b53d7a6e740041642b4979 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Thu, 18 Nov 2021 01:06:48 +0530 Subject: [PATCH 004/401] Add database based vault implementation --- lib/galaxy/model/__init__.py | 10 +++++ .../migrate/versions/180_add_vault_table.py | 25 +++++++++++ lib/galaxy/security/vault.py | 37 ++++++++++++--- .../fixtures/vault_conf_database.yaml | 13 ++++++ .../vault_conf_database_invalid_keys.yaml | 12 +++++ .../fixtures/vault_conf_database_rotated.yaml | 14 ++++++ test/unit/security/test_vault.py | 45 ++++++++++++++++--- 7 files changed, 144 insertions(+), 12 deletions(-) create mode 100644 lib/galaxy/model/migrate/versions/180_add_vault_table.py create mode 100644 test/unit/security/fixtures/vault_conf_database.yaml create mode 100644 test/unit/security/fixtures/vault_conf_database_invalid_keys.yaml create mode 100644 test/unit/security/fixtures/vault_conf_database_rotated.yaml diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ae93c59c3775..838468304346 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8420,6 +8420,16 @@ class LibraryDatasetCollectionAnnotationAssociation(Base, RepresentById): user = relationship('User') +class Vault(Base, RepresentById): + __tablename__ = 'vault' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + key = Column(Text, index=True, unique=True) + value = Column(Text) + + # Item rating classes. class ItemRatingAssociation(Base): __abstract__ = True diff --git a/lib/galaxy/model/migrate/versions/180_add_vault_table.py b/lib/galaxy/model/migrate/versions/180_add_vault_table.py new file mode 100644 index 000000000000..68a575c8bd23 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/180_add_vault_table.py @@ -0,0 +1,25 @@ +import datetime + +from sqlalchemy import Column, DateTime, Integer, MetaData, Table, Text + +now = datetime.datetime.utcnow +meta = MetaData() + +vault = Table( + 'vault', meta, + Column('id', Integer, primary_key=True), + Column("create_time", DateTime, default=now), + Column("update_time", DateTime, default=now, onupdate=now), + Column('key', Text, index=True, unique=True), + Column('value', Text), +) + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + vault.create() + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + vault.drop() diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index 59aaaf3b802d..b6ad211dadc8 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -1,4 +1,6 @@ from abc import ABC +from cryptography.fernet import Fernet, MultiFernet +import json import os import yaml try: @@ -6,6 +8,8 @@ except ImportError: hvac = None +from galaxy import model + class UnknownVaultTypeException(Exception): pass @@ -41,7 +45,28 @@ def write_secret(self, path: str, value: dict) -> None: class DatabaseVault(Vault): - pass + + def __init__(self, sa_session, config): + self.sa_session = sa_session + self.encryption_keys = config.get('encryption_keys') + + def _get_multi_fernet(self) -> MultiFernet: + fernet_keys = [Fernet(key.encode('utf-8')) for key in self.encryption_keys] + return MultiFernet(fernet_keys) + + def read_secret(self, path: str) -> dict: + key_obj = self.sa_session.query(model.Vault).filter_by(key=path).first() + if key_obj: + f = self._get_multi_fernet() + return json.loads(f.decrypt(key_obj.value.encode('utf-8')).decode('utf-8')) + return None + + def write_secret(self, path: str, value: dict) -> None: + f = self._get_multi_fernet() + token = f.encrypt(json.dumps(value).encode('utf-8')) + vault_entry = model.Vault(key=path, value=token.decode('utf-8')) + self.sa_session.add(vault_entry) + self.sa_session.flush() class CustosVault(Vault): @@ -71,19 +96,19 @@ def load_vault_config(vault_conf_yml): return None @staticmethod - def from_vault_type(vault_type, cfg): + def from_vault_type(app, vault_type, cfg): if vault_type == "hashicorp": return HashicorpVault(cfg) elif vault_type == "database": - return DatabaseVault(cfg) + return DatabaseVault(app.model.context, cfg) elif vault_type == "custos": return CustosVault(cfg) else: raise UnknownVaultTypeException(f"Unknown vault type: {vault_type}") @staticmethod - def from_app_config(config): - vault_config = VaultFactory.load_vault_config(config.vault_config_file) + def from_app_config(app): + vault_config = VaultFactory.load_vault_config(app.config.vault_config_file) if vault_config: - return VaultFactory.from_vault_type(vault_config.get('type'), vault_config) + return VaultFactory.from_vault_type(app, vault_config.get('type'), vault_config) return None diff --git a/test/unit/security/fixtures/vault_conf_database.yaml b/test/unit/security/fixtures/vault_conf_database.yaml new file mode 100644 index 000000000000..9064f7a4734e --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_database.yaml @@ -0,0 +1,13 @@ +type: database +# Encryption keys must be valid fernet keys +# To generate a valid key: +# >>> from cryptography.fernet import Fernet +# >>> Fernet.generate_key() +# b'pZDP8_baVs3oWT4597HJWCysm49j-XELONQ-EdoU0DE=' +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: + - 5RrT94ji178vQwha7TAmEix7DojtsLlxVz8Ef17KWgg= + - iNdXd7tRjLnSqRHxuhqQ98GTLU8HUbd5_Xx38iF8nZ0= + - IK83IXhE4_7W7xCFEtD9op0BAs11pJqYN236Spppp7g= diff --git a/test/unit/security/fixtures/vault_conf_database_invalid_keys.yaml b/test/unit/security/fixtures/vault_conf_database_invalid_keys.yaml new file mode 100644 index 000000000000..6f77954997a5 --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_database_invalid_keys.yaml @@ -0,0 +1,12 @@ +type: database +# Encryption keys must be valid fernet keys +# To generate a valid key: +# >>> from cryptography.fernet import Fernet +# >>> Fernet.generate_key() +# b'pZDP8_baVs3oWT4597HJWCysm49j-XELONQ-EdoU0DE=' +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: + - EDC-1x8Xm9XqsXCdHbjvCTVE9oK83NOjJnWxc7IPYk= + - DB-cW4mX3116YN6y2tewUTZrUeGMKEtTXl9uQGk8lnM= diff --git a/test/unit/security/fixtures/vault_conf_database_rotated.yaml b/test/unit/security/fixtures/vault_conf_database_rotated.yaml new file mode 100644 index 000000000000..e1d802dadb9a --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_database_rotated.yaml @@ -0,0 +1,14 @@ +type: database +# Encryption keys must be valid fernet keys +# To generate a valid key: +# >>> from cryptography.fernet import Fernet +# >>> Fernet.generate_key() +# b'pZDP8_baVs3oWT4597HJWCysm49j-XELONQ-EdoU0DE=' +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: + - 5peww5oT8NMpxE31LpTiEE8qhccy_pPl4GW8iYu9gzU= + - 5RrT94ji178vQwha7TAmEix7DojtsLlxVz8Ef17KWgg= + - iNdXd7tRjLnSqRHxuhqQ98GTLU8HUbd5_Xx38iF8nZ0= + - IK83IXhE4_7W7xCFEtD9op0BAs11pJqYN236Spppp7g= diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py index 49904ca2e8de..b440667088b2 100644 --- a/test/unit/security/test_vault.py +++ b/test/unit/security/test_vault.py @@ -1,6 +1,7 @@ import os import unittest +from galaxy.app_unittest_utils.galaxy_mock import MockApp, MockAppConfig from galaxy.security import vault @@ -15,14 +16,46 @@ def test_read_write_secret(self): VAULT_CONF_HASHICORP = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_hashicorp.yaml") -class MockAppConfig: +class TestHashicorpVault(VaultTestBase, unittest.TestCase): - def __init__(self, vault_conf_file): - self.vault_config_file = vault_conf_file + def setUp(self) -> None: + config = MockAppConfig(vault_config_file=VAULT_CONF_HASHICORP) + app = MockApp(config=config) + self.vault = vault.VaultFactory.from_app_config(app) -class TestHashicorpVault(VaultTestBase, unittest.TestCase): +VAULT_CONF_DATABASE = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_database.yaml") +VAULT_CONF_DATABASE_ROTATED = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_database_rotated.yaml") +VAULT_CONF_DATABASE_INVALID = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_database_invalid_keys.yaml") + + +class TestDatabaseVault(VaultTestBase, unittest.TestCase): def setUp(self) -> None: - config = MockAppConfig(vault_conf_file=VAULT_CONF_HASHICORP) - self.vault = vault.VaultFactory.from_app_config(config) + config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) + app = MockApp(config=config) + self.vault = vault.VaultFactory.from_app_config(app) + + def test_rotate_keys(self): + config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) + app = MockApp(config=config) + self.vault = vault.VaultFactory.from_app_config(app) + self.vault.write_secret("my/rotated/secret", {"value": "hello rotated"}) + + # should succeed after rotation + app.config.vault_config_file = VAULT_CONF_DATABASE_ROTATED + self.vault = vault.VaultFactory.from_app_config(app) + self.assertEqual(self.vault.read_secret("my/rotated/secret"), {"value": "hello rotated"}) + super().test_read_write_secret() + + def test_wrong_keys(self): + config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) + app = MockApp(config=config) + self.vault = vault.VaultFactory.from_app_config(app) + self.vault.write_secret("my/incorrect/secret", {"value": "hello incorrect"}) + + # should fail because decryption keys are the wrong + app.config.vault_config_file = VAULT_CONF_DATABASE_INVALID + self.vault = vault.VaultFactory.from_app_config(app) + with self.assertRaises(Exception): + self.vault.read_secret("my/incorrect/secret") From e1535db412454453e3bdc45ba1e7cf72c9ca2fb3 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Thu, 18 Nov 2021 01:08:45 +0530 Subject: [PATCH 005/401] Add vault conditional dependencies --- doc/source/admin/galaxy_options.rst | 10 +++++++- lib/galaxy/app.py | 6 +++++ lib/galaxy/config/sample/galaxy.yml.sample | 4 +++ lib/galaxy/dependencies/__init__.py | 18 ++++++++++++++ lib/galaxy/files/__init__.py | 9 +++++++ lib/galaxy/managers/context.py | 6 +++++ lib/galaxy/structured_app.py | 2 ++ lib/galaxy/webapps/galaxy/config_schema.yml | 10 +++++++- test/integration/test_config_defaults.py | 2 ++ test/unit/app/dependencies/test_deps.py | 27 +++++++++++++++++++++ test/unit/config/test_config_values.py | 1 + 11 files changed, 93 insertions(+), 2 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 105f905c112e..2703f524b809 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -4809,5 +4809,13 @@ :Default: ``plugins/welcome_page/new_user/static/topics/`` :Type: str +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``vault_config_file`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - +:Description: + Vault configuration + The value of this option will be resolved with respect to + . +:Default: ``vault_conf.yml`` +:Type: str diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index b238a57737c6..3c8904996453 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -45,6 +45,10 @@ ) from galaxy.quota import get_quota_agent, QuotaAgent from galaxy.security.idencoding import IdEncodingHelper +from galaxy.security.vault import ( + Vault, + VaultFactory +) from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager from galaxy.tool_util.deps.views import DependencyResolversView @@ -183,6 +187,8 @@ def __init__(self, **kwargs): # ConfiguredFileSources self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)) + self.vault = self._register_singleton(Vault, VaultFactory.from_app_config(self.config)) + # We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need # to setup the installed repositories ... this is not ideal self._configure_tool_config_files() diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 83fe7890909b..70fe52f31404 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -2353,3 +2353,7 @@ galaxy: # is relative to galaxy/static #welcome_directory: plugins/welcome_page/new_user/static/topics/ + # Vault config file + # The value of this option will be resolved with respect to + # . + vault_config_file: vault_conf.yml \ No newline at end of file diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 27a9ed53c8d9..d9b4d845e567 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -33,6 +33,7 @@ def __init__(self, config_file, config=None): self.container_interface_types = [] self.job_rule_modules = [] self.error_report_modules = [] + self.vault_type = None if config is None: self.config = load_app_properties(config_file=self.config_file) else: @@ -151,6 +152,17 @@ def collect_types(from_dict): file_sources_conf = [] self.file_sources = [c.get('type', None) for c in file_sources_conf] + # Parse vault config + vault_conf_yml = self.config.get( + "vault_config_file", + join(dirname(self.config_file), 'vault_conf.yml')) + if exists(vault_conf_yml): + with open(vault_conf_yml) as f: + vault_conf = yaml.safe_load(f) + else: + vault_conf = {} + self.vault_type = vault_conf.get('type', '').lower() + def get_conditional_requirements(self): crfile = join(dirname(__file__), 'conditional-requirements.txt') for req in pkg_resources.parse_requirements(open(crfile).readlines()): @@ -261,6 +273,12 @@ def check_weasyprint(self): # See notes in ./conditional-requirements.txt for more information. return os.environ.get("GALAXY_DEPENDENCIES_INSTALL_WEASYPRINT") == "1" + def check_custos_sdk(self): + return 'custos' == self.vault_type + + def check_hvac(self): + return 'hashicorp' == self.vault_type + def optional(config_file=None): if not config_file: diff --git a/lib/galaxy/files/__init__.py b/lib/galaxy/files/__init__.py index 71a1cbc2b041..867ebb5147e3 100644 --- a/lib/galaxy/files/__init__.py +++ b/lib/galaxy/files/__init__.py @@ -263,6 +263,11 @@ def is_admin(self): """Whether this user is an administrator.""" return self.trans.user_is_admin + @property + def vault(self): + user = self.trans.user + return user and user.personal_vault or defaultdict(lambda: None) + class DictFileSourcesUserContext: @@ -296,3 +301,7 @@ def group_names(self): @property def is_admin(self): return self._kwd.get("is_admin") + + @property + def vault(self): + return self._kwd.get("vault") diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index 241eb2076ea1..9e73f5a718df 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -50,6 +50,7 @@ ) from galaxy.model.base import ModelMapping from galaxy.security.idencoding import IdEncodingHelper +from galaxy.security.vault import UserVaultWrapper from galaxy.structured_app import MinimalManagerApp from galaxy.util import bunch @@ -198,6 +199,11 @@ class ProvidesUserContext(ProvidesAppContext): def user(self): """Provide access to the user object.""" + @property + def user_vault(self): + """Provide access to a user's personal vault.""" + return UserVaultWrapper(self.app.vault, self.user) + @property def anonymous(self) -> bool: return self.user is None diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index c0dfbcf736e0..2436f50a2284 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -16,6 +16,7 @@ from galaxy.objectstore import ObjectStore from galaxy.quota import QuotaAgent from galaxy.security.idencoding import IdEncodingHelper +from galaxy.security.vault import Vault from galaxy.tool_util.deps.views import DependencyResolversView from galaxy.tool_util.verify import test_data from galaxy.util.dbkeys import GenomeBuilds @@ -98,6 +99,7 @@ class StructuredApp(MinimalManagerApp): security_agent: GalaxyRBACAgent host_security_agent: HostAgent trs_proxy: TrsProxy + vault: Vault webhooks_registry: WebhooksRegistry queue_worker: Any # 'galaxy.queue_worker.GalaxyQueueWorker' diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 2292290f9cea..fd382d070db7 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -3468,4 +3468,12 @@ mapping: desc: | Location of New User Welcome data, a single directory containing the images and JSON of Topics/Subtopics/Slides as export. This location - is relative to galaxy/static \ No newline at end of file + is relative to galaxy/static + + vault_config_file: + type: str + default: vault_conf.yml + path_resolves_to: config_dir + required: false + desc: | + Vault config file. diff --git a/test/integration/test_config_defaults.py b/test/integration/test_config_defaults.py index 9359417ab460..ce52047eaf17 100644 --- a/test/integration/test_config_defaults.py +++ b/test/integration/test_config_defaults.py @@ -94,6 +94,7 @@ 'tool_sheds_config_file', 'trs_servers_config_file', 'user_preferences_extra_conf_path', + 'vault_config_file', 'webhooks_dir', 'workflow_resource_params_file', 'workflow_resource_params_mapper', @@ -131,6 +132,7 @@ 'tool_path': 'root_dir', 'tool_sheds_config_file': 'config_dir', 'user_preferences_extra_conf_path': 'config_dir', + 'vault_config_file': 'config_dir', 'workflow_resource_params_file': 'config_dir', 'workflow_schedulers_config_file': 'config_dir', } diff --git a/test/unit/app/dependencies/test_deps.py b/test/unit/app/dependencies/test_deps.py index c97f81a18e08..9e7bd4a04006 100644 --- a/test/unit/app/dependencies/test_deps.py +++ b/test/unit/app/dependencies/test_deps.py @@ -28,6 +28,11 @@ runners: runner1: load: job_runner_A +VAULT_CONF_CUSTOS = """ +type: custos +""" +VAULT_CONF_HASHICORP = """ +type: hashicorp """ @@ -95,6 +100,28 @@ def test_yaml_jobconf_runners(): assert 'job_runner_A' in cds.job_runners +def test_vault_custos_configured(): + with _config_context() as cc: + vault_conf = cc.write_config("vault_conf.yml", VAULT_CONF_CUSTOS) + config = { + "vault_config_file": vault_conf, + } + cds = cc.get_cond_deps(config=config) + assert cds.check_custos_sdk() + assert not cds.check_hvac() + + +def test_vault_hashicorp_configured(): + with _config_context() as cc: + vault_conf = cc.write_config("vault_conf.yml", VAULT_CONF_HASHICORP) + config = { + "vault_config_file": vault_conf, + } + cds = cc.get_cond_deps(config=config) + assert cds.check_hvac() + assert not cds.check_custos_sdk() + + @contextmanager def _config_context(): config_dir = mkdtemp() diff --git a/test/unit/config/test_config_values.py b/test/unit/config/test_config_values.py index 954b59cbdcda..ca978860a130 100644 --- a/test/unit/config/test_config_values.py +++ b/test/unit/config/test_config_values.py @@ -157,6 +157,7 @@ def _load_paths(self): 'tool_test_data_directories': self._in_root_dir('test-data'), 'trs_servers_config_file': self._in_config_dir('trs_servers_conf.yml'), 'user_preferences_extra_conf_path': self._in_config_dir('user_preferences_extra_conf.yml'), + 'vault_config_file': self._in_config_dir('vault_conf.yml'), 'workflow_resource_params_file': self._in_config_dir('workflow_resource_params_conf.xml'), 'workflow_schedulers_config_file': self._in_config_dir('workflow_schedulers_conf.xml'), } From a05f3c2813881b40a6851e81788d8e5b7580eacf Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Thu, 18 Nov 2021 19:23:48 +0530 Subject: [PATCH 006/401] Add test for secret overwrites --- lib/galaxy/security/vault.py | 27 ++++++++++++------- .../vault_conf_database_invalid_keys.yaml | 4 +-- test/unit/security/test_vault.py | 8 +++++- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index b6ad211dadc8..461692a60e97 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -1,8 +1,10 @@ from abc import ABC -from cryptography.fernet import Fernet, MultiFernet import json import os import yaml + +from cryptography.fernet import Fernet, MultiFernet + try: import hvac except ImportError: @@ -49,10 +51,19 @@ class DatabaseVault(Vault): def __init__(self, sa_session, config): self.sa_session = sa_session self.encryption_keys = config.get('encryption_keys') + self.fernet_keys = [Fernet(key.encode('utf-8')) for key in self.encryption_keys] def _get_multi_fernet(self) -> MultiFernet: - fernet_keys = [Fernet(key.encode('utf-8')) for key in self.encryption_keys] - return MultiFernet(fernet_keys) + return MultiFernet(self.fernet_keys) + + def _update_or_create(self, key, value): + vault_entry = self.sa_session.query(model.Vault).filter_by(key=key).first() + if vault_entry: + vault_entry.value = value + else: + vault_entry = model.Vault(key=key, value=value) + self.sa_session.merge(vault_entry) + self.sa_session.flush() def read_secret(self, path: str) -> dict: key_obj = self.sa_session.query(model.Vault).filter_by(key=path).first() @@ -64,9 +75,7 @@ def read_secret(self, path: str) -> dict: def write_secret(self, path: str, value: dict) -> None: f = self._get_multi_fernet() token = f.encrypt(json.dumps(value).encode('utf-8')) - vault_entry = model.Vault(key=path, value=token.decode('utf-8')) - self.sa_session.add(vault_entry) - self.sa_session.flush() + self._update_or_create(key=path, value=token.decode('utf-8')) class CustosVault(Vault): @@ -89,14 +98,14 @@ def write_secret(self, path: str, value: dict) -> None: class VaultFactory(object): @staticmethod - def load_vault_config(vault_conf_yml): + def load_vault_config(vault_conf_yml: str) -> dict: if os.path.exists(vault_conf_yml): with open(vault_conf_yml) as f: return yaml.safe_load(f) return None @staticmethod - def from_vault_type(app, vault_type, cfg): + def from_vault_type(app, vault_type: str, cfg: dict) -> Vault: if vault_type == "hashicorp": return HashicorpVault(cfg) elif vault_type == "database": @@ -107,7 +116,7 @@ def from_vault_type(app, vault_type, cfg): raise UnknownVaultTypeException(f"Unknown vault type: {vault_type}") @staticmethod - def from_app_config(app): + def from_app_config(app) -> Vault: vault_config = VaultFactory.load_vault_config(app.config.vault_config_file) if vault_config: return VaultFactory.from_vault_type(app, vault_config.get('type'), vault_config) diff --git a/test/unit/security/fixtures/vault_conf_database_invalid_keys.yaml b/test/unit/security/fixtures/vault_conf_database_invalid_keys.yaml index 6f77954997a5..eb4b9974dc4e 100644 --- a/test/unit/security/fixtures/vault_conf_database_invalid_keys.yaml +++ b/test/unit/security/fixtures/vault_conf_database_invalid_keys.yaml @@ -8,5 +8,5 @@ type: database # Use the ascii string value as a key # For more details, see: https://cryptography.io/en/latest/fernet/# encryption_keys: - - EDC-1x8Xm9XqsXCdHbjvCTVE9oK83NOjJnWxc7IPYk= - - DB-cW4mX3116YN6y2tewUTZrUeGMKEtTXl9uQGk8lnM= + - aXOrK0R8eXRztiy8CHo-fNnwKhBBhMSS8cPv4JY4jOQ= + - 4KSQIhUU5W8oK36XxVBxXanA2Ge9Yu4ofe4-328E11o= diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py index b440667088b2..5962890bbe2b 100644 --- a/test/unit/security/test_vault.py +++ b/test/unit/security/test_vault.py @@ -1,3 +1,4 @@ +from abc import ABC import os import unittest @@ -5,13 +6,18 @@ from galaxy.security import vault -class VaultTestBase: +class VaultTestBase(ABC): def test_read_write_secret(self): self.assertIsNone(self.vault.read_secret("my/test/secret"), "Vault secret should initially be empty") self.vault.write_secret("my/test/secret", {"value": "hello world"}) self.assertEqual(self.vault.read_secret("my/test/secret"), {"value": "hello world"}) + def test_overwrite_secret(self): + self.vault.write_secret("my/new/secret", {"value": "hello world"}) + self.vault.write_secret("my/new/secret", {"value": "hello overwritten"}) + self.assertEqual(self.vault.read_secret("my/new/secret"), {"value": "hello overwritten"}) + VAULT_CONF_HASHICORP = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_hashicorp.yaml") From 5fcfb00d66211a91355b4a6ef71e235df5fbcdb1 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Thu, 18 Nov 2021 23:25:19 +0530 Subject: [PATCH 007/401] Add initial custos vault backend --- lib/galaxy/security/vault.py | 38 ++++++++++++++++++- .../security/fixtures/vault_conf_custos.yaml | 5 +++ test/unit/security/test_vault.py | 22 ++++++++++- 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 test/unit/security/fixtures/vault_conf_custos.yaml diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index 461692a60e97..039825bf4322 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -5,6 +5,14 @@ from cryptography.fernet import Fernet, MultiFernet +try: + from custos.clients.resource_secret_management_client import ResourceSecretManagementClient + from custos.transport.settings import CustosServerClientSettings + import custos.clients.utils.utilities as custos_util + custos_sdk_available = True +except ImportError: + custos_sdk_available = False + try: import hvac except ImportError: @@ -79,7 +87,35 @@ def write_secret(self, path: str, value: dict) -> None: class CustosVault(Vault): - pass + + def __init__(self, config): + if not custos_sdk_available: + raise UnknownVaultTypeException("Custos sdk library 'custos-sdk' is not available. Make sure the custos-sdk is installed.") + self.custos_settings = CustosServerClientSettings(custos_host=config.get('custos_host'), + custos_port=config.get('custos_port'), + custos_client_id=config.get('custos_client_id'), + custos_client_sec=config.get('custos_client_sec')) + self.b64_encoded_custos_token = custos_util.get_token(custos_settings=self.custos_settings) + self.client = ResourceSecretManagementClient(self.custos_settings) + + def read_secret(self, path: str) -> dict: + try: + response = self.client.get_KV_credential(token=self.b64_encoded_custos_token, + client_id=self.custos_settings.CUSTOS_CLIENT_ID, + key=path) + return json.loads(json.loads(response)['value']) + except Exception: + return None + + def write_secret(self, path: str, value: dict) -> None: + if self.read_secret(path): + self.client.update_KV_credential(token=self.b64_encoded_custos_token, + client_id=self.custos_settings.CUSTOS_CLIENT_ID, + key=path, value=json.dumps(value)) + else: + self.client.set_KV_credential(token=self.b64_encoded_custos_token, + client_id=self.custos_settings.CUSTOS_CLIENT_ID, + key=path, value=json.dumps(value)) class UserVaultWrapper(Vault): diff --git a/test/unit/security/fixtures/vault_conf_custos.yaml b/test/unit/security/fixtures/vault_conf_custos.yaml new file mode 100644 index 000000000000..8812c7af1b9a --- /dev/null +++ b/test/unit/security/fixtures/vault_conf_custos.yaml @@ -0,0 +1,5 @@ +type: custos +custos_host: service.staging.usecustos.org +custos_port: 30170 +custos_client_id: ${custos_client_id} +custos_client_sec: ${custos_client_secret} diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py index 5962890bbe2b..d200036b4f79 100644 --- a/test/unit/security/test_vault.py +++ b/test/unit/security/test_vault.py @@ -1,5 +1,7 @@ from abc import ABC import os +import string +import tempfile import unittest from galaxy.app_unittest_utils.galaxy_mock import MockApp, MockAppConfig @@ -9,7 +11,6 @@ class VaultTestBase(ABC): def test_read_write_secret(self): - self.assertIsNone(self.vault.read_secret("my/test/secret"), "Vault secret should initially be empty") self.vault.write_secret("my/test/secret", {"value": "hello world"}) self.assertEqual(self.vault.read_secret("my/test/secret"), {"value": "hello world"}) @@ -65,3 +66,22 @@ def test_wrong_keys(self): self.vault = vault.VaultFactory.from_app_config(app) with self.assertRaises(Exception): self.vault.read_secret("my/incorrect/secret") + + +VAULT_CONF_CUSTOS = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_custos.yaml") + + +class TestCustosVault(VaultTestBase, unittest.TestCase): + + def setUp(self) -> None: + with tempfile.NamedTemporaryFile(mode="w", prefix="vault_custos", delete=False) as tempconf, open(VAULT_CONF_CUSTOS) as f: + content = string.Template(f.read()).safe_substitute(custos_client_id=os.environ.get('CUSTOS_CLIENT_ID'), + custos_client_secret=os.environ.get('CUSTOS_CLIENT_SECRET')) + tempconf.write(content) + self.vault_temp_conf = tempconf.name + config = MockAppConfig(vault_config_file=self.vault_temp_conf) + app = MockApp(config=config) + self.vault = vault.VaultFactory.from_app_config(app) + + def tearDown(self) -> None: + os.remove(self.vault_temp_conf) \ No newline at end of file From 4695dc8a7a6de6e2f10e87252a717f05a9be11c4 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Thu, 18 Nov 2021 23:56:22 +0530 Subject: [PATCH 008/401] Further simplify vault interface to be string key value pairs --- lib/galaxy/security/vault.py | 50 ++++++++++++++++---------------- test/unit/security/test_vault.py | 18 ++++++------ 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index 039825bf4322..afb6cef81862 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -27,10 +27,10 @@ class UnknownVaultTypeException(Exception): class Vault(ABC): - def read_secret(self, path: str) -> dict: + def read_secret(self, key: str) -> str: pass - def write_secret(self, path: str, value: dict) -> None: + def write_secret(self, key: str, value: str) -> None: pass @@ -43,15 +43,15 @@ def __init__(self, config): self.vault_token = config.get('vault_token') self.client = hvac.Client(url=self.vault_address, token=self.vault_token) - def read_secret(self, path: str) -> dict: + def read_secret(self, key: str) -> str: try: - response = self.client.secrets.kv.read_secret_version(path=path) - return response['data']['data'] + response = self.client.secrets.kv.read_secret_version(path=key) + return response['data']['data'].get('value') except hvac.exceptions.InvalidPath: return None - def write_secret(self, path: str, value: dict) -> None: - self.client.secrets.kv.v2.create_or_update_secret(path=path, secret=value) + def write_secret(self, key: str, value: str) -> None: + self.client.secrets.kv.v2.create_or_update_secret(path=key, secret={'value': value}) class DatabaseVault(Vault): @@ -64,7 +64,7 @@ def __init__(self, sa_session, config): def _get_multi_fernet(self) -> MultiFernet: return MultiFernet(self.fernet_keys) - def _update_or_create(self, key, value): + def _update_or_create(self, key: str, value: str): vault_entry = self.sa_session.query(model.Vault).filter_by(key=key).first() if vault_entry: vault_entry.value = value @@ -73,17 +73,17 @@ def _update_or_create(self, key, value): self.sa_session.merge(vault_entry) self.sa_session.flush() - def read_secret(self, path: str) -> dict: - key_obj = self.sa_session.query(model.Vault).filter_by(key=path).first() + def read_secret(self, key: str) -> str: + key_obj = self.sa_session.query(model.Vault).filter_by(key=key).first() if key_obj: f = self._get_multi_fernet() - return json.loads(f.decrypt(key_obj.value.encode('utf-8')).decode('utf-8')) + return f.decrypt(key_obj.value.encode('utf-8')).decode('utf-8') return None - def write_secret(self, path: str, value: dict) -> None: + def write_secret(self, key: str, value: str) -> None: f = self._get_multi_fernet() - token = f.encrypt(json.dumps(value).encode('utf-8')) - self._update_or_create(key=path, value=token.decode('utf-8')) + token = f.encrypt(value.encode('utf-8')) + self._update_or_create(key=key, value=token.decode('utf-8')) class CustosVault(Vault): @@ -98,24 +98,24 @@ def __init__(self, config): self.b64_encoded_custos_token = custos_util.get_token(custos_settings=self.custos_settings) self.client = ResourceSecretManagementClient(self.custos_settings) - def read_secret(self, path: str) -> dict: + def read_secret(self, key: str) -> str: try: response = self.client.get_KV_credential(token=self.b64_encoded_custos_token, client_id=self.custos_settings.CUSTOS_CLIENT_ID, - key=path) - return json.loads(json.loads(response)['value']) + key=key) + return json.loads(response).get('value') except Exception: return None - def write_secret(self, path: str, value: dict) -> None: - if self.read_secret(path): + def write_secret(self, key: str, value: str) -> None: + if self.read_secret(key): self.client.update_KV_credential(token=self.b64_encoded_custos_token, client_id=self.custos_settings.CUSTOS_CLIENT_ID, - key=path, value=json.dumps(value)) + key=key, value=value) else: self.client.set_KV_credential(token=self.b64_encoded_custos_token, client_id=self.custos_settings.CUSTOS_CLIENT_ID, - key=path, value=json.dumps(value)) + key=key, value=value) class UserVaultWrapper(Vault): @@ -124,11 +124,11 @@ def __init__(self, vault: Vault, user): self.vault = vault self.user = user - def read_secret(self, path: str) -> dict: - return self.vault.read_secret(f"user/{self.user.id}/{path}") + def read_secret(self, key: str) -> str: + return self.vault.read_secret(f"user/{self.user.id}/{key}") - def write_secret(self, path: str, value: dict) -> None: - return self.vault.write_secret(f"user/{self.user.id}/{path}", value) + def write_secret(self, key: str, value: str) -> None: + return self.vault.write_secret(f"user/{self.user.id}/{key}", value) class VaultFactory(object): diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py index d200036b4f79..f473cf55a156 100644 --- a/test/unit/security/test_vault.py +++ b/test/unit/security/test_vault.py @@ -11,13 +11,13 @@ class VaultTestBase(ABC): def test_read_write_secret(self): - self.vault.write_secret("my/test/secret", {"value": "hello world"}) - self.assertEqual(self.vault.read_secret("my/test/secret"), {"value": "hello world"}) + self.vault.write_secret("my/test/secret", "hello world") + self.assertEqual(self.vault.read_secret("my/test/secret"), "hello world") def test_overwrite_secret(self): - self.vault.write_secret("my/new/secret", {"value": "hello world"}) - self.vault.write_secret("my/new/secret", {"value": "hello overwritten"}) - self.assertEqual(self.vault.read_secret("my/new/secret"), {"value": "hello overwritten"}) + self.vault.write_secret("my/new/secret", "hello world") + self.vault.write_secret("my/new/secret", "hello overwritten") + self.assertEqual(self.vault.read_secret("my/new/secret"), "hello overwritten") VAULT_CONF_HASHICORP = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_hashicorp.yaml") @@ -47,19 +47,19 @@ def test_rotate_keys(self): config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) app = MockApp(config=config) self.vault = vault.VaultFactory.from_app_config(app) - self.vault.write_secret("my/rotated/secret", {"value": "hello rotated"}) + self.vault.write_secret("my/rotated/secret", "hello rotated") # should succeed after rotation app.config.vault_config_file = VAULT_CONF_DATABASE_ROTATED self.vault = vault.VaultFactory.from_app_config(app) - self.assertEqual(self.vault.read_secret("my/rotated/secret"), {"value": "hello rotated"}) + self.assertEqual(self.vault.read_secret("my/rotated/secret"), "hello rotated") super().test_read_write_secret() def test_wrong_keys(self): config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) app = MockApp(config=config) self.vault = vault.VaultFactory.from_app_config(app) - self.vault.write_secret("my/incorrect/secret", {"value": "hello incorrect"}) + self.vault.write_secret("my/incorrect/secret", "hello incorrect") # should fail because decryption keys are the wrong app.config.vault_config_file = VAULT_CONF_DATABASE_INVALID @@ -84,4 +84,4 @@ def setUp(self) -> None: self.vault = vault.VaultFactory.from_app_config(app) def tearDown(self) -> None: - os.remove(self.vault_temp_conf) \ No newline at end of file + os.remove(self.vault_temp_conf) From a42ff36d3ac3f8d36a991c6407ae1f174b6d6ea1 Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Thu, 18 Nov 2021 15:30:44 -0800 Subject: [PATCH 009/401] single dataset update --- client/src/mvc/ui/ui-select-content.js | 25 +++++++++++++++++++++++-- lib/galaxy/workflow/modules.py | 16 ++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/client/src/mvc/ui/ui-select-content.js b/client/src/mvc/ui/ui-select-content.js index 07de392b726a..b7949bb3d7dd 100644 --- a/client/src/mvc/ui/ui-select-content.js +++ b/client/src/mvc/ui/ui-select-content.js @@ -125,6 +125,9 @@ const Configurations = { /** View for hda and dce content selector ui elements */ const View = Backbone.View.extend({ initialize: function (options) { + console.log('Workflow info'); + console.log(options); + console.log(options.tag); const self = this; this.model = (options && options.model) || @@ -321,6 +324,9 @@ const View = Backbone.View.extend({ icon: c.icon, tooltip: c.tooltip, }); + console.log("Test location"); + console.log(self.model); + console.log(self.model.attributes.tag); self.fields.push( new Select.View({ optional: self.model.get("optional"), @@ -425,15 +431,30 @@ const View = Backbone.View.extend({ _.each(options, (items, src) => { _.each(items, (item) => { self._patchValue(item); + console.log(item) const current_src = item.src || src; - select_options[current_src].push({ + if (this.model.attributes.tag) { + if(item.tags.includes(this.model.attributes.tag)) { + select_options[current_src].push({ + hid: item.hid, + keep: item.keep, + label: `${item.hid || "Selected"}: ${item.name}`, + value: item.id, + origin: item.origin, + tags: item.tags, + }); + }; + } + else { + select_options[current_src].push({ hid: item.hid, keep: item.keep, label: `${item.hid || "Selected"}: ${item.name}`, value: item.id, origin: item.origin, tags: item.tags, - }); + }); + } self.cache[`${item.id}_${current_src}`] = item; }); }); diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 97930d0183ee..6831ba88d0ba 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -680,6 +680,12 @@ def _parse_state_into_dict(self): formats = None if formats: rval["format"] = formats + if "tag" in inputs: + tag = inputs["tag"] + else: + tag = None + rval["tag"] = tag + print(rval) return rval def step_state_to_tool_state(self, state): @@ -725,22 +731,27 @@ def get_filter_set(self, connections=None): def get_runtime_inputs(self, connections=None): parameter_def = self._parse_state_into_dict() optional = parameter_def["optional"] + tag = parameter_def["tag"] formats = parameter_def.get("format") if not formats: formats = self.get_filter_set(connections) else: formats = ",".join(listify(formats)) - data_src = dict(name="input", label=self.label, multiple=False, type="data", format=formats, optional=optional) + data_src = dict(name="input", label=self.label, multiple=False, type="data", format=formats, tag=tag, optional=optional) input_param = DataToolParameter(None, data_src, self.trans) return dict(input=input_param) def get_inputs(self): parameter_def = self._parse_state_into_dict() + tag = parameter_def["tag"] optional = parameter_def["optional"] + tag_source = dict(name="tag", label="Tag filter", type="text", value=tag, help="Tags to automatically filter inputs") + input_tag = TextToolParameter(None, tag_source) inputs = {} inputs["optional"] = optional_param(optional) inputs["format"] = format_param(self.trans, parameter_def.get("format")) + inputs["tag"] = input_tag return inputs @@ -775,8 +786,9 @@ def get_runtime_inputs(self, **kwds): parameter_def = self._parse_state_into_dict() collection_type = parameter_def["collection_type"] optional = parameter_def["optional"] + tag = parameter_def["tag"] formats = parameter_def.get("format") - collection_param_source = dict(name="input", label=self.label, type="data_collection", collection_type=collection_type, optional=optional) + collection_param_source = dict(name="input", label=self.label, type="data_collection", collection_type=collection_type, tag=tag, optional=optional) if formats: collection_param_source["format"] = ",".join(listify(formats)) input_param = DataCollectionToolParameter(None, collection_param_source, self.trans) From 7f2a627f48f284d49721d0a60e192d805d3ed231 Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Thu, 18 Nov 2021 16:08:51 -0800 Subject: [PATCH 010/401] working single dataset --- lib/galaxy/tools/parameters/basic.py | 6 ++++++ lib/galaxy/workflow/modules.py | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 3425f4d18282..f6011acea43b 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1864,11 +1864,13 @@ def __init__(self, tool, input_source, trans=None): if not input_source.get_bool('no_validation', False): self.validators.append(validation.MetadataValidator()) self._parse_formats(trans, input_source) + tag = input_source.get("tag") self.multiple = input_source.get_bool('multiple', False) if not self.multiple and (self.min is not None): raise ParameterValueError("cannot specify 'min' property on single data parameter. Set multiple=\"true\" to enable this option", self.name) if not self.multiple and (self.max is not None): raise ParameterValueError("cannot specify 'max' property on single data parameter. Set multiple=\"true\" to enable this option", self.name) + self.tag = tag self.is_dynamic = True self._parse_options(input_source) # Load conversions required for the dataset input @@ -2047,6 +2049,7 @@ def to_dict(self, trans, other_values=None): d['min'] = self.min d['max'] = self.max d['options'] = {'hda': [], 'hdca': []} + d['tag'] = self.tag # return dictionary without options if context is unavailable history = trans.history @@ -2134,9 +2137,11 @@ def __init__(self, tool, input_source, trans=None): super().__init__(tool, input_source, trans) self._parse_formats(trans, input_source) collection_types = input_source.get("collection_type", None) + tag = input_source.get("tag") if collection_types: collection_types = [t.strip() for t in collection_types.split(",")] self._collection_types = collection_types + self.tag = tag self.multiple = False # Accessed on DataToolParameter a lot, may want in future self.is_dynamic = True self._parse_options(input_source) # TODO: Review and test. @@ -2228,6 +2233,7 @@ def to_dict(self, trans, other_values=None): d['extensions'] = self.extensions d['multiple'] = self.multiple d['options'] = {'hda': [], 'hdca': [], 'dce': []} + d['tag'] = self.tag # return dictionary without options if context is unavailable history = trans.history diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 6831ba88d0ba..add6def5c707 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -685,7 +685,6 @@ def _parse_state_into_dict(self): else: tag = None rval["tag"] = tag - print(rval) return rval def step_state_to_tool_state(self, state): From 06b52df119a33e8834ad9b4e09aef814f44f9368 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 19 Nov 2021 14:28:37 -0500 Subject: [PATCH 011/401] Use flag to determine if an option should be added or not --- client/src/mvc/ui/ui-select-content.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/client/src/mvc/ui/ui-select-content.js b/client/src/mvc/ui/ui-select-content.js index b7949bb3d7dd..29a2416b5434 100644 --- a/client/src/mvc/ui/ui-select-content.js +++ b/client/src/mvc/ui/ui-select-content.js @@ -433,19 +433,15 @@ const View = Backbone.View.extend({ self._patchValue(item); console.log(item) const current_src = item.src || src; + let addOption = false; if (this.model.attributes.tag) { if(item.tags.includes(this.model.attributes.tag)) { - select_options[current_src].push({ - hid: item.hid, - keep: item.keep, - label: `${item.hid || "Selected"}: ${item.name}`, - value: item.id, - origin: item.origin, - tags: item.tags, - }); + addOption = true }; + } else { + addOption = true; } - else { + if (addOption) { select_options[current_src].push({ hid: item.hid, keep: item.keep, From b2d0cba1663003e850b127e3f586224ffbe688eb Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 19 Nov 2021 14:36:15 -0500 Subject: [PATCH 012/401] Collapse if/else flag statement --- client/src/mvc/ui/ui-select-content.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/client/src/mvc/ui/ui-select-content.js b/client/src/mvc/ui/ui-select-content.js index 29a2416b5434..9e118b293d69 100644 --- a/client/src/mvc/ui/ui-select-content.js +++ b/client/src/mvc/ui/ui-select-content.js @@ -433,14 +433,7 @@ const View = Backbone.View.extend({ self._patchValue(item); console.log(item) const current_src = item.src || src; - let addOption = false; - if (this.model.attributes.tag) { - if(item.tags.includes(this.model.attributes.tag)) { - addOption = true - }; - } else { - addOption = true; - } + const addOption = !this.model.attributes.tag || item.tags.includes(this.model.attributes.tag); if (addOption) { select_options[current_src].push({ hid: item.hid, From c8f25949a245a52389eca686f0615d46f43b51da Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Fri, 19 Nov 2021 11:37:27 -0800 Subject: [PATCH 013/401] lint stuff --- client/src/mvc/ui/ui-select-content.js | 9 +-------- lib/galaxy/workflow/modules.py | 1 - 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/client/src/mvc/ui/ui-select-content.js b/client/src/mvc/ui/ui-select-content.js index b7949bb3d7dd..d6a684518f31 100644 --- a/client/src/mvc/ui/ui-select-content.js +++ b/client/src/mvc/ui/ui-select-content.js @@ -125,9 +125,6 @@ const Configurations = { /** View for hda and dce content selector ui elements */ const View = Backbone.View.extend({ initialize: function (options) { - console.log('Workflow info'); - console.log(options); - console.log(options.tag); const self = this; this.model = (options && options.model) || @@ -324,9 +321,6 @@ const View = Backbone.View.extend({ icon: c.icon, tooltip: c.tooltip, }); - console.log("Test location"); - console.log(self.model); - console.log(self.model.attributes.tag); self.fields.push( new Select.View({ optional: self.model.get("optional"), @@ -431,7 +425,6 @@ const View = Backbone.View.extend({ _.each(options, (items, src) => { _.each(items, (item) => { self._patchValue(item); - console.log(item) const current_src = item.src || src; if (this.model.attributes.tag) { if(item.tags.includes(this.model.attributes.tag)) { @@ -443,7 +436,7 @@ const View = Backbone.View.extend({ origin: item.origin, tags: item.tags, }); - }; + } } else { select_options[current_src].push({ diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index add6def5c707..70c2d210a11c 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -740,7 +740,6 @@ def get_runtime_inputs(self, connections=None): input_param = DataToolParameter(None, data_src, self.trans) return dict(input=input_param) - def get_inputs(self): parameter_def = self._parse_state_into_dict() tag = parameter_def["tag"] From 23f204de4039708436b7eae719ab0b602a7616d4 Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Fri, 19 Nov 2021 12:50:39 -0800 Subject: [PATCH 014/401] fix null parameter --- client/src/mvc/ui/ui-select-content.js | 14 +++++++------- lib/galaxy/workflow/modules.py | 9 +++++---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/client/src/mvc/ui/ui-select-content.js b/client/src/mvc/ui/ui-select-content.js index 9be61e10399d..9e2d9098d53c 100644 --- a/client/src/mvc/ui/ui-select-content.js +++ b/client/src/mvc/ui/ui-select-content.js @@ -429,13 +429,13 @@ const View = Backbone.View.extend({ const addOption = !this.model.attributes.tag || item.tags.includes(this.model.attributes.tag); if (addOption) { select_options[current_src].push({ - hid: item.hid, - keep: item.keep, - label: `${item.hid || "Selected"}: ${item.name}`, - value: item.id, - origin: item.origin, - tags: item.tags, - }); + hid: item.hid, + keep: item.keep, + label: `${item.hid || "Selected"}: ${item.name}`, + value: item.id, + origin: item.origin, + tags: item.tags, + }); } self.cache[`${item.id}_${current_src}`] = item; }); diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 70c2d210a11c..27f4453a8ce5 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -742,14 +742,15 @@ def get_runtime_inputs(self, connections=None): def get_inputs(self): parameter_def = self._parse_state_into_dict() - tag = parameter_def["tag"] + if parameter_def["tag"]: + tag = parameter_def["tag"] + tag_source = dict(name="tag", label="Tag filter", type="text", value=tag, help="Tags to automatically filter inputs") + input_tag = TextToolParameter(None, tag_source) + inputs["tag"] = input_tag optional = parameter_def["optional"] - tag_source = dict(name="tag", label="Tag filter", type="text", value=tag, help="Tags to automatically filter inputs") - input_tag = TextToolParameter(None, tag_source) inputs = {} inputs["optional"] = optional_param(optional) inputs["format"] = format_param(self.trans, parameter_def.get("format")) - inputs["tag"] = input_tag return inputs From 039c433938e3890bd8bea220f404ea7f8ef3ee99 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 19 Nov 2021 16:20:55 -0500 Subject: [PATCH 015/401] Remove duplicate tag field handling --- lib/galaxy/workflow/modules.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 27f4453a8ce5..7c3b46f66cdd 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -742,15 +742,14 @@ def get_runtime_inputs(self, connections=None): def get_inputs(self): parameter_def = self._parse_state_into_dict() - if parameter_def["tag"]: - tag = parameter_def["tag"] - tag_source = dict(name="tag", label="Tag filter", type="text", value=tag, help="Tags to automatically filter inputs") - input_tag = TextToolParameter(None, tag_source) - inputs["tag"] = input_tag + tag = parameter_def["tag"] + tag_source = dict(name="tag", label="Tag filter", type="text", value=tag, help="Tags to automatically filter inputs") + input_tag = TextToolParameter(None, tag_source) optional = parameter_def["optional"] inputs = {} inputs["optional"] = optional_param(optional) inputs["format"] = format_param(self.trans, parameter_def.get("format")) + inputs["tag"] = input_tag return inputs @@ -819,11 +818,6 @@ def _parse_state_into_dict(self): else: collection_type = self.default_collection_type state_as_dict["collection_type"] = collection_type - if "tag" in inputs: - tag = inputs["tag"] - else: - tag = None - state_as_dict["tag"] = tag return state_as_dict From 9053f78c1c3bc6116fde2ba68fdfae9b19ac8552 Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Fri, 19 Nov 2021 13:37:28 -0800 Subject: [PATCH 016/401] add dict type lint lint --- lib/galaxy/workflow/modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 7c3b46f66cdd..f181043eb852 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -618,6 +618,7 @@ def format_param(trans, formats): class InputModuleState(TypedDict, total=False): optional: bool format: List[str] + tag: str class InputModule(WorkflowModule): From bf248f7d1c7e57178ce5d01c0e6d7ebff5e74227 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Sat, 20 Nov 2021 17:58:52 +0530 Subject: [PATCH 017/401] Rename VaultFactory.from_app_config to from_app --- lib/galaxy/app.py | 2 +- lib/galaxy/security/vault.py | 2 +- test/unit/security/test_vault.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 3c8904996453..598e882c8995 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -187,7 +187,7 @@ def __init__(self, **kwargs): # ConfiguredFileSources self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)) - self.vault = self._register_singleton(Vault, VaultFactory.from_app_config(self.config)) + self.vault = self._register_singleton(Vault, VaultFactory.from_app(self)) # We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need # to setup the installed repositories ... this is not ideal diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index afb6cef81862..64e010a4dcb7 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -152,7 +152,7 @@ def from_vault_type(app, vault_type: str, cfg: dict) -> Vault: raise UnknownVaultTypeException(f"Unknown vault type: {vault_type}") @staticmethod - def from_app_config(app) -> Vault: + def from_app(app) -> Vault: vault_config = VaultFactory.load_vault_config(app.config.vault_config_file) if vault_config: return VaultFactory.from_vault_type(app, vault_config.get('type'), vault_config) diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py index f473cf55a156..9cf5adb73f46 100644 --- a/test/unit/security/test_vault.py +++ b/test/unit/security/test_vault.py @@ -28,7 +28,7 @@ class TestHashicorpVault(VaultTestBase, unittest.TestCase): def setUp(self) -> None: config = MockAppConfig(vault_config_file=VAULT_CONF_HASHICORP) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app_config(app) + self.vault = vault.VaultFactory.from_app(app) VAULT_CONF_DATABASE = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_database.yaml") @@ -41,29 +41,29 @@ class TestDatabaseVault(VaultTestBase, unittest.TestCase): def setUp(self) -> None: config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app_config(app) + self.vault = vault.VaultFactory.from_app(app) def test_rotate_keys(self): config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app_config(app) + self.vault = vault.VaultFactory.from_app(app) self.vault.write_secret("my/rotated/secret", "hello rotated") # should succeed after rotation app.config.vault_config_file = VAULT_CONF_DATABASE_ROTATED - self.vault = vault.VaultFactory.from_app_config(app) + self.vault = vault.VaultFactory.from_app(app) self.assertEqual(self.vault.read_secret("my/rotated/secret"), "hello rotated") super().test_read_write_secret() def test_wrong_keys(self): config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app_config(app) + self.vault = vault.VaultFactory.from_app(app) self.vault.write_secret("my/incorrect/secret", "hello incorrect") # should fail because decryption keys are the wrong app.config.vault_config_file = VAULT_CONF_DATABASE_INVALID - self.vault = vault.VaultFactory.from_app_config(app) + self.vault = vault.VaultFactory.from_app(app) with self.assertRaises(Exception): self.vault.read_secret("my/incorrect/secret") @@ -81,7 +81,7 @@ def setUp(self) -> None: self.vault_temp_conf = tempconf.name config = MockAppConfig(vault_config_file=self.vault_temp_conf) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app_config(app) + self.vault = vault.VaultFactory.from_app(app) def tearDown(self) -> None: os.remove(self.vault_temp_conf) From e6494b16cea47283d0935691c4681dbf9483e1ad Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Sun, 21 Nov 2021 23:36:28 +0530 Subject: [PATCH 018/401] Added support for storing extra preferences in the vault + tests --- lib/galaxy/security/vault.py | 9 ++- lib/galaxy/webapps/galaxy/api/users.py | 41 ++++++++----- lib/galaxy_test/api/test_vault.py | 60 +++++++++++++++++++ .../api/user_preferences_extra_conf.yml | 23 +++++++ lib/galaxy_test/api/vault_conf.yml | 13 ++++ 5 files changed, 128 insertions(+), 18 deletions(-) create mode 100644 lib/galaxy_test/api/test_vault.py create mode 100644 lib/galaxy_test/api/user_preferences_extra_conf.yml create mode 100644 lib/galaxy_test/api/vault_conf.yml diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index 64e010a4dcb7..360bd16379f3 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -1,5 +1,6 @@ from abc import ABC import json +import logging import os import yaml @@ -20,6 +21,8 @@ from galaxy import model +log = logging.getLogger(__name__) + class UnknownVaultTypeException(Exception): pass @@ -28,10 +31,10 @@ class UnknownVaultTypeException(Exception): class Vault(ABC): def read_secret(self, key: str) -> str: - pass + raise UnknownVaultTypeException("No vault configured. Make sure the vault_config_file setting is defined in galaxy.yml") def write_secret(self, key: str, value: str) -> None: - pass + raise UnknownVaultTypeException("No vault configured. Make sure the vault_config_file setting is defined in galaxy.yml") class HashicorpVault(Vault): @@ -156,4 +159,4 @@ def from_app(app) -> Vault: vault_config = VaultFactory.load_vault_config(app.config.vault_config_file) if vault_config: return VaultFactory.from_vault_type(app, vault_config.get('type'), vault_config) - return None + log.warning("No vault configured. We recommend defining the vault_config_file setting in galaxy.yml") diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py index dfc0f65a813b..46d1dc69d37d 100644 --- a/lib/galaxy/webapps/galaxy/api/users.py +++ b/lib/galaxy/webapps/galaxy/api/users.py @@ -30,6 +30,7 @@ validate_password, validate_publicname ) +from galaxy.security.vault import UserVaultWrapper from galaxy.tool_util.toolbox.filters import FilterFactory from galaxy.util import ( docstring_trim, @@ -288,7 +289,7 @@ def _get_extra_user_preferences(self, trans): """ return trans.app.config.user_preferences_extra['preferences'] - def _build_extra_user_pref_inputs(self, preferences, user): + def _build_extra_user_pref_inputs(self, trans, preferences, user): """ Build extra user preferences inputs list. Add values to the fields if present @@ -297,6 +298,7 @@ def _build_extra_user_pref_inputs(self, preferences, user): return [] extra_pref_inputs = list() # Build sections for different categories of inputs + user_vault = UserVaultWrapper(trans.app.vault, user) for item, value in preferences.items(): if value is not None: input_fields = copy.deepcopy(value["inputs"]) @@ -307,10 +309,14 @@ def _build_extra_user_pref_inputs(self, preferences, user): input['help'] = f"{help} {required}" else: input['help'] = required - field = f"{item}|{input['name']}" - for data_item in user.extra_preferences: - if field in data_item: - input['value'] = user.extra_preferences[data_item] + if input.get('store') == 'vault': + field = f"{item}/{input['name']}" + input['value'] = user_vault.read_secret(f'preferences/{field}') + else: + field = f"{item}|{input['name']}" + for data_item in user.extra_preferences: + if field in data_item: + input['value'] = user.extra_preferences[data_item] extra_pref_inputs.append({'type': 'section', 'title': value['description'], 'name': item, 'expanded': True, 'inputs': input_fields}) return extra_pref_inputs @@ -382,7 +388,7 @@ def get_information(self, trans, id, **kwd): inputs.append(address_repeat) # Build input sections for extra user preferences - extra_user_pref = self._build_extra_user_pref_inputs(self._get_extra_user_preferences(trans), user) + extra_user_pref = self._build_extra_user_pref_inputs(trans, self._get_extra_user_preferences(trans), user) for item in extra_user_pref: inputs.append(item) else: @@ -460,20 +466,25 @@ def set_information(self, trans, id, payload=None, **kwd): # Update values for extra user preference items extra_user_pref_data = dict() extra_pref_keys = self._get_extra_user_preferences(trans) + user_vault = UserVaultWrapper(trans.app.vault, user) if extra_pref_keys is not None: for key in extra_pref_keys: key_prefix = f"{key}|" for item in payload: if item.startswith(key_prefix): - # Show error message if the required field is empty - if payload[item] == "": - # Raise an exception when a required field is empty while saving the form - keys = item.split("|") - section = extra_pref_keys[keys[0]] - for input in section['inputs']: - if input['name'] == keys[1] and input['required']: - raise exceptions.ObjectAttributeMissingException("Please fill the required field") - extra_user_pref_data[item] = payload[item] + keys = item.split("|") + section = extra_pref_keys[keys[0]] + matching_input = [input for input in section['inputs'] if input['name'] == keys[1]] + if matching_input: + input = matching_input[0] + if input.get('required') and payload[item] == "": + raise exceptions.ObjectAttributeMissingException("Please fill the required field") + if input.get('store') == 'vault': + user_vault.write_secret(f'preferences/{keys[0]}/{keys[1]}', str(payload[item])) + else: + extra_user_pref_data[item] = payload[item] + else: + extra_user_pref_data[item] = payload[item] user.preferences["extra_user_preferences"] = json.dumps(extra_user_pref_data) # Update user addresses diff --git a/lib/galaxy_test/api/test_vault.py b/lib/galaxy_test/api/test_vault.py new file mode 100644 index 000000000000..f8e83bf92ec4 --- /dev/null +++ b/lib/galaxy_test/api/test_vault.py @@ -0,0 +1,60 @@ +import json +import os + +from requests import ( + delete, + get, + put +) + +from ._framework import ApiTestCase + +TEST_USER_EMAIL = "user_for_users_index_test@bx.psu.edu" + + +class VaultApiTestCase(ApiTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["vault_config_file"] = os.path.join(os.path.dirname(__file__), "vault_conf.yml") + config["user_preferences_extra_conf_path"] = os.path.join(os.path.dirname(__file__), "user_preferences_extra_conf.yml") + + def test_extra_prefs_vault_storage(self): + user = self._setup_user(TEST_USER_EMAIL) + url = self.__url("information/inputs", user) + app = self._test_driver.app + + # create some initial data + put(url, data=json.dumps({ + "vaulttestsection|client_id": "hello_client_id", + "vaulttestsection|client_secret": "hello_client_secret", + })) + + # retrieve saved data + response = get(url).json() + + def get_input_by_name(inputs, name): + return [input for input in inputs if input['name'] == name][0] + + inputs = [section for section in response["inputs"] if section['name'] == 'vaulttestsection'][0]["inputs"] + db_user = app.model.context.query(app.model.User).filter(app.model.User.email == user['email']).first() + + # value should be what we saved + input_client_id = get_input_by_name(inputs, 'client_id') + self.assertEqual(input_client_id['value'], "hello_client_id") + + # however, this value should not be in the vault + self.assertIsNone(app.vault.read_secret(f"user/{db_user.id}/preferences/vaulttestsection/client_id")) + # it should be in the user preferences model + self.assertEqual(db_user.extra_preferences['vaulttestsection|client_id'], "hello_client_id") + + # the secret however, was configured to be stored in the vault + input_client_secret = get_input_by_name(inputs, 'client_secret') + self.assertEqual(input_client_secret['value'], "hello_client_secret") + self.assertEqual(app.vault.read_secret( + f"user/{db_user.id}/preferences/vaulttestsection/client_secret"), "hello_client_secret") + # it should not be stored in the user preferences model + self.assertIsNone(db_user.extra_preferences['vaulttestsection|client_secret']) + + def __url(self, action, user): + return self._api_url(f"users/{user['id']}/{action}", params=dict(key=self.master_api_key)) diff --git a/lib/galaxy_test/api/user_preferences_extra_conf.yml b/lib/galaxy_test/api/user_preferences_extra_conf.yml new file mode 100644 index 000000000000..51a5e7d2f9c8 --- /dev/null +++ b/lib/galaxy_test/api/user_preferences_extra_conf.yml @@ -0,0 +1,23 @@ +preferences: + vaulttestsection: + description: A dummy extra prefs section + inputs: + - name: client_id + label: Client ID + type: text + required: True + - name: client_secret + label: Client Secret + type: password + store: vault + required: True + - name: access_token + label: Access token + type: password + store: vault + required: True + - name: refresh_token + label: Refresh Token + type: password + store: vault + required: True diff --git a/lib/galaxy_test/api/vault_conf.yml b/lib/galaxy_test/api/vault_conf.yml new file mode 100644 index 000000000000..9064f7a4734e --- /dev/null +++ b/lib/galaxy_test/api/vault_conf.yml @@ -0,0 +1,13 @@ +type: database +# Encryption keys must be valid fernet keys +# To generate a valid key: +# >>> from cryptography.fernet import Fernet +# >>> Fernet.generate_key() +# b'pZDP8_baVs3oWT4597HJWCysm49j-XELONQ-EdoU0DE=' +# +# Use the ascii string value as a key +# For more details, see: https://cryptography.io/en/latest/fernet/# +encryption_keys: + - 5RrT94ji178vQwha7TAmEix7DojtsLlxVz8Ef17KWgg= + - iNdXd7tRjLnSqRHxuhqQ98GTLU8HUbd5_Xx38iF8nZ0= + - IK83IXhE4_7W7xCFEtD9op0BAs11pJqYN236Spppp7g= From 0572f62075f54f3e48836dc711f4b4cb9feb19f6 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Mon, 22 Nov 2021 00:49:12 +0530 Subject: [PATCH 019/401] Fix vault linting errors --- lib/galaxy/security/vault.py | 4 ++-- lib/galaxy_test/api/test_vault.py | 3 +-- test/unit/app/dependencies/test_deps.py | 1 + test/unit/security/test_vault.py | 7 ++++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index 360bd16379f3..6a3fb4798eb8 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -1,9 +1,9 @@ -from abc import ABC import json import logging import os -import yaml +from abc import ABC +import yaml from cryptography.fernet import Fernet, MultiFernet try: diff --git a/lib/galaxy_test/api/test_vault.py b/lib/galaxy_test/api/test_vault.py index f8e83bf92ec4..519d41f8f51e 100644 --- a/lib/galaxy_test/api/test_vault.py +++ b/lib/galaxy_test/api/test_vault.py @@ -2,14 +2,13 @@ import os from requests import ( - delete, get, put ) from ._framework import ApiTestCase -TEST_USER_EMAIL = "user_for_users_index_test@bx.psu.edu" +TEST_USER_EMAIL = "vault_test_user@bx.psu.edu" class VaultApiTestCase(ApiTestCase): diff --git a/test/unit/app/dependencies/test_deps.py b/test/unit/app/dependencies/test_deps.py index 9e7bd4a04006..216cd0308b72 100644 --- a/test/unit/app/dependencies/test_deps.py +++ b/test/unit/app/dependencies/test_deps.py @@ -28,6 +28,7 @@ runners: runner1: load: job_runner_A +""" VAULT_CONF_CUSTOS = """ type: custos """ diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py index 9cf5adb73f46..3c9768c84260 100644 --- a/test/unit/security/test_vault.py +++ b/test/unit/security/test_vault.py @@ -1,8 +1,10 @@ -from abc import ABC import os import string import tempfile import unittest +from abc import ABC + +from cryptography.fernet import InvalidToken from galaxy.app_unittest_utils.galaxy_mock import MockApp, MockAppConfig from galaxy.security import vault @@ -53,7 +55,6 @@ def test_rotate_keys(self): app.config.vault_config_file = VAULT_CONF_DATABASE_ROTATED self.vault = vault.VaultFactory.from_app(app) self.assertEqual(self.vault.read_secret("my/rotated/secret"), "hello rotated") - super().test_read_write_secret() def test_wrong_keys(self): config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) @@ -64,7 +65,7 @@ def test_wrong_keys(self): # should fail because decryption keys are the wrong app.config.vault_config_file = VAULT_CONF_DATABASE_INVALID self.vault = vault.VaultFactory.from_app(app) - with self.assertRaises(Exception): + with self.assertRaises(InvalidToken): self.vault.read_secret("my/incorrect/secret") From 55280263b424183125b7941a9bd7cd566029159a Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Mon, 22 Nov 2021 16:56:37 -0800 Subject: [PATCH 020/401] add test lint --- lib/galaxy_test/api/test_workflows.py | 13 ++++ .../data/test_workflow_with_input_tags.ga | 70 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 lib/galaxy_test/base/data/test_workflow_with_input_tags.ga diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index bdc5c24e95cf..2373b8b85d6b 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -1280,6 +1280,19 @@ def test_workflow_run_dynamic_output_collections_3(self): invocation_id = self.__invoke_workflow(workflow_id, inputs=inputs, history_id=history_id) self.wait_for_invocation_and_jobs(history_id, workflow_id, invocation_id) + @skip_without_tool("cat1") + @skip_without_tool("__FLATTEN__") + def test_workflow_input_tags(self): + workflow = self.workflow_populator.load_workflow_from_resource(name="test_workflow_with_input_tags") + workflow_id = self.workflow_populator.create_workflow(workflow) + downloaded_workflow = self._download_workflow(workflow_id) + count = 0 + tag_test = ["tag1", "tag2"] + for step in downloaded_workflow["steps"]: + current = json.loads(downloaded_workflow["steps"][step]["tool_state"]) + assert current["tag"] == tag_test[count] + count += 1 + @skip_without_tool('column_param') def test_empty_file_data_column_specified(self): # Regression test for https://github.com/galaxyproject/galaxy/pull/10981 diff --git a/lib/galaxy_test/base/data/test_workflow_with_input_tags.ga b/lib/galaxy_test/base/data/test_workflow_with_input_tags.ga new file mode 100644 index 000000000000..95a4d5d9c718 --- /dev/null +++ b/lib/galaxy_test/base/data/test_workflow_with_input_tags.ga @@ -0,0 +1,70 @@ +{ + "a_galaxy_workflow": "true", + "annotation": "simple workflow", + "format-version": "0.1", + "name": "TestWorkflow1", + "steps": { + "0": { + "annotation": "input1 description", + "content_id": null, + "errors": null, + "id": 0, + "input_connections": {}, + "inputs": [ + { + "description": "input1 description", + "name": "WorkflowInput1" + } + ], + "label": "WorkflowInput1", + "name": "Input dataset", + "outputs": [], + "position": { + "bottom": 370.65625, + "height": 61, + "left": 613.546875, + "right": 813.546875, + "top": 309.65625, + "width": 200, + "x": 613.546875, + "y": 309.65625 + }, + "tool_id": null, + "tool_state": "{\"optional\": false, \"tag\": \"tag1\"}", + "tool_version": null, + "type": "data_input", + "uuid": "5262f9f9-28d7-486f-9cf8-f7c01da4f31c", + "workflow_outputs": [] + }, + "1": { + "annotation": "", + "content_id": null, + "errors": null, + "id": 1, + "input_connections": {}, + "inputs": [], + "label": null, + "name": "Input dataset collection", + "outputs": [], + "position": { + "bottom": 472.5, + "height": 81, + "left": 616, + "right": 816, + "top": 391.5, + "width": 200, + "x": 616, + "y": 391.5 + }, + "tool_id": null, + "tool_state": "{\"optional\": false, \"tag\": \"tag2\", \"collection_type\": \"list\"}", + "tool_version": null, + "type": "data_collection_input", + "uuid": "f6ee8140-92cf-4bbd-8683-dc9076fd72fa", + "workflow_outputs": [] + } + }, + "tags": [], + "uuid": "6e6d28b0-cf38-47d3-937c-18738da04306", + "version": 4 +} \ No newline at end of file From 062618bd2e4ce3922ad8f66a44de16773745e9a2 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:22:38 +0530 Subject: [PATCH 021/401] Fix typing errors in vault --- lib/galaxy/app_unittest_utils/galaxy_mock.py | 1 + lib/galaxy/config/sample/galaxy.yml.sample | 2 +- lib/galaxy/security/vault.py | 33 ++++++++++++++------ lib/galaxy_test/api/test_vault.py | 3 +- test/unit/security/test_vault.py | 21 +++++++------ 5 files changed, 39 insertions(+), 21 deletions(-) diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index bb1d7f130f24..3e82de3f4c39 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -182,6 +182,7 @@ def __init__(self, **kwargs): self.enable_tool_shed_check = False self.monitor_thread_join_timeout = 1 self.integrated_tool_panel_config = None + self.vault_config_file = None @property def config_dict(self): diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 70fe52f31404..391243a5fec3 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -2356,4 +2356,4 @@ galaxy: # Vault config file # The value of this option will be resolved with respect to # . - vault_config_file: vault_conf.yml \ No newline at end of file + vault_config_file: vault_conf.yml diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index 6a3fb4798eb8..8a6de5cd4943 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -1,7 +1,8 @@ +import abc import json import logging import os -from abc import ABC +from typing import Optional import yaml from cryptography.fernet import Fernet, MultiFernet @@ -28,9 +29,20 @@ class UnknownVaultTypeException(Exception): pass -class Vault(ABC): +class Vault(abc.ABC): - def read_secret(self, key: str) -> str: + @abc.abstractmethod + def read_secret(self, key: str) -> Optional[str]: + pass + + @abc.abstractmethod + def write_secret(self, key: str, value: str) -> None: + pass + + +class NullVault(Vault): + + def read_secret(self, key: str) -> Optional[str]: raise UnknownVaultTypeException("No vault configured. Make sure the vault_config_file setting is defined in galaxy.yml") def write_secret(self, key: str, value: str) -> None: @@ -46,7 +58,7 @@ def __init__(self, config): self.vault_token = config.get('vault_token') self.client = hvac.Client(url=self.vault_address, token=self.vault_token) - def read_secret(self, key: str) -> str: + def read_secret(self, key: str) -> Optional[str]: try: response = self.client.secrets.kv.read_secret_version(path=key) return response['data']['data'].get('value') @@ -76,7 +88,7 @@ def _update_or_create(self, key: str, value: str): self.sa_session.merge(vault_entry) self.sa_session.flush() - def read_secret(self, key: str) -> str: + def read_secret(self, key: str) -> Optional[str]: key_obj = self.sa_session.query(model.Vault).filter_by(key=key).first() if key_obj: f = self._get_multi_fernet() @@ -101,7 +113,7 @@ def __init__(self, config): self.b64_encoded_custos_token = custos_util.get_token(custos_settings=self.custos_settings) self.client = ResourceSecretManagementClient(self.custos_settings) - def read_secret(self, key: str) -> str: + def read_secret(self, key: str) -> Optional[str]: try: response = self.client.get_KV_credential(token=self.b64_encoded_custos_token, client_id=self.custos_settings.CUSTOS_CLIENT_ID, @@ -127,7 +139,7 @@ def __init__(self, vault: Vault, user): self.vault = vault self.user = user - def read_secret(self, key: str) -> str: + def read_secret(self, key: str) -> Optional[str]: return self.vault.read_secret(f"user/{self.user.id}/{key}") def write_secret(self, key: str, value: str) -> None: @@ -137,14 +149,14 @@ def write_secret(self, key: str, value: str) -> None: class VaultFactory(object): @staticmethod - def load_vault_config(vault_conf_yml: str) -> dict: + def load_vault_config(vault_conf_yml: str) -> Optional[dict]: if os.path.exists(vault_conf_yml): with open(vault_conf_yml) as f: return yaml.safe_load(f) return None @staticmethod - def from_vault_type(app, vault_type: str, cfg: dict) -> Vault: + def from_vault_type(app, vault_type: Optional[str], cfg: dict) -> Vault: if vault_type == "hashicorp": return HashicorpVault(cfg) elif vault_type == "database": @@ -158,5 +170,6 @@ def from_vault_type(app, vault_type: str, cfg: dict) -> Vault: def from_app(app) -> Vault: vault_config = VaultFactory.load_vault_config(app.config.vault_config_file) if vault_config: - return VaultFactory.from_vault_type(app, vault_config.get('type'), vault_config) + return VaultFactory.from_vault_type(app, vault_config.get('type', None), vault_config) log.warning("No vault configured. We recommend defining the vault_config_file setting in galaxy.yml") + return NullVault() diff --git a/lib/galaxy_test/api/test_vault.py b/lib/galaxy_test/api/test_vault.py index 519d41f8f51e..459bffff12ae 100644 --- a/lib/galaxy_test/api/test_vault.py +++ b/lib/galaxy_test/api/test_vault.py @@ -1,5 +1,6 @@ import json import os +from typing import cast, Any from requests import ( get, @@ -21,7 +22,7 @@ def handle_galaxy_config_kwds(cls, config): def test_extra_prefs_vault_storage(self): user = self._setup_user(TEST_USER_EMAIL) url = self.__url("information/inputs", user) - app = self._test_driver.app + app = cast(Any, self._test_driver.app if self._test_driver else None) # create some initial data put(url, data=json.dumps({ diff --git a/test/unit/security/test_vault.py b/test/unit/security/test_vault.py index 3c9768c84260..4585b5773029 100644 --- a/test/unit/security/test_vault.py +++ b/test/unit/security/test_vault.py @@ -7,10 +7,13 @@ from cryptography.fernet import InvalidToken from galaxy.app_unittest_utils.galaxy_mock import MockApp, MockAppConfig -from galaxy.security import vault +from galaxy.security.vault import NullVault, Vault, VaultFactory -class VaultTestBase(ABC): +class VaultTestBase(ABC, unittest.TestCase): + + def __init__(self): + self.vault = NullVault() # type: Vault def test_read_write_secret(self): self.vault.write_secret("my/test/secret", "hello world") @@ -30,7 +33,7 @@ class TestHashicorpVault(VaultTestBase, unittest.TestCase): def setUp(self) -> None: config = MockAppConfig(vault_config_file=VAULT_CONF_HASHICORP) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app(app) + self.vault = VaultFactory.from_app(app) VAULT_CONF_DATABASE = os.path.join(os.path.dirname(__file__), "fixtures/vault_conf_database.yaml") @@ -43,28 +46,28 @@ class TestDatabaseVault(VaultTestBase, unittest.TestCase): def setUp(self) -> None: config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app(app) + self.vault = VaultFactory.from_app(app) def test_rotate_keys(self): config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app(app) + self.vault = VaultFactory.from_app(app) self.vault.write_secret("my/rotated/secret", "hello rotated") # should succeed after rotation app.config.vault_config_file = VAULT_CONF_DATABASE_ROTATED - self.vault = vault.VaultFactory.from_app(app) + self.vault = VaultFactory.from_app(app) self.assertEqual(self.vault.read_secret("my/rotated/secret"), "hello rotated") def test_wrong_keys(self): config = MockAppConfig(vault_config_file=VAULT_CONF_DATABASE) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app(app) + self.vault = VaultFactory.from_app(app) self.vault.write_secret("my/incorrect/secret", "hello incorrect") # should fail because decryption keys are the wrong app.config.vault_config_file = VAULT_CONF_DATABASE_INVALID - self.vault = vault.VaultFactory.from_app(app) + self.vault = VaultFactory.from_app(app) with self.assertRaises(InvalidToken): self.vault.read_secret("my/incorrect/secret") @@ -82,7 +85,7 @@ def setUp(self) -> None: self.vault_temp_conf = tempconf.name config = MockAppConfig(vault_config_file=self.vault_temp_conf) app = MockApp(config=config) - self.vault = vault.VaultFactory.from_app(app) + self.vault = VaultFactory.from_app(app) def tearDown(self) -> None: os.remove(self.vault_temp_conf) From 4e17fcc851e469eca6ca3fa3ac3078d0e2a523cd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 26 Nov 2021 10:05:44 +0100 Subject: [PATCH 022/401] Reconstructed separate scoped session for ToolShedRepositoryCache If we just construct a single session, but the underlying connection is closed, we can't use the session anymore, at all. --- lib/galaxy/tools/cache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index 25a551d7f4bd..ca23c0768c8c 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -13,6 +13,7 @@ defer, joinedload, ) +from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlitedict import SqliteDict @@ -287,7 +288,8 @@ class ToolShedRepositoryCache: repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]] def __init__(self, session: sessionmaker): - self.session = session() + engine = session().bind + self.session = scoped_session(sessionmaker(engine)) # Contains ToolConfRepository objects created from shed_tool_conf.xml entries self.local_repositories = [] # Repositories loaded from database From 1cc6905c3f124adf541d71f7518e9ecb5d77566e Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 15 Nov 2021 20:18:31 +0100 Subject: [PATCH 023/401] Add get_filter_query_params function to api/common.py module --- lib/galaxy/webapps/galaxy/api/common.py | 61 ++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/common.py b/lib/galaxy/webapps/galaxy/api/common.py index 287d478af7a5..3632cfef498f 100644 --- a/lib/galaxy/webapps/galaxy/api/common.py +++ b/lib/galaxy/webapps/galaxy/api/common.py @@ -1,9 +1,17 @@ """This module contains utility functions shared across the api package.""" -from typing import Any, Dict, Optional +from typing import ( + Any, + Dict, + List, + Optional, +) from fastapi import Query -from galaxy.schema import SerializationParams +from galaxy.schema import ( + FilterQueryParams, + SerializationParams, +) from galaxy.schema.schema import UpdateDatasetPermissionsPayload SerializationViewQueryParam: Optional[str] = Query( @@ -45,6 +53,55 @@ def query_serialization_params( return parse_serialization_params(view=view, keys=keys, default_view=default_view) +def get_filter_query_params( + q: Optional[List[str]] = Query( + default=None, + title="Filter Query", + description="Generally a property name to filter by followed by an (often optional) hyphen and operator string.", + example="create_time-gt", + ), + qv: Optional[List[str]] = Query( + default=None, + title="Filter Value", + description="The value to filter by.", + example="2015-01-29", + ), + offset: Optional[int] = Query( + default=0, + ge=0, + title="Offset", + description="Starts at the beginning skip the first ( offset - 1 ) items and begin returning at the Nth item", + ), + limit: Optional[int] = Query( + default=None, + ge=1, + title="Limit", + description="The maximum number of items to return.", + ), + order: Optional[str] = Query( + default=None, + title="Order", + description=( + "String containing one of the valid ordering attributes followed (optionally) " + "by '-asc' or '-dsc' for ascending and descending order respectively. " + "Orders can be stacked as a comma-separated list of values." + ), + example="name-dsc,create_time", + ), +) -> FilterQueryParams: + """ + This function is meant to be used as a Dependency. + See https://fastapi.tiangolo.com/tutorial/dependencies/#first-steps + """ + return FilterQueryParams( + q=q, + qv=qv, + offset=offset, + limit=limit, + order=order, + ) + + def get_update_permission_payload(payload: Dict[str, Any]) -> UpdateDatasetPermissionsPayload: """Coverts the generic payload dictionary into a UpdateDatasetPermissionsPayload model with custom parsing. This is an attempt on supporting multiple aliases for the permissions params.""" From 6add64c8ed1060f25036d89a7e8a54f485901d94 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 15 Nov 2021 20:23:38 +0100 Subject: [PATCH 024/401] Small refactoring and fixes - Fix some default values - Add open_file option to get_metadata_file to for use in in legacy mode --- lib/galaxy/webapps/galaxy/api/datasets.py | 19 ++++++-- .../webapps/galaxy/services/datasets.py | 48 +++++++++++++------ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 75b111448e8e..17c420e3bbde 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -7,8 +7,12 @@ util, web ) -from galaxy.schema import FilterQueryParams -from galaxy.webapps.base.controller import UsesVisualizationMixin +from galaxy.schema import ( + FilterQueryParams, +) +from galaxy.schema.schema import ( + DatasetSourceType, +) from galaxy.webapps.galaxy.api.common import ( get_update_permission_payload, parse_serialization_params, @@ -81,8 +85,10 @@ def index(self, trans, limit=500, offset=0, history_id=None, **kwd): """ serialization_params = parse_serialization_params(**kwd) filter_parameters = FilterQueryParams(**kwd) + filter_parameters.limit = filter_parameters.limit or limit + filter_parameters.offset = filter_parameters.offset or offset return self.service.index( - trans, limit, offset, history_id, serialization_params, filter_parameters + trans, history_id, serialization_params, filter_parameters ) @web.expose_api_anonymous_and_sessionless @@ -131,7 +137,7 @@ def update_permissions(self, trans, dataset_id, payload, **kwd): :rtype: dict :returns: dictionary containing new permissions """ - hda_ldda = kwd.pop('hda_ldda', 'hda') + hda_ldda = kwd.pop('hda_ldda', DatasetSourceType.hda) if payload: kwd.update(payload) update_payload = get_update_permission_payload(kwd) @@ -173,7 +179,10 @@ def get_metadata_file(self, trans, history_content_id, history_id, metadata_file """ GET /api/histories/{history_id}/contents/{history_content_id}/metadata_file """ - metadata_file, headers = self.service.get_metadata_file(trans, history_content_id, metadata_file) + # TODO: remove open_file parameter when deleting this legacy endpoint + metadata_file, headers = self.service.get_metadata_file( + trans, history_content_id, metadata_file, open_file=True + ) trans.response.headers.update(headers) return metadata_file diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index 7b7698e303d6..fc48cea5fc0f 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -12,10 +12,14 @@ Union, ) -from pydantic import BaseModel, Extra, Field +from pydantic import ( + BaseModel, + Extra, + Field, +) +from galaxy import exceptions as galaxy_exceptions from galaxy import ( - exceptions as galaxy_exceptions, model, util, web, @@ -23,12 +27,21 @@ from galaxy.datatypes import dataproviders from galaxy.managers.base import ModelSerializer from galaxy.managers.context import ProvidesHistoryContext -from galaxy.managers.hdas import HDAManager, HDASerializer +from galaxy.managers.hdas import ( + HDAManager, + HDASerializer, +) from galaxy.managers.hdcas import HDCASerializer from galaxy.managers.histories import HistoryManager -from galaxy.managers.history_contents import HistoryContentsFilters, HistoryContentsManager +from galaxy.managers.history_contents import ( + HistoryContentsFilters, + HistoryContentsManager, +) from galaxy.managers.lddas import LDDAManager -from galaxy.schema import FilterQueryParams, SerializationParams +from galaxy.schema import ( + FilterQueryParams, + SerializationParams, +) from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.schema.schema import ( AnyHDA, @@ -40,13 +53,11 @@ ) from galaxy.schema.types import RelativeUrl from galaxy.security.idencoding import IdEncodingHelper -from galaxy.util.path import ( - safe_walk -) +from galaxy.util.path import safe_walk from galaxy.visualization.data_providers.genome import ( BamDataProvider, FeatureLocationIndexDataProvider, - SamDataProvider + SamDataProvider, ) from galaxy.visualization.data_providers.registry import DataProviderRegistry from galaxy.webapps.base.controller import UsesVisualizationMixin @@ -54,6 +65,8 @@ log = logging.getLogger(__name__) +DEFAULT_LIMIT = 500 + class RequestDataType(str, Enum): """Particular pieces of information that can be requested for a dataset.""" @@ -189,9 +202,7 @@ def serializer_by_type(self) -> Dict[str, ModelSerializer]: def index( self, trans: ProvidesHistoryContext, - limit: Optional[int], - offset: Optional[int], - history_id: EncodedDatabaseIdField, + history_id: Optional[EncodedDatabaseIdField], serialization_params: SerializationParams, filter_query_params: FilterQueryParams, ) -> List[AnyHistoryContentItem]: @@ -207,7 +218,12 @@ def index( if history_id: container = self.history_manager.get_accessible(self.decode_id(history_id), user) contents = self.history_contents_manager.contents( - container=container, filters=filters, limit=limit, offset=offset, order_by=order_by, user_id=user.id, + container=container, + filters=filters, + limit=filter_query_params.limit or DEFAULT_LIMIT, + offset=filter_query_params.offset, + order_by=order_by, + user_id=user.id, ) return [ self.serializer_by_type[content.history_content_type].serialize_to_view(content, user=user, trans=trans, view=view) @@ -419,6 +435,7 @@ def get_metadata_file( trans: ProvidesHistoryContext, history_content_id: EncodedDatabaseIdField, metadata_file: Optional[str] = None, + open_file: bool = False, ): """ Gets the associated metadata file. @@ -430,7 +447,10 @@ def get_metadata_file( headers = {} headers["Content-Type"] = "application/octet-stream" headers["Content-Disposition"] = f'attachment; filename="Galaxy{hda.hid}-[{fname}].{file_ext}"' - return open(hda.metadata.get(metadata_file).file_name, 'rb'), headers + file_path = hda.metadata.get(metadata_file).file_name + if open_file: + return open(file_path, 'rb'), headers + return file_path, headers def converted( self, From eb801968691327337ac374ec972f27976a3e15b3 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 15 Nov 2021 20:27:42 +0100 Subject: [PATCH 025/401] Add FastAPI routes for datasets API --- lib/galaxy/webapps/galaxy/api/datasets.py | 213 +++++++++++++++++++++- 1 file changed, 210 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 17c420e3bbde..b737aec7ebc5 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -2,31 +2,238 @@ API operations on the contents of a history dataset. """ import logging +from typing import ( + Any, + cast, + Dict, + List, + Optional, +) + +from fastapi import ( + Body, + Depends, + Path, + Query, + Request, +) +from starlette.responses import FileResponse from galaxy import ( util, - web + web, ) from galaxy.schema import ( FilterQueryParams, + SerializationParams, ) +from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.schema.schema import ( + AnyHistoryContentItem, + DatasetAssociationRoles, DatasetSourceType, + UpdateDatasetPermissionsPayload, ) from galaxy.webapps.galaxy.api.common import ( + get_filter_query_params, get_update_permission_payload, parse_serialization_params, + query_serialization_params, ) from galaxy.webapps.galaxy.services.datasets import ( + DatasetInheritanceChainEntry, DatasetShowParams, DatasetsService, + DatasetTextContentDetails, +) +from . import ( + BaseGalaxyAPIController, + depends, + DependsOnTrans, + Router, ) -from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) +router = Router(tags=['datasets']) + +DatasetIDPathParam: EncodedDatabaseIdField = Path( + ..., + description="The encoded database identifier of the dataset." +) + +DatasetSourceQueryParam: DatasetSourceType = Query( + default=DatasetSourceType.hda, + description="Whether this dataset belongs to a history (HDA) or a library (LDDA).", +) + +HistoryIDPathParam: EncodedDatabaseIdField = Path( + ..., + description="The encoded database identifier of the History." +) + + +@router.cbv +class FastAPIDatasets: + service: DatasetsService = depends(DatasetsService) + + @router.get( + '/api/datasets', + summary='Search datasets or collections using a query system.', + ) + def index( + self, + trans=DependsOnTrans, + history_id: Optional[EncodedDatabaseIdField] = Query( + default=None, + description="Optional identifier of a History. Use it to restrict the search whithin a particular History." + ), + serialization_params: SerializationParams = Depends(query_serialization_params), + filter_query_params: FilterQueryParams = Depends(get_filter_query_params), + ) -> List[AnyHistoryContentItem]: + return self.service.index(trans, history_id, serialization_params, filter_query_params) + + @router.get( + '/api/datasets/{dataset_id}/storage', + summary='Display user-facing storage details related to the objectstore a dataset resides in.', + ) + def show_storage( + self, + trans=DependsOnTrans, + dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, + hda_ldda: DatasetSourceType = DatasetSourceQueryParam, + ): + return self.service.show_storage(trans, dataset_id, hda_ldda) + + @router.get( + '/api/datasets/{dataset_id}/inheritance_chain', + summary='For internal use, this endpoint may change without warning.', + include_in_schema=False, + ) + def show_inheritance_chain( + self, + trans=DependsOnTrans, + dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, + hda_ldda: DatasetSourceType = DatasetSourceQueryParam, + ) -> List[DatasetInheritanceChainEntry]: + return self.service.show_inheritance_chain(trans, dataset_id, hda_ldda) + + @router.get( + '/api/datasets/{dataset_id}/get_content_as_text', + summary='Returns item content as Text.', + ) + def get_content_as_text( + self, + trans=DependsOnTrans, + dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, + ) -> DatasetTextContentDetails: + return self.service.get_content_as_text(trans, dataset_id) + + @router.get( + '/api/datasets/{dataset_id}/converted/{ext}', + summary='Return information about datasets made by converting this dataset to a new format', + ) + def converted( + self, + trans=DependsOnTrans, + dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, + ext: Optional[str] = Query( + default=None, + description="TODO", + ), + serialization_params: SerializationParams = Depends(query_serialization_params), + ): + return self.service.converted(trans, dataset_id, ext, serialization_params) + + @router.put( + '/api/datasets/{dataset_id}/permissions', + summary='Set permissions of the given history dataset to the given role ids.', + ) + def update_permissions( + self, + trans=DependsOnTrans, + dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, + # Using a generic Dict here as an attempt on supporting multiple aliases for the permissions params. + payload: Dict[str, Any] = Body( + default=..., + example=UpdateDatasetPermissionsPayload(), + ), + ) -> DatasetAssociationRoles: + """Set permissions of the given history dataset to the given role ids.""" + update_payload = get_update_permission_payload(payload) + return self.service.update_permissions(trans, dataset_id, update_payload) + + @router.get( + '/api/histories/{history_id}/contents/{history_content_id}/extra_files', + summary='Generate list of extra files.', + tags=["histories"], + ) + def extra_files( + self, + trans=DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + history_content_id: EncodedDatabaseIdField = DatasetIDPathParam, + ): + return self.service.extra_files(trans, history_content_id) -class DatasetsController(BaseGalaxyAPIController, UsesVisualizationMixin): + @router.get( + '/api/histories/{history_id}/contents/{history_content_id}/display', + summary='Displays history content (dataset).', + tags=["histories"], + ) + def display( + self, + request: Request, + trans=DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + history_content_id: EncodedDatabaseIdField = DatasetIDPathParam, + preview: bool = Query( + default=False, + description="TODO", + ), + filename: Optional[str] = Query( + default=None, + description="TODO", + ), + to_ext: Optional[str] = Query( + default=None, + description="TODO", + ), + raw: bool = Query( + default=False, + description=( + "The query parameter 'raw' should be considered experimental and may be dropped at " + "some point in the future without warning. Generally, data should be processed by its " + "datatype prior to display." + ), + ), + ): + exclude_params = set(["preview", "filename", "to_ext", "raw"]) + extra_params = request.query_params._dict + for p in exclude_params: + extra_params.pop(p, None) + return self.service.display(trans, history_content_id, history_id, preview, filename, to_ext, raw, **extra_params) + + @router.get( + '/api/histories/{history_id}/contents/{history_content_id}/metadata_file', + summary='Returns the metadata file associated with this history item.', + tags=["histories"], + ) + def get_metadata_file( + self, + trans=DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + history_content_id: EncodedDatabaseIdField = DatasetIDPathParam, + metadata_file: Optional[str] = Query( + default=None, + description="TODO", + ), + ): + metadata_file_path, headers = self.service.get_metadata_file(trans, history_content_id, metadata_file) + return FileResponse(path=cast(str, metadata_file_path), headers=headers) + + +class DatasetsController(BaseGalaxyAPIController): service: DatasetsService = depends(DatasetsService) @web.expose_api From 943ea3c7994fdc22b75f1facee4f0d9d3d94cd06 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 16 Nov 2021 16:21:25 +0100 Subject: [PATCH 026/401] Remove unneeded key deletion from kwd This was added in 2013 to prevent the API key to get into the display data but is not needed anymore. --- lib/galaxy/webapps/galaxy/services/datasets.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index fc48cea5fc0f..a9d1e648415c 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -398,10 +398,7 @@ def display( file_path = hda.file_name rval = open(file_path, 'rb') else: - display_kwd = kwd.copy() - if 'key' in display_kwd: - del display_kwd["key"] - rval, headers = hda.datatype.display_data(trans, hda, preview, filename, to_ext, **display_kwd) + rval, headers = hda.datatype.display_data(trans, hda, preview, filename, to_ext, **kwd) except galaxy_exceptions.MessageException: raise except Exception as e: From da163872906bbfc67850864555f003610cbde9f6 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 16 Nov 2021 17:13:10 +0100 Subject: [PATCH 027/401] Add common way of extracting query params (excluding those that are already handled by the endpoint definition) --- lib/galaxy/webapps/galaxy/api/common.py | 19 ++++++++++++++++++- lib/galaxy/webapps/galaxy/api/datasets.py | 6 ++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/common.py b/lib/galaxy/webapps/galaxy/api/common.py index 3632cfef498f..2a9885db6c9a 100644 --- a/lib/galaxy/webapps/galaxy/api/common.py +++ b/lib/galaxy/webapps/galaxy/api/common.py @@ -4,9 +4,13 @@ Dict, List, Optional, + Set, ) -from fastapi import Query +from fastapi import ( + Query, + Request, +) from galaxy.schema import ( FilterQueryParams, @@ -113,3 +117,16 @@ def get_update_permission_payload(payload: Dict[str, Any]) -> UpdateDatasetPermi payload["modify_ids"] = payload.get("modify_ids[]") or payload.get("modify") update_payload = UpdateDatasetPermissionsPayload(**payload) return update_payload + + +def get_query_parameters_from_request_excluding(request: Request, exclude: Set[str]) -> dict: + """Gets all the request query parameters excluding the given parameters names in `exclude` set. + + This is useful when an endpoint uses arbitrary or dynamic query parameters that + cannot be anticipated or documented beforehand. The `exclude` set can be used to avoid + including those parameters that are already handled by the endpoint. + """ + extra_params = request.query_params._dict + for param_name in exclude: + extra_params.pop(param_name, None) + return extra_params diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index b737aec7ebc5..dabb54f923c1 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -36,6 +36,7 @@ ) from galaxy.webapps.galaxy.api.common import ( get_filter_query_params, + get_query_parameters_from_request_excluding, get_update_permission_payload, parse_serialization_params, query_serialization_params, @@ -208,10 +209,7 @@ def display( ), ), ): - exclude_params = set(["preview", "filename", "to_ext", "raw"]) - extra_params = request.query_params._dict - for p in exclude_params: - extra_params.pop(p, None) + extra_params = get_query_parameters_from_request_excluding(request, {"preview", "filename", "to_ext", "raw"}) return self.service.display(trans, history_content_id, history_id, preview, filename, to_ext, raw, **extra_params) @router.get( From f4a7a76de28452a71f2419ec4462bcd6208c4e9d Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 17 Nov 2021 11:15:27 +0100 Subject: [PATCH 028/401] Fix display operation This operation returns mostly file contents, chunks of files, raw text, etc. Looks like StreamingResponse is the a good solution for these kind of contents. --- lib/galaxy/webapps/galaxy/api/datasets.py | 9 +++++++-- lib/galaxy/webapps/galaxy/services/datasets.py | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index dabb54f923c1..1d17b00ec793 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -17,7 +17,10 @@ Query, Request, ) -from starlette.responses import FileResponse +from starlette.responses import ( + FileResponse, + StreamingResponse, +) from galaxy import ( util, @@ -181,6 +184,7 @@ def extra_files( '/api/histories/{history_id}/contents/{history_content_id}/display', summary='Displays history content (dataset).', tags=["histories"], + response_class=StreamingResponse, ) def display( self, @@ -210,7 +214,8 @@ def display( ), ): extra_params = get_query_parameters_from_request_excluding(request, {"preview", "filename", "to_ext", "raw"}) - return self.service.display(trans, history_content_id, history_id, preview, filename, to_ext, raw, **extra_params) + display_content = self.service.display(trans, history_content_id, history_id, preview, filename, to_ext, raw, **extra_params) + return StreamingResponse(display_content) @router.get( '/api/histories/{history_id}/contents/{history_content_id}/metadata_file', diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index a9d1e648415c..57d87dbecdd8 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -436,6 +436,9 @@ def get_metadata_file( ): """ Gets the associated metadata file. + + The `open_file` parameter determines if we return the path of the file or the opened file handle. + TODO: Remove the `open_file` parameter when removing the associated legacy endpoint. """ decoded_content_id = self.decode_id(history_content_id) hda = self.hda_manager.get_accessible(decoded_content_id, trans.user) From 3bc6bbc358157ad2eb2536b5ae4de6afe29bc096 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Nov 2021 12:35:52 +0100 Subject: [PATCH 029/401] Split converted operation to improve documentation This operation was doing two completely different things, using different parameters, etc. This should improve the documentation as well as the readability. --- lib/galaxy/webapps/galaxy/api/datasets.py | 45 +++++++++++++++---- .../webapps/galaxy/services/datasets.py | 45 +++++++++++++------ 2 files changed, 68 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 1d17b00ec793..3fb53e99aa89 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -32,6 +32,7 @@ ) from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.schema.schema import ( + AnyHDA, AnyHistoryContentItem, DatasetAssociationRoles, DatasetSourceType, @@ -45,6 +46,7 @@ query_serialization_params, ) from galaxy.webapps.galaxy.services.datasets import ( + ConvertedDatasetsMap, DatasetInheritanceChainEntry, DatasetShowParams, DatasetsService, @@ -135,19 +137,42 @@ def get_content_as_text( @router.get( '/api/datasets/{dataset_id}/converted/{ext}', - summary='Return information about datasets made by converting this dataset to a new format', + summary='Return information about datasets made by converting this dataset to a new format.', ) - def converted( + def converted_ext( self, trans=DependsOnTrans, dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, - ext: Optional[str] = Query( - default=None, - description="TODO", + ext: str = Path( + ..., + description="File extension of the new format to convert this dataset to.", ), serialization_params: SerializationParams = Depends(query_serialization_params), - ): - return self.service.converted(trans, dataset_id, ext, serialization_params) + ) -> AnyHDA: + """ + Return information about datasets made by converting this dataset to a new format. + + If there is no existing converted dataset for the format in `ext`, one will be created. + + **Note**: `view` and `keys` are also available to control the serialization of the dataset. + """ + return self.service.converted_ext(trans, dataset_id, ext, serialization_params) + + @router.get( + '/api/datasets/{dataset_id}/converted', + summary=( + "Return a a map with all the existing converted datasets associated with this instance." + ), + ) + def converted( + self, + trans=DependsOnTrans, + dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, + ) -> ConvertedDatasetsMap: + """ + Return a map of ` : ` containing all the *existing* converted datasets. + """ + return self.service.converted(trans, dataset_id) @router.put( '/api/datasets/{dataset_id}/permissions', @@ -423,5 +448,7 @@ def converted(self, trans, dataset_id, ext, **kwargs): :returns: dictionary containing detailed HDA information or (if `ext` is None) an extension->dataset_id map """ - serialization_params = parse_serialization_params(**kwargs) - return self.service.converted(trans, dataset_id, ext, serialization_params) + if ext: + serialization_params = parse_serialization_params(**kwargs) + return self.service.converted_ext(trans, dataset_id, ext, serialization_params) + return self.service.converted(trans, dataset_id) diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index 57d87dbecdd8..b8e4ee41fbbc 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -131,7 +131,16 @@ class DatasetTextContentDetails(Model): ) -ConvertedDatasetsMap = Dict[str, EncodedDatabaseIdField] # extension -> dataset ID +class ConvertedDatasetsMap(BaseModel): + """Map of `file extension` -> `converted dataset encoded id`""" + __root__: Dict[str, EncodedDatabaseIdField] # extension -> dataset ID + + class Config: + schema_extra = { + "example": { + "csv": "dataset_id", + } + } class DataMode(str, Enum): @@ -452,28 +461,38 @@ def get_metadata_file( return open(file_path, 'rb'), headers return file_path, headers - def converted( + def converted_ext( self, trans: ProvidesHistoryContext, dataset_id: EncodedDatabaseIdField, - ext: Optional[str], + ext: str, serialization_params: SerializationParams, - ) -> Union[AnyHDA, ConvertedDatasetsMap]: + ) -> AnyHDA: """ Return information about datasets made by converting this dataset to a new format """ decoded_id = self.decode_id(dataset_id) hda = self.hda_manager.get_accessible(decoded_id, trans.user) - if ext: - serialization_params.default_view = "detailed" - converted = self._get_or_create_converted(trans, hda, ext) - return self.hda_serializer.serialize_to_view( - converted, - user=trans.user, - trans=trans, - **serialization_params.dict() - ) + serialization_params.default_view = "detailed" + converted = self._get_or_create_converted(trans, hda, ext) + return self.hda_serializer.serialize_to_view( + converted, + user=trans.user, + trans=trans, + **serialization_params.dict() + ) + def converted( + self, + trans: ProvidesHistoryContext, + dataset_id: EncodedDatabaseIdField, + ) -> ConvertedDatasetsMap: + """ + Return a `file extension` -> `converted dataset encoded id` map + with all the existing converted datasets associated with this instance. + """ + decoded_id = self.decode_id(dataset_id) + hda = self.hda_manager.get_accessible(decoded_id, trans.user) return self.hda_serializer.serialize_converted_datasets(hda, 'converted') def _get_or_create_converted(self, trans, original: model.DatasetInstance, target_ext: str): From 89ecddee32ba4e8b00cfafec4cd152ecfc71eac2 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Nov 2021 13:20:52 +0100 Subject: [PATCH 030/401] Additional small improvements to documentation --- lib/galaxy/webapps/galaxy/api/common.py | 2 +- lib/galaxy/webapps/galaxy/api/datasets.py | 23 +++++++++++-------- .../webapps/galaxy/services/datasets.py | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/common.py b/lib/galaxy/webapps/galaxy/api/common.py index 2a9885db6c9a..be66dddedfff 100644 --- a/lib/galaxy/webapps/galaxy/api/common.py +++ b/lib/galaxy/webapps/galaxy/api/common.py @@ -107,7 +107,7 @@ def get_filter_query_params( def get_update_permission_payload(payload: Dict[str, Any]) -> UpdateDatasetPermissionsPayload: - """Coverts the generic payload dictionary into a UpdateDatasetPermissionsPayload model with custom parsing. + """Converts the generic payload dictionary into a UpdateDatasetPermissionsPayload model with custom parsing. This is an attempt on supporting multiple aliases for the permissions params.""" # There are several allowed names for the same role list parameter, i.e.: `access`, `access_ids`, `access_ids[]` # The `access_ids[]` name is not pydantic friendly, so this will be modelled as an alias but we can only set one alias diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 3fb53e99aa89..5906538fa06d 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -50,6 +50,7 @@ DatasetInheritanceChainEntry, DatasetShowParams, DatasetsService, + DatasetStorageDetails, DatasetTextContentDetails, ) from . import ( @@ -68,16 +69,16 @@ description="The encoded database identifier of the dataset." ) -DatasetSourceQueryParam: DatasetSourceType = Query( - default=DatasetSourceType.hda, - description="Whether this dataset belongs to a history (HDA) or a library (LDDA).", -) - HistoryIDPathParam: EncodedDatabaseIdField = Path( ..., description="The encoded database identifier of the History." ) +DatasetSourceQueryParam: DatasetSourceType = Query( + default=DatasetSourceType.hda, + description="Whether this dataset belongs to a history (HDA) or a library (LDDA).", +) + @router.cbv class FastAPIDatasets: @@ -108,13 +109,13 @@ def show_storage( trans=DependsOnTrans, dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, hda_ldda: DatasetSourceType = DatasetSourceQueryParam, - ): + ) -> DatasetStorageDetails: return self.service.show_storage(trans, dataset_id, hda_ldda) @router.get( '/api/datasets/{dataset_id}/inheritance_chain', summary='For internal use, this endpoint may change without warning.', - include_in_schema=False, + include_in_schema=True, # Can be changed to False if we don't really want to expose this ) def show_inheritance_chain( self, @@ -126,7 +127,7 @@ def show_inheritance_chain( @router.get( '/api/datasets/{dataset_id}/get_content_as_text', - summary='Returns item content as Text.', + summary='Returns dataset content as Text.', ) def get_content_as_text( self, @@ -207,7 +208,7 @@ def extra_files( @router.get( '/api/histories/{history_id}/contents/{history_content_id}/display', - summary='Displays history content (dataset).', + summary='Displays dataset (preview) content.', tags=["histories"], response_class=StreamingResponse, ) @@ -238,6 +239,7 @@ def display( ), ), ): + """Streams the preview contents of a dataset to be displayed in a browser.""" extra_params = get_query_parameters_from_request_excluding(request, {"preview", "filename", "to_ext", "raw"}) display_content = self.service.display(trans, history_content_id, history_id, preview, filename, to_ext, raw, **extra_params) return StreamingResponse(display_content) @@ -246,6 +248,7 @@ def display( '/api/histories/{history_id}/contents/{history_content_id}/metadata_file', summary='Returns the metadata file associated with this history item.', tags=["histories"], + response_class=FileResponse, ) def get_metadata_file( self, @@ -254,7 +257,7 @@ def get_metadata_file( history_content_id: EncodedDatabaseIdField = DatasetIDPathParam, metadata_file: Optional[str] = Query( default=None, - description="TODO", + description="The name of the metadata file to retrieve.", ), ): metadata_file_path, headers = self.service.get_metadata_file(trans, history_content_id, metadata_file) diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index b8e4ee41fbbc..6a4168e99ced 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -421,7 +421,7 @@ def get_content_as_text( trans: ProvidesHistoryContext, dataset_id: EncodedDatabaseIdField, ) -> DatasetTextContentDetails: - """ Returns item content as Text. """ + """ Returns dataset content as Text. """ user = self.get_authenticated_user(trans) decoded_id = self.decode_id(dataset_id) hda = self.hda_manager.get_accessible(decoded_id, user) From 8711339379db366bc09bf96972954bb2a9c86290 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 18 Nov 2021 14:59:36 +0100 Subject: [PATCH 031/401] Add FastAPI `show` route with limited documentation --- lib/galaxy/webapps/galaxy/api/datasets.py | 43 +++++++++++++++-- .../webapps/galaxy/services/datasets.py | 46 ++++++------------- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index 5906538fa06d..01b3dc6a983a 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -48,10 +48,10 @@ from galaxy.webapps.galaxy.services.datasets import ( ConvertedDatasetsMap, DatasetInheritanceChainEntry, - DatasetShowParams, DatasetsService, DatasetStorageDetails, DatasetTextContentDetails, + RequestDataType, ) from . import ( BaseGalaxyAPIController, @@ -263,6 +263,42 @@ def get_metadata_file( metadata_file_path, headers = self.service.get_metadata_file(trans, history_content_id, metadata_file) return FileResponse(path=cast(str, metadata_file_path), headers=headers) + @router.get( + '/api/datasets/{dataset_id}', + summary="Displays information about and/or content of a dataset.", + ) + def show( + self, + request: Request, + trans=DependsOnTrans, + dataset_id: EncodedDatabaseIdField = DatasetIDPathParam, + hda_ldda: DatasetSourceType = Query( + default=DatasetSourceType.hda, + description=( + "The type of information about the dataset to be requested." + ), + ), + data_type: Optional[RequestDataType] = Query( + default=None, + description=( + "The type of information about the dataset to be requested. " + "Each of these values may require additional parameters in the request and " + "may return different responses." + ), + ), + serialization_params: SerializationParams = Depends(query_serialization_params), + ): + """ + **Note**: Due to the multipurpose nature of this endpoint, which can receive a wild variety of parameters + and return different kinds of responses, the documentation here will be limited. + To get more information please check the source code. + """ + exclude_params = set(["hda_ldda", "data_type"]) + exclude_params.update(SerializationParams.__fields__.keys()) + extra_params = get_query_parameters_from_request_excluding(request, exclude_params) + + return self.service.show(trans, dataset_id, hda_ldda, serialization_params, data_type, **extra_params) + class DatasetsController(BaseGalaxyAPIController): service: DatasetsService = depends(DatasetsService) @@ -337,12 +373,9 @@ def show(self, trans, id, hda_ldda='hda', data_type=None, provider=None, **kwd): """ serialization_params = parse_serialization_params(**kwd) kwd.update({ - "hda_ldda": hda_ldda, - "data_type": data_type, "provider": provider, }) - params = DatasetShowParams(**kwd) - rval = self.service.show(trans, id, params, serialization_params) + rval = self.service.show(trans, id, hda_ldda, serialization_params, data_type, **kwd) return rval @web.expose_api_anonymous diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py index 6a4168e99ced..403e9af33d57 100644 --- a/lib/galaxy/webapps/galaxy/services/datasets.py +++ b/lib/galaxy/webapps/galaxy/services/datasets.py @@ -14,7 +14,6 @@ from pydantic import ( BaseModel, - Extra, Field, ) @@ -148,26 +147,6 @@ class DataMode(str, Enum): Auto = "Auto" -class DatasetShowParams(BaseModel): - hda_ldda: DatasetSourceType = Field(default=DatasetSourceType.hda) - data_type: Optional[RequestDataType] = Field(default=None) - provider: Optional[str] = Field(default=None) - # Converted - chrom: Optional[str] = Field(default=None) - retry: bool = Field(default=False) - # Data - low: Optional[int] = Field(default=None) - high: Optional[int] = Field(default=None) - start_val: int = Field(default=0) - max_vals: Optional[int] = Field(default=None) - mode: Optional[DataMode] = Field(default=DataMode.Auto) - query: Optional[str] = Field(default=None) - dbkey: Optional[str] = Field(default=None) - - class Config: - extra = Extra.allow - - class DataResult(BaseModel): data: List[Any] dataset_type: Optional[str] @@ -243,38 +222,41 @@ def show( self, trans: ProvidesHistoryContext, id: EncodedDatabaseIdField, - params: DatasetShowParams, + hda_ldda: DatasetSourceType, serialization_params: SerializationParams, + data_type: Optional[RequestDataType] = None, + **extra_params, ): """ Displays information about and/or content of a dataset. """ - # Get dataset. - dataset = self.get_hda_or_ldda(trans, hda_ldda=params.hda_ldda, dataset_id=id) - params_dict = params.dict(exclude_unset=True) + dataset = self.get_hda_or_ldda(trans, hda_ldda=hda_ldda, dataset_id=id) # Use data type to return particular type of data. - data_type = params.data_type rval: Any if data_type == RequestDataType.state: rval = self._dataset_state(dataset) elif data_type == RequestDataType.converted_datasets_state: - rval = self._converted_datasets_state(trans, dataset, params.chrom, params.retry) + rval = self._converted_datasets_state( + trans, dataset, + chrom=extra_params.get("chrom", None), + retry=extra_params.get("retry", False), + ) elif data_type == RequestDataType.data: - rval = self._data(trans, dataset, **params_dict) + rval = self._data(trans, dataset, **extra_params) elif data_type == RequestDataType.features: - rval = self._search_features(trans, dataset, params.query) + rval = self._search_features(trans, dataset, query=extra_params.get("query", None)) elif data_type == RequestDataType.raw_data: - rval = self._raw_data(trans, dataset, **params_dict) + rval = self._raw_data(trans, dataset, **extra_params) elif data_type == RequestDataType.track_config: rval = self.get_new_track_config(trans, dataset) elif data_type == RequestDataType.genome_data: - rval = self._get_genome_data(trans, dataset, params.dbkey) + rval = self._get_genome_data(trans, dataset, dbkey=extra_params.get("dbkey", None)) elif data_type == RequestDataType.in_use_state: rval = self._dataset_in_use_state(dataset) else: # Default: return dataset as dict. - if params.hda_ldda == DatasetSourceType.hda: + if hda_ldda == DatasetSourceType.hda: return self.hda_serializer.serialize_to_view( dataset, view=serialization_params.view or 'detailed', From c64d1bc20046c6484443fff8e02dd7c1d8833458 Mon Sep 17 00:00:00 2001 From: oleg Date: Thu, 25 Nov 2021 20:18:34 +0100 Subject: [PATCH 032/401] fix history error --- client/src/components/HistoryView.vue | 97 ++++++++++--------- .../webapps/galaxy/controllers/history.py | 2 +- 2 files changed, 53 insertions(+), 46 deletions(-) diff --git a/client/src/components/HistoryView.vue b/client/src/components/HistoryView.vue index 2097ff141fdb..50b91acc1e3c 100644 --- a/client/src/components/HistoryView.vue +++ b/client/src/components/HistoryView.vue @@ -1,38 +1,43 @@ diff --git a/client/src/components/PluginList.vue b/client/src/components/Visualizations/PluginList.vue similarity index 96% rename from client/src/components/PluginList.vue rename to client/src/components/Visualizations/PluginList.vue index 519d6ced29f9..e8754eeb104f 100644 --- a/client/src/components/PluginList.vue +++ b/client/src/components/Visualizations/PluginList.vue @@ -67,6 +67,12 @@ import { getGalaxyInstance } from "app"; import axios from "axios"; export default { + props: { + datasetId: { + type: String, + default: null, + }, + }, data() { return { plugins: [], @@ -82,13 +88,11 @@ export default { }; }, created() { - const Galaxy = getGalaxyInstance(); let url = `${getAppRoot()}api/plugins`; - const dataset_id = Galaxy.params.dataset_id; - if (dataset_id) { + if (this.datasetId) { this.fixed = true; - this.selected = dataset_id; - url += `?dataset_id=${dataset_id}`; + this.selected = this.datasetId; + url += `?dataset_id=${this.datasetId}`; } axios .get(url) diff --git a/client/src/entry/analysis/AnalysisRouter.js b/client/src/entry/analysis/AnalysisRouter.js index d6218eda6770..506ada7aa74e 100644 --- a/client/src/entry/analysis/AnalysisRouter.js +++ b/client/src/entry/analysis/AnalysisRouter.js @@ -41,7 +41,7 @@ import RecentInvocations from "components/User/RecentInvocations.vue"; import ToolsView from "components/ToolsView/ToolsView.vue"; import ToolsJson from "components/ToolsView/ToolsSchemaJson/ToolsJson.vue"; import HistoryList from "mvc/history/history-list"; -import PluginList from "components/PluginList.vue"; +import VisualizationsList from "components/Visualizations/Index"; import QueryStringParsing from "utils/query-string-parsing"; import DatasetError from "components/DatasetInformation/DatasetError"; import DatasetAttributes from "components/DatasetInformation/DatasetAttributes"; @@ -75,10 +75,10 @@ export const getAnalysisRouter = (Galaxy) => { "(/)pages(/)edit(/)": "show_pages_edit", "(/)pages(/)sharing(/)": "show_pages_sharing", "(/)pages(/)(:action_id)": "show_pages", - "(/)visualizations(/)": "show_plugins", + "(/)visualizations(/)": "show_visualizations", "(/)visualizations(/)edit(/)": "show_visualizations_edit", "(/)visualizations(/)sharing(/)": "show_visualizations_sharing", - "(/)visualizations/(:action_id)": "show_visualizations", + "(/)visualizations/(:action_id)": "show_visualizations_grid", "(/)workflows/import": "show_workflows_import", "(/)workflows/trs_import": "show_workflows_trs_import", "(/)workflows/trs_search": "show_workflows_trs_search", @@ -175,7 +175,13 @@ export const getAnalysisRouter = (Galaxy) => { this._display_vue_helper(ExternalIdentities); }, - show_visualizations: function (action_id) { + show_visualizations: function () { + this._display_vue_helper(VisualizationsList, { + datasetId: QueryStringParsing.get("id"), + }); + }, + + show_visualizations_grid: function (action_id) { const activeTab = action_id == "list_published" ? "shared" : "user"; this.page.display( new GridShared.View({ @@ -329,10 +335,6 @@ export const getAnalysisRouter = (Galaxy) => { }); }, - show_plugins: function () { - this._display_vue_helper(PluginList); - }, - show_workflows: function () { this._display_vue_helper(WorkflowList); }, From b75bdc3a1e84d41e184d4d34f23c36130a2345de Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 1 Dec 2021 10:02:42 -0500 Subject: [PATCH 131/401] Prepare display application component and insert into visualizations view --- .../Visualizations/DisplayApplications.vue | 15 +++++++++++++++ client/src/components/Visualizations/Index.vue | 2 ++ 2 files changed, 17 insertions(+) create mode 100644 client/src/components/Visualizations/DisplayApplications.vue diff --git a/client/src/components/Visualizations/DisplayApplications.vue b/client/src/components/Visualizations/DisplayApplications.vue new file mode 100644 index 000000000000..509781002b21 --- /dev/null +++ b/client/src/components/Visualizations/DisplayApplications.vue @@ -0,0 +1,15 @@ + + diff --git a/client/src/components/Visualizations/Index.vue b/client/src/components/Visualizations/Index.vue index ab0853db80be..25d5bd1c8a4d 100644 --- a/client/src/components/Visualizations/Index.vue +++ b/client/src/components/Visualizations/Index.vue @@ -1,5 +1,6 @@ @@ -9,6 +10,7 @@ import PluginList from "./PluginList"; export default { components: { PluginList, + DisplayApplications, }, props: { datasetId: { From a76d13109e8f2f444a35b8d6347723e5a500ec4c Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 1 Dec 2021 10:23:31 -0500 Subject: [PATCH 132/401] Use simple dataprovider to retrieve display application links --- .../Visualizations/DisplayApplications.vue | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/client/src/components/Visualizations/DisplayApplications.vue b/client/src/components/Visualizations/DisplayApplications.vue index 509781002b21..cedacba252ce 100644 --- a/client/src/components/Visualizations/DisplayApplications.vue +++ b/client/src/components/Visualizations/DisplayApplications.vue @@ -1,15 +1,28 @@ From fd3375adecc4a01c4a4002653ac9d49700cd45b6 Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 1 Dec 2021 20:01:39 -0500 Subject: [PATCH 133/401] Show available display application in alert? --- .../Visualizations/DisplayApplications.vue | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/client/src/components/Visualizations/DisplayApplications.vue b/client/src/components/Visualizations/DisplayApplications.vue index cedacba252ce..4c537ce5ee58 100644 --- a/client/src/components/Visualizations/DisplayApplications.vue +++ b/client/src/components/Visualizations/DisplayApplications.vue @@ -1,9 +1,23 @@ From bd6527513e3e3a3930dab0f4ab784a0f6851dcba Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 1 Dec 2021 22:54:13 -0500 Subject: [PATCH 134/401] Use datasetprovider instead of url provider --- .../Visualizations/DisplayApplications.vue | 17 ++++++----------- client/src/components/Visualizations/Index.vue | 2 +- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/client/src/components/Visualizations/DisplayApplications.vue b/client/src/components/Visualizations/DisplayApplications.vue index 4c537ce5ee58..11686da37261 100644 --- a/client/src/components/Visualizations/DisplayApplications.vue +++ b/client/src/components/Visualizations/DisplayApplications.vue @@ -1,8 +1,8 @@ diff --git a/client/src/components/Visualizations/Index.vue b/client/src/components/Visualizations/Index.vue index 25d5bd1c8a4d..b69e55d1b19b 100644 --- a/client/src/components/Visualizations/Index.vue +++ b/client/src/components/Visualizations/Index.vue @@ -15,7 +15,7 @@ export default { props: { datasetId: { type: String, - required: true, + default: null, }, }, }; From 1b332ce7d7de41bb5f1861643d17a37dbd1fcdd8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 2 Dec 2021 14:28:10 +0100 Subject: [PATCH 135/401] Drop stream_template_mako function, replace with fill_template_mako --- lib/galaxy/datatypes/data.py | 6 +++--- lib/galaxy/datatypes/sequence.py | 6 +++--- lib/galaxy/datatypes/tabular.py | 6 +++--- lib/galaxy/webapps/base/webapp.py | 18 ------------------ .../webapps/galaxy/controllers/dataset.py | 2 +- .../webapps/galaxy/controllers/history.py | 2 +- .../galaxy/controllers/visualization.py | 14 +++++++------- .../webapps/galaxy/controllers/workflow.py | 2 +- 8 files changed, 19 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 2aa931bb359b..a3a0364ddd80 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -474,9 +474,9 @@ def display_data(self, trans, data, preview=False, filename=None, to_ext=None, * return self._yield_user_file_content(trans, data, data.file_name, headers), headers else: headers["content-type"] = "text/html" - return trans.stream_template_mako("/dataset/large_file.mako", - truncated_data=open(data.file_name, 'rb').read(max_peek_size), - data=data), headers + return trans.fill_template_mako("/dataset/large_file.mako", + truncated_data=open(data.file_name, 'rb').read(max_peek_size), + data=data), headers def display_as_markdown(self, dataset_instance, markdown_format_helpers): """Prepare for embedding dataset into a basic Markdown document. diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 55d42954ffbb..b134d36f2942 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -721,9 +721,9 @@ def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None mime = "text/plain" self._clean_and_set_mime_type(trans, mime, headers) return fh.read(), headers - return trans.stream_template_mako("/dataset/large_file.mako", - truncated_data=fh.read(max_peek_size), - data=dataset), headers + return trans.fill_template_mako("/dataset/large_file.mako", + truncated_data=fh.read(max_peek_size), + data=dataset), headers else: return Sequence.display_data(self, trans, dataset, preview, filename, to_ext, **kwd) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 22d305581954..933a9511ef05 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -103,9 +103,9 @@ def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None return open(dataset.file_name, mode='rb'), headers else: headers["content-type"] = "text/html" - return trans.stream_template_mako("/dataset/large_file.mako", - truncated_data=open(dataset.file_name).read(max_peek_size), - data=dataset), headers + return trans.fill_template_mako("/dataset/large_file.mako", + truncated_data=open(dataset.file_name).read(max_peek_size), + data=dataset), headers else: column_names = 'null' if dataset.metadata.column_names: diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 7d9d54b71598..1b0e87aa6d0d 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -960,24 +960,6 @@ def fill_template_mako(self, filename, template_lookup=None, **kwargs): data.update(kwargs) return template.render(**data) - def stream_template_mako(self, filename, **kwargs): - template = self.webapp.mako_template_lookup.get_template(filename) - data = dict(caller=self, t=self, trans=self, h=helpers, util=util, request=self.request, response=self.response, app=self.app) - data.update(self.template_context) - data.update(kwargs) - - def render(environ, start_response): - response_write = start_response(self.response.wsgi_status(), self.response.wsgi_headeritems()) - - class StreamBuffer: - def write(self, d): - response_write(d.encode('utf-8')) - buffer = StreamBuffer() - context = mako.runtime.Context(buffer, **data) - template.render_context(context) - return [] - return render - def qualified_url_for_path(self, path): return url_for(path, qualified=True) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 9c2fffa07fc9..c8eb99fdc333 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -554,7 +554,7 @@ def get_item_content_async(self, trans, id): truncated, dataset_data = self.hda_manager.text_data(dataset, preview=True) # Get annotation. dataset.annotation = self.get_item_annotation_str(trans.sa_session, trans.user, dataset) - return trans.stream_template_mako("/dataset/item_content.mako", item=dataset, item_data=dataset_data, truncated=truncated) + return trans.fill_template_mako("/dataset/item_content.mako", item=dataset, item_data=dataset_data, truncated=truncated) @web.expose def annotate_async(self, trans, id, new_annotation=None, **kwargs): diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index bcd9de570409..c7f51f35fffd 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -635,7 +635,7 @@ def display_by_username_and_slug(self, trans, username, slug): view='dev-detailed', user=trans.user, trans=trans) history_dictionary['annotation'] = self.get_item_annotation_str(trans.sa_session, history.user, history) - return trans.stream_template_mako("history/display.mako", item=history, item_data=[], + return trans.fill_template_mako("history/display.mako", item=history, item_data=[], user_is_owner=user_is_owner, history_dict=history_dictionary, user_item_rating=user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings) diff --git a/lib/galaxy/webapps/galaxy/controllers/visualization.py b/lib/galaxy/webapps/galaxy/controllers/visualization.py index 3900d330e1c7..d5336ec2100b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/visualization.py +++ b/lib/galaxy/webapps/galaxy/controllers/visualization.py @@ -442,15 +442,15 @@ def display_by_username_and_slug(self, trans, username, slug): # TODO: simplest path from A to B but not optimal - will be difficult to do reg visualizations any other way # TODO: this will load the visualization twice (once above, once when the iframe src calls 'saved') encoded_visualization_id = trans.security.encode_id(visualization.id) - return trans.stream_template_mako('visualization/display_in_frame.mako', - item=visualization, encoded_visualization_id=encoded_visualization_id, - user_item_rating=user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings, - content_only=True) + return trans.fill_template_mako('visualization/display_in_frame.mako', + item=visualization, encoded_visualization_id=encoded_visualization_id, + user_item_rating=user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings, + content_only=True) visualization_config = self.get_visualization_config(trans, visualization) - return trans.stream_template_mako("visualization/display.mako", item=visualization, item_data=visualization_config, - user_item_rating=user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings, - content_only=True) + return trans.fill_template_mako("visualization/display.mako", item=visualization, item_data=visualization_config, + user_item_rating=user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings, + content_only=True) @web.expose @web.json diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index e181a0824568..3d6dcfd732dd 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -288,7 +288,7 @@ def get_item_content_async(self, trans, id): stored.annotation = self.get_item_annotation_str(trans.sa_session, stored.user, stored) for step in stored.latest_workflow.steps: step.annotation = self.get_item_annotation_str(trans.sa_session, stored.user, step) - return trans.stream_template_mako("/workflow/item_content.mako", item=stored, item_data=stored.latest_workflow.steps) + return trans.fill_template_mako("/workflow/item_content.mako", item=stored, item_data=stored.latest_workflow.steps) @web.expose @web.require_login("use Galaxy workflows") From 620a749913018ee5dd1e6b75bb7927cbac7bedf6 Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 2 Dec 2021 08:41:15 -0500 Subject: [PATCH 136/401] Remove unused parameter in display applications component --- client/src/components/Visualizations/DisplayApplications.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/Visualizations/DisplayApplications.vue b/client/src/components/Visualizations/DisplayApplications.vue index 11686da37261..131496b36f2f 100644 --- a/client/src/components/Visualizations/DisplayApplications.vue +++ b/client/src/components/Visualizations/DisplayApplications.vue @@ -1,6 +1,6 @@ From 0bdc93a7cce6fa0be1aa8ba1a80c0fdd372ce72b Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 12:34:08 +0000 Subject: [PATCH 164/401] Drop support for Python 3.6, add Python 3.10 --- .circleci/config.yml | 8 +- .github/workflows/dependencies.yaml | 2 +- .github/workflows/first_startup.yaml | 4 +- .github/workflows/lint.yaml | 2 +- .github/workflows/osx_startup.yaml | 4 +- .github/workflows/unit.yaml | 2 +- lib/galaxy/dependencies/dev-requirements.txt | 177 +++++++----------- .../dependencies/pinned-lint-requirements.txt | 2 +- .../dependencies/pinned-requirements.txt | 131 ++++--------- .../dependencies/update_lint_requirements.sh | 2 +- lib/galaxy_test/driver/driver_util.py | 3 +- packages/app/setup.py | 2 +- packages/auth/setup.py | 2 +- packages/containers/setup.py | 2 +- packages/data/setup.py | 2 +- packages/files/setup.py | 2 +- packages/job_execution/setup.py | 2 +- packages/job_metrics/setup.py | 2 +- packages/meta/setup.py | 2 +- packages/objectstore/setup.py | 2 +- packages/selenium/setup.py | 2 +- packages/test_api/setup.py | 2 +- packages/test_base/setup.py | 2 +- packages/test_driver/setup.py | 2 +- packages/test_selenium/setup.py | 2 +- packages/tool_util/setup.py | 2 +- packages/util/setup.py | 2 +- packages/web_framework/setup.py | 2 +- packages/web_stack/setup.py | 2 +- packages/webapps/setup.py | 2 +- pyproject.toml | 11 +- scripts/check_python.py | 4 +- scripts/common_startup.sh | 2 +- ...test_request_scoped_sqlalchemy_sessions.py | 6 +- test/unit/webapps/test_webapp_base.py | 7 +- 35 files changed, 148 insertions(+), 257 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index acddc1aa54bb..79e405da3e50 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,7 +27,7 @@ variables: jobs: get_code: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 <<: *set_workdir steps: # Replace standard code checkout with shallow clone to speed things up. @@ -85,7 +85,7 @@ jobs: - ~/repo validate_test_tools: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 <<: *set_workdir steps: - *restore_repo_cache @@ -95,7 +95,7 @@ jobs: - run: tox -e validate_test_tools test_galaxy_release: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 <<: *set_workdir steps: - *restore_repo_cache @@ -104,7 +104,7 @@ jobs: - run: tox -e test_galaxy_release test_galaxy_packages: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 <<: *set_workdir steps: - *restore_repo_cache diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml index 38399d42ca30..8cd1dd04051f 100644 --- a/.github/workflows/dependencies.yaml +++ b/.github/workflows/dependencies.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.6'] + python-version: ['3.7'] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/.github/workflows/first_startup.yaml b/.github/workflows/first_startup.yaml index 4d04e43ca112..dd1fba26d776 100644 --- a/.github/workflows/first_startup.yaml +++ b/.github/workflows/first_startup.yaml @@ -19,10 +19,10 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.9'] + python-version: ['3.7', '3.10'] webserver: ['uwsgi'] include: - - python-version: '3.9' + - python-version: '3.10' webserver: 'dev' defaults: run: diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 4c9d7cd1ad6a..9dfd554cf3cf 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.6', '3.9'] + python-version: ['3.7', '3.10'] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/.github/workflows/osx_startup.yaml b/.github/workflows/osx_startup.yaml index b7ffee4d64de..1100c557dbd1 100644 --- a/.github/workflows/osx_startup.yaml +++ b/.github/workflows/osx_startup.yaml @@ -29,8 +29,8 @@ jobs: id: pip-cache with: path: ~/Library/Caches/pip - # scripts/common_startup.sh creates a conda env for Galaxy containing Python 3.6 - key: pip-cache-3.6-${{ hashFiles('galaxy root/requirements.txt') }} + # scripts/common_startup.sh creates a conda env for Galaxy containing Python 3.7 + key: pip-cache-3.7-${{ hashFiles('galaxy root/requirements.txt') }} - name: Cache tox env uses: actions/cache@v2 with: diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml index 7538f328f245..7a237ef3c912 100644 --- a/.github/workflows/unit.yaml +++ b/.github/workflows/unit.yaml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.6', '3.9'] + python-version: ['3.7', '3.10'] steps: - uses: actions/checkout@v2 with: diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 75dcf95ee850..793e082e34a9 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -3,57 +3,48 @@ adal==1.2.7 aiofiles==0.8.0; python_version >= "3.6" and python_version < "4.0" alabaster==0.7.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -amqp==5.0.6; python_version >= "3.6" -appdirs==1.4.4; python_version >= "3.6" +amqp==5.0.6; python_version >= "3.7" +anyio==3.4.0; python_full_version >= "3.6.2" and python_version >= "3.6" +appdirs==1.4.4 argcomplete==1.12.3; python_version >= "3.6" and python_version < "4" asgiref==3.4.1; python_version >= "3.6" -async-exit-stack==1.0.1; python_version >= "3.6" and python_version < "3.7" -async-generator==1.10; python_version < "3.7" and python_version >= "3.6" -atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" +async-generator==1.10; python_version >= "3.7" and python_version < "4.0" +atomicwrites==1.4.0; python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.4.0" attmap==0.13.2 -attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -autopage==0.4.0; python_version >= "3.6" +attrs==21.2.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.5.0" and python_version >= "3.7" and python_version < "4.0" babel==2.9.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") bagit-profile==1.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" bagit==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" bcrypt==3.2.0; python_version >= "3.6" bdbag==1.6.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4") beaker==1.11.0 -billiard==3.6.4.0; python_version >= "3.6" +billiard==3.6.4.0; python_version >= "3.7" bioblend==0.16.0; python_version >= "3.6" bleach==4.1.0; python_version >= "3.6" boltons==21.0.0 -boto3==1.20.20; python_version >= "3.6" boto==2.49.0 -botocore==1.23.20; python_version >= "3.6" bx-python==0.8.12; python_version >= "3.6" cachecontrol==0.12.10; python_version >= "3.6" and python_version < "4" -cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6" -cachetools==4.2.4; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") -celery==5.1.2; python_version >= "3.6" -certifi==2021.10.8; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" -cffi==1.15.0; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") +cached-property==1.5.2; python_version < "3.8" and python_version >= "3.7" +celery==5.2.1; python_version >= "3.7" +certifi==2021.10.8; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7" +cffi==1.15.0 charset-normalizer==2.0.9; python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") circus==0.17.1 -click-didyoumean==0.0.3; python_version >= "3.6" -click-plugins==1.1.1; python_version >= "3.6" -click-repl==0.2.0; python_version >= "3.6" -click==7.1.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -cliff==3.10.0; python_version >= "3.6" +click-didyoumean==0.3.0; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.7" +click-plugins==1.1.1; python_version >= "3.7" +click-repl==0.2.0; python_version >= "3.7" +click==8.0.3; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.7" cloudauthz==0.6.0 -cloudbridge==2.2.0 -cmd2==2.3.3; python_version >= "3.6" -colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and python_version < "4.0" and platform_system == "Windows" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0" and platform_system == "Windows" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") +cloudbridge==3.0.0 +colorama==0.4.4; sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.6.2" and platform_system == "Windows" and python_full_version < "4.0.0" and (python_version >= "2.7" and python_full_version < "3.0.0" and platform_system == "Windows" or python_full_version >= "3.5.0" and platform_system == "Windows") and (python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.5.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and python_full_version >= "3.5.0") coloredlogs==15.0.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" -commonmark==0.9.1; python_version >= "3.6" and python_version < "4.0" -contextvars==2.4; python_version < "3.7" and python_version >= "3.6" +commonmark==0.9.1; python_full_version >= "3.6.2" and python_full_version < "4.0.0" coverage==6.2; python_version >= "3.6" -cryptography==36.0.0; python_version >= "3.6" +cryptography==36.0.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7" cwltest==2.2.20210901154959; python_version >= "3.6" and python_version < "4" cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4" -dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.6.1" -debtcollector==2.3.0; python_version >= "3.6" decorator==5.1.0; python_version >= "3.6" and python_version < "4" defusedxml==0.7.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4" deprecated==1.2.13; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" @@ -61,7 +52,6 @@ deprecation==2.1.0 dictobj==0.4 docopt==0.6.2 docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") -dogpile.cache==1.1.4; python_version >= "3.6" ecdsa==0.17.0; python_version >= "2.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" edam-ontology==1.25.2 fabric3==1.14.post1 @@ -72,37 +62,25 @@ fs==2.4.14 funcsigs==1.0.2 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") galaxy-sequence-utils==1.1.5 -google-api-core==2.2.2; python_version >= "3.6" -google-api-python-client==2.32.0; python_version >= "3.6" -google-auth-httplib2==0.1.0; python_version >= "3.6" -google-auth==2.3.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -googleapis-common-protos==1.53.0; python_version >= "3.6" greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0" gunicorn==20.1.0; python_version >= "3.5" gxformat2==0.15.0 -h11==0.12.0; python_version >= "3.6" -h5py==3.1.0; python_version >= "3.6" -httpcore==0.13.3; python_version >= "3.6" -httplib2==0.20.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -httpx==0.20.0; python_version >= "3.6" +h11==0.12.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.1" +h5py==3.6.0; python_version >= "3.7" +httpcore==0.14.3; python_version >= "3.6" +httpx==0.21.1; python_version >= "3.6" humanfriendly==10.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" -idna==3.3; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" +idna==3.3; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2" imagesize==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -immutables==0.16; python_version >= "3.6" and python_version < "3.7" -importlib-metadata==4.8.2; python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6") and (python_version == "3.6" or python_version == "3.7") -importlib-resources==5.4.0; python_version < "3.7" and python_version >= "3.6" -iniconfig==1.1.1; python_version >= "3.6" +importlib-metadata==4.8.2; python_version == "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6") +importlib-resources==5.4.0; python_version < "3.9" and python_version >= "3.7" +iniconfig==1.1.1; python_version >= "3.7" isa-rwval==0.10.10 -iso8601==0.1.16; python_version >= "3.6" -isodate==0.6.0; python_version >= "3.6" and python_version < "4" +isodate==0.6.0; python_version >= "3.7" and python_version < "4" jinja2==3.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -jsonpatch==1.32; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -jsonpointer==2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -jsonschema==4.0.0 +jsonschema==4.2.1; python_version >= "3.7" junit-xml==1.9; python_version >= "3.6" and python_version < "4" -keystoneauth1==4.4.0; python_version >= "3.6" -kombu==5.1.0; python_version >= "3.6" +kombu==5.2.2; python_version >= "3.7" lagom==1.7.0; python_version >= "3.6" and python_version < "4.0" lockfile==0.12.2; python_version >= "3.6" and python_version < "4" lxml==4.6.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") @@ -111,65 +89,50 @@ markdown-it-reporter==0.0.2 markdown==3.3.6; python_version >= "3.6" markupsafe==2.0.1; python_version >= "3.6" mercurial==6.0 -mirakuru==2.3.0; python_version >= "3.6" +mirakuru==2.4.1; python_version >= "3.7" mistune==0.8.4; python_version >= "3.6" and python_version < "4" mrcfile==1.3.0 msgpack==1.0.3; python_version >= "3.6" and python_version < "4" -munch==2.5.0; python_version >= "3.6" mypy-extensions==0.4.3; python_version >= "3.6" and python_version < "4" -netaddr==0.8.0; python_version >= "3.6" -netifaces==0.11.0; python_version >= "3.6" networkx==2.5; python_version >= "3.6" and python_version < "4" nodeenv==1.6.0 nose==1.3.7 nosehtml==0.4.6 -numpy==1.19.5; python_version >= "3.6" +numpy==1.21.1; python_version >= "3.7" oauthlib==3.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -openstacksdk==0.61.0; python_version >= "3.6" -os-client-config==2.1.0; python_version >= "3.6" -os-service-types==1.7.0; python_version >= "3.6" -osc-lib==2.4.2; python_version >= "3.6" -oslo.config==8.7.1; python_version >= "3.6" -oslo.context==3.4.0; python_version >= "3.6" -oslo.i18n==5.1.0; python_version >= "3.6" -oslo.log==4.6.1; python_version >= "3.6" -oslo.serialization==4.2.0; python_version >= "3.6" -oslo.utils==4.12.0; python_version >= "3.6" +outcome==1.1.0; python_version >= "3.7" and python_version < "4.0" oyaml==1.0 -packaging==21.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +packaging==21.3; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.7" paramiko==2.8.1 parsley==1.3 paste==3.5.0 pastedeploy==2.1.1 -pbr==5.8.0; python_version >= "3.6" -pluggy==1.0.0; python_version >= "3.6" -port-for==0.5.0; python_version >= "3.6" +pbr==5.8.0; python_version >= "2.6" +pluggy==1.0.0; python_version >= "3.7" +port-for==0.6.1; python_version >= "3.7" prettytable==2.4.0; python_version >= "3.6" -prompt-toolkit==3.0.19; python_full_version >= "3.6.1" and python_version >= "3.6" -protobuf==3.19.1; python_version >= "3.6" +prompt-toolkit==3.0.23; python_full_version >= "3.6.2" and python_version >= "3.7" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") pulsar-galaxy-lib==0.14.13 -py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.5.0" and python_version >= "3.6" and implementation_name == "pypy" -pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") -pycparser==2.21; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and python_full_version >= "3.4.0" -pycryptodome==3.11.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +py==1.11.0; python_version >= "3.7" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.5.0" and python_version >= "3.7" and implementation_name == "pypy" +pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" +pycparser==2.21 +pycryptodome==3.12.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") pydantic==1.8.2; python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.1" pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0" pyeventsystem==0.1.0 pyfaidx==0.6.3.1 pygithub==1.55; python_version >= "3.6" -pygments==2.10.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") -pyinotify==0.9.6; sys_platform != "win32" and sys_platform != "darwin" and sys_platform != "sunos5" and python_version >= "3.6" +pygments==2.10.0; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") pyjwt==2.3.0; python_version >= "3.6" pykwalify==1.8.0 pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -pyparsing==2.4.7; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") -pyperclip==1.8.2; python_version >= "3.6" +pyopenssl==21.0.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7" +pyparsing==3.0.6; python_version >= "3.6" pyreadline3==3.3; sys_platform == "win32" and python_version >= "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0") -pyreadline==2.1; sys_platform == "win32" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0") and python_version >= "3.6" -pyrsistent==0.18.0; python_version >= "3.6" +pyreadline==2.1; sys_platform == "win32" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0") +pyrsistent==0.18.0; python_version >= "3.7" pysam==0.18.0 pytest-asyncio==0.16.0; python_version >= "3.6" pytest-celery==0.0.0 @@ -178,46 +141,39 @@ pytest-html==3.1.1; python_version >= "3.6" pytest-json-report==1.4.1 pytest-metadata==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" pytest-mock==3.6.1; python_version >= "3.6" -pytest-postgresql==2.6.1; python_version >= "3.6" +pytest-postgresql==4.0.0; python_version >= "3.7" pytest-pythonpath==0.7.3 pytest-shard==0.1.2; python_version >= "3.6" pytest==6.2.5; python_version >= "3.6" python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.3.0" python-irodsclient==1.0.0 python-jose==3.3.0 -python-keystoneclient==4.3.0; python_version >= "3.6" python-multipart==0.0.5 -python-neutronclient==7.7.0; python_version >= "3.6" -python-novaclient==17.6.0; python_version >= "3.6" -python-swiftclient==3.13.0 python3-openid==3.2.0; python_version >= "3.0" -pytz==2021.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" +pytz==2021.3; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.7" pyuwsgi==2.0.20 pyyaml==6.0; python_version >= "3.6" pyzmq==22.3.0; python_version >= "3.6" -rdflib==5.0.0; python_version >= "3.6" and python_version < "4" +rdflib==6.0.2; python_version >= "3.7" and python_version < "4" recommonmark==0.7.1 refgenconf==0.12.2 repoze.lru==0.7 requests-oauthlib==1.3.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" requests-toolbelt==0.9.1; python_version >= "3.6" requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") -requestsexceptions==1.4.0; python_version >= "3.6" responses==0.16.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") rfc3986==1.5.0; python_version >= "3.6" -rich==10.11.0; python_version >= "3.6" and python_version < "4.0" +rich==10.15.2; python_full_version >= "3.6.2" and python_full_version < "4.0.0" routes==2.5.1 -rsa==4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +rsa==4.8; python_version >= "3.6" and python_version < "4" ruamel.yaml.clib==0.2.6; platform_python_implementation == "CPython" and python_version < "3.10" and python_version >= "3.6" ruamel.yaml==0.17.17; python_version >= "3.6" and python_version < "4" -s3transfer==0.5.0; python_version >= "3.6" schema-salad==8.2.20211116214159; python_version >= "3.6" and python_version < "4" -selenium==3.141.0 +selenium==4.1.0; python_version >= "3.7" and python_version < "4.0" setuptools-scm==5.0.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" shellescape==3.8.1; python_version >= "3.6" and python_version < "4" -simplejson==3.17.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" -sniffio==1.2.0; python_version >= "3.6" +six==1.16.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") +sniffio==1.2.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2" snowballstemmer==2.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" social-auth-core==4.0.3 sortedcontainers==2.4.0 @@ -237,34 +193,35 @@ sqlparse==0.4.2; python_version >= "3.5" starlette-context==0.3.3; python_version >= "3.7" starlette==0.14.2; python_version >= "3.6" statsd==3.3.0 -stevedore==3.5.0; python_version >= "3.6" svgwrite==1.4.1; python_version >= "3.6" tempita==0.5.2 tenacity==8.0.1; python_version >= "3.6" testfixtures==6.18.3 -tifffile==2020.9.3; python_version >= "3.6" -tinydb==4.5.2; python_version >= "3.5" and python_version < "4.0" -toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" +tifffile==2021.11.2; python_version >= "3.7" +tinydb==4.5.0; python_version >= "3.5" and python_version < "4.0" +toml==0.10.2; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7" tomli==1.2.2; python_version >= "3.6" tornado==6.1; python_version >= "3.5" tqdm==4.62.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" +trio-websocket==0.9.2; python_version >= "3.7" and python_version < "4.0" +trio==0.19.0; python_version >= "3.7" and python_version < "4.0" tuspy==0.2.5 tuswsgi==0.5.4 -twill==3.0 -typing-extensions==3.10.0.2 +twill==3.0.1 +typing-extensions==4.0.1; python_version >= "3.6" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" ubiquerg==0.6.2 -uritemplate==4.1.1; python_version >= "3.6" -urllib3==1.26.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" +urllib3==1.26.7; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7" uvicorn==0.15.0 -vine==5.0.0; python_version >= "3.6" +vine==5.0.0; python_version >= "3.7" watchdog==2.1.6; python_version >= "3.6" -wcwidth==0.2.5; python_full_version >= "3.6.1" and python_version >= "3.6" +wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7" webencodings==0.5.1; python_version >= "3.6" webob==1.8.7; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") whoosh==2.7.4 wrapt==1.13.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +wsproto==1.0.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.1" xmlrunner==1.7.7 yacman==0.8.4 -zipp==3.6.0; python_version < "3.7" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6") and (python_version == "3.6" or python_version == "3.7") +zipp==3.6.0; python_version == "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6") zipstream-new==1.1.8 diff --git a/lib/galaxy/dependencies/pinned-lint-requirements.txt b/lib/galaxy/dependencies/pinned-lint-requirements.txt index 4c71f59b2864..aa2c3c99a6cb 100644 --- a/lib/galaxy/dependencies/pinned-lint-requirements.txt +++ b/lib/galaxy/dependencies/pinned-lint-requirements.txt @@ -19,7 +19,7 @@ types-docutils==0.17.1 types-enum34==1.1.1 types-ipaddress==1.0.1 types-Markdown==3.3.9 -types-paramiko==2.8.2 +types-paramiko==2.8.3 types-pkg-resources==0.1.3 types-python-dateutil==2.8.3 types-PyYAML==6.0.1 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 206ae4c641dd..ed6524a8bb71 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -2,61 +2,49 @@ adal==1.2.7 aiofiles==0.8.0; python_version >= "3.6" and python_version < "4.0" -amqp==5.0.6; python_version >= "3.6" -appdirs==1.4.4; python_version >= "3.6" +amqp==5.0.6; python_version >= "3.7" +appdirs==1.4.4 argcomplete==1.12.3; python_version >= "3.6" and python_version < "4" asgiref==3.4.1; python_version >= "3.6" -async-exit-stack==1.0.1; python_version >= "3.6" and python_version < "3.7" -async-generator==1.10; python_version >= "3.6" and python_version < "3.7" attmap==0.13.2 -attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -autopage==0.4.0; python_version >= "3.6" +attrs==21.2.0; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7" babel==2.9.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") bagit-profile==1.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" bagit==1.8.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" bcrypt==3.2.0; python_version >= "3.6" bdbag==1.6.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0" and python_version < "4") beaker==1.11.0 -billiard==3.6.4.0; python_version >= "3.6" +billiard==3.6.4.0; python_version >= "3.7" bioblend==0.16.0; python_version >= "3.6" bleach==4.1.0; python_version >= "3.6" boltons==21.0.0 -boto3==1.20.20; python_version >= "3.6" boto==2.49.0 -botocore==1.23.20; python_version >= "3.6" bx-python==0.8.12; python_version >= "3.6" cachecontrol==0.12.10; python_version >= "3.6" and python_version < "4" -cached-property==1.5.2; python_version < "3.8" and python_version >= "3.6" -cachetools==4.2.4; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") -celery==5.1.2; python_version >= "3.6" +cached-property==1.5.2; python_version < "3.8" and python_version >= "3.7" +celery==5.2.1; python_version >= "3.7" certifi==2021.10.8; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" cffi==1.15.0; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0") charset-normalizer==2.0.9; python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") circus==0.17.1 -click-didyoumean==0.0.3; python_version >= "3.6" -click-plugins==1.1.1; python_version >= "3.6" -click-repl==0.2.0; python_version >= "3.6" -click==7.1.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -cliff==3.10.0; python_version >= "3.6" +click-didyoumean==0.3.0; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.7" +click-plugins==1.1.1; python_version >= "3.7" +click-repl==0.2.0; python_version >= "3.7" +click==8.0.3; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.7" cloudauthz==0.6.0 -cloudbridge==2.2.0 -cmd2==2.3.3; python_version >= "3.6" -colorama==0.4.4; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and platform_system == "Windows" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.5.0" and platform_system == "Windows" +cloudbridge==3.0.0 +colorama==0.4.4; platform_system == "Windows" and python_version >= "3.7" and python_full_version >= "3.6.2" and python_full_version < "4.0.0" and (python_version >= "2.7" and python_full_version < "3.0.0" and platform_system == "Windows" or python_full_version >= "3.5.0" and platform_system == "Windows") coloredlogs==15.0.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" -commonmark==0.9.1; python_version >= "3.6" and python_version < "4.0" -contextvars==2.4; python_version >= "3.6" and python_version < "3.7" -cryptography==36.0.0; python_version >= "3.6" +commonmark==0.9.1; python_full_version >= "3.6.2" and python_full_version < "4.0.0" +cryptography==36.0.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4" -dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.6.1" -debtcollector==2.3.0; python_version >= "3.6" decorator==5.1.0; python_version >= "3.6" and python_version < "4" defusedxml==0.7.1; python_version >= "3.0" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.0" deprecation==2.1.0 dictobj==0.4 docopt==0.6.2 docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") -dogpile.cache==1.1.4; python_version >= "3.6" ecdsa==0.17.0; python_version >= "2.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" edam-ontology==1.25.2 fabric3==1.14.post1 @@ -66,30 +54,18 @@ fs==2.4.14 funcsigs==1.0.2 future==0.18.2; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") galaxy-sequence-utils==1.1.5 -google-api-core==2.2.2; python_version >= "3.6" -google-api-python-client==2.32.0; python_version >= "3.6" -google-auth-httplib2==0.1.0; python_version >= "3.6" -google-auth==2.3.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -googleapis-common-protos==1.53.0; python_version >= "3.6" greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") or python_version >= "3" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.0") and python_full_version >= "3.5.0" gxformat2==0.15.0 h11==0.12.0; python_version >= "3.6" -h5py==3.1.0; python_version >= "3.6" -httplib2==0.20.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +h5py==3.6.0; python_version >= "3.7" humanfriendly==10.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0" idna==3.3; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" -immutables==0.16; python_version >= "3.6" and python_version < "3.7" -importlib-metadata==4.8.2; python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6") and (python_version == "3.6" or python_version == "3.7") -importlib-resources==5.4.0; python_version < "3.7" and python_version >= "3.6" +importlib-metadata==4.8.2; python_version == "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6") +importlib-resources==5.4.0; python_version < "3.9" and python_version >= "3.7" isa-rwval==0.10.10 -iso8601==0.1.16; python_version >= "3.6" -isodate==0.6.0; python_version >= "3.6" and python_version < "4" -jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -jsonpatch==1.32; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -jsonpointer==2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" -jsonschema==4.0.0 -keystoneauth1==4.4.0; python_version >= "3.6" -kombu==5.1.0; python_version >= "3.6" +isodate==0.6.0; python_version >= "3.7" and python_version < "4" +jsonschema==4.2.1; python_version >= "3.7" +kombu==5.2.2; python_version >= "3.7" lagom==1.7.0; python_version >= "3.6" and python_version < "4.0" lockfile==0.12.2; python_version >= "3.6" and python_version < "4" lxml==4.6.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") @@ -100,89 +76,63 @@ mercurial==6.0 mistune==0.8.4; python_version >= "3.6" and python_version < "4" mrcfile==1.3.0 msgpack==1.0.3; python_version >= "3.6" and python_version < "4" -munch==2.5.0; python_version >= "3.6" mypy-extensions==0.4.3; python_version >= "3.6" and python_version < "4" -netaddr==0.8.0; python_version >= "3.6" -netifaces==0.11.0; python_version >= "3.6" networkx==2.5; python_version >= "3.6" and python_version < "4" nodeenv==1.6.0 nose==1.3.7 -numpy==1.19.5; python_version >= "3.6" +numpy==1.21.1; python_version >= "3.7" oauthlib==3.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" -openstacksdk==0.61.0; python_version >= "3.6" -os-client-config==2.1.0; python_version >= "3.6" -os-service-types==1.7.0; python_version >= "3.6" -osc-lib==2.4.2; python_version >= "3.6" -oslo.config==8.7.1; python_version >= "3.6" -oslo.context==3.4.0; python_version >= "3.6" -oslo.i18n==5.1.0; python_version >= "3.6" -oslo.log==4.6.1; python_version >= "3.6" -oslo.serialization==4.2.0; python_version >= "3.6" -oslo.utils==4.12.0; python_version >= "3.6" oyaml==1.0 packaging==21.3; python_version >= "3.6" paramiko==2.8.1 parsley==1.3 paste==3.5.0 pastedeploy==2.1.1 -pbr==5.8.0; python_version >= "3.6" -prettytable==2.4.0; python_version >= "3.6" -prompt-toolkit==3.0.19; python_full_version >= "3.6.1" and python_version >= "3.6" -protobuf==3.19.1; python_version >= "3.6" +pbr==5.8.0; python_version >= "2.6" +prompt-toolkit==3.0.23; python_full_version >= "3.6.2" and python_version >= "3.7" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") pulsar-galaxy-lib==0.14.13 py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.5.0" -pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" -pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" pycparser==2.21; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0") or implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0") and python_full_version >= "3.4.0" -pycryptodome==3.11.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +pycryptodome==3.12.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") pydantic==1.8.2; python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.6.1" pydot==1.4.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.4.0" pyeventsystem==0.1.0 pyfaidx==0.6.3.1 -pygments==2.10.0; python_version >= "3.6" and python_version < "4.0" -pyinotify==0.9.6; sys_platform != "win32" and sys_platform != "darwin" and sys_platform != "sunos5" and python_version >= "3.6" +pygments==2.10.0; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.5" pyjwt==2.3.0; python_version >= "3.6" pykwalify==1.8.0 pynacl==1.4.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" -pyparsing==2.4.7; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") -pyperclip==1.8.2; python_version >= "3.6" +pyparsing==3.0.6; python_version >= "3.6" pyreadline3==3.3; sys_platform == "win32" and python_version >= "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0") -pyreadline==2.1; sys_platform == "win32" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0") and python_version >= "3.6" -pyrsistent==0.18.0; python_version >= "3.6" +pyreadline==2.1; sys_platform == "win32" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.5.0") +pyrsistent==0.18.0; python_version >= "3.7" pysam==0.18.0 python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_version >= "3.6" and python_version < "4" and python_full_version >= "3.3.0" python-jose==3.3.0 -python-keystoneclient==4.3.0; python_version >= "3.6" python-multipart==0.0.5 -python-neutronclient==7.7.0; python_version >= "3.6" -python-novaclient==17.6.0; python_version >= "3.6" -python-swiftclient==3.13.0 python3-openid==3.2.0; python_version >= "3.0" -pytz==2021.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" +pytz==2021.3; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.7" pyuwsgi==2.0.20 pyyaml==6.0; python_version >= "3.6" pyzmq==22.3.0; python_version >= "3.6" -rdflib==5.0.0; python_version >= "3.6" and python_version < "4" +rdflib==6.0.2; python_version >= "3.7" and python_version < "4" refgenconf==0.12.2 repoze.lru==0.7 requests-oauthlib==1.3.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" requests-toolbelt==0.9.1; python_version >= "3.6" requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") -requestsexceptions==1.4.0; python_version >= "3.6" -rfc3986==1.5.0; python_version >= "3.6" -rich==10.11.0; python_version >= "3.6" and python_version < "4.0" +rich==10.15.2; python_full_version >= "3.6.2" and python_full_version < "4.0.0" routes==2.5.1 -rsa==4.8; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +rsa==4.8; python_version >= "3.6" and python_version < "4" ruamel.yaml.clib==0.2.6; platform_python_implementation == "CPython" and python_version < "3.10" and python_version >= "3.6" ruamel.yaml==0.17.17; python_version >= "3.6" and python_version < "4" -s3transfer==0.5.0; python_version >= "3.6" schema-salad==8.2.20211116214159; python_version >= "3.6" and python_version < "4" setuptools-scm==5.0.2; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" shellescape==3.8.1; python_version >= "3.6" and python_version < "4" -simplejson==3.17.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" -six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4" +six==1.16.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.4.0" and python_version >= "3.7" and python_version < "4" social-auth-core==4.0.3 sortedcontainers==2.4.0 sqlalchemy-migrate==0.13.0 @@ -191,26 +141,23 @@ sqlitedict==1.7.0 sqlparse==0.4.2; python_version >= "3.5" starlette-context==0.3.3; python_version >= "3.7" starlette==0.14.2; python_version >= "3.6" -stevedore==3.5.0; python_version >= "3.6" svgwrite==1.4.1; python_version >= "3.6" tempita==0.5.2 tenacity==8.0.1; python_version >= "3.6" -tifffile==2020.9.3; python_version >= "3.6" +tifffile==2021.11.2; python_version >= "3.7" tornado==6.1; python_version >= "3.5" tqdm==4.62.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" tuswsgi==0.5.4 -typing-extensions==3.10.0.2 +typing-extensions==4.0.1; python_version >= "3.6" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" ubiquerg==0.6.2 -uritemplate==4.1.1; python_version >= "3.6" urllib3==1.26.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" uvicorn==0.15.0 -vine==5.0.0; python_version >= "3.6" -wcwidth==0.2.5; python_full_version >= "3.6.1" and python_version >= "3.6" +vine==5.0.0; python_version >= "3.7" +wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7" webencodings==0.5.1; python_version >= "3.6" webob==1.8.7; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0") whoosh==2.7.4 -wrapt==1.13.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" yacman==0.8.4 -zipp==3.6.0; python_version < "3.7" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6") and (python_version == "3.6" or python_version == "3.7") +zipp==3.6.0; python_version == "3.7" and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6") zipstream-new==1.1.8 diff --git a/lib/galaxy/dependencies/update_lint_requirements.sh b/lib/galaxy/dependencies/update_lint_requirements.sh index 5c2e9b798e5b..d477d941ab96 100755 --- a/lib/galaxy/dependencies/update_lint_requirements.sh +++ b/lib/galaxy/dependencies/update_lint_requirements.sh @@ -4,7 +4,7 @@ set -e THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)" LINT_VENV=$(mktemp -d "${TMPDIR:-/tmp}/lint_venv.XXXXXXXXXX") -python3 -m venv "${LINT_VENV}" +python3.7 -m venv "${LINT_VENV}" . "${LINT_VENV}/bin/activate" pip install --upgrade pip setuptools pip install -r "${THIS_DIRECTORY}/lint-requirements.txt" diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index 461ecb39c1ad..3c3ad913ae99 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -78,8 +78,7 @@ """) DEFAULT_LOCALES = "en" -CAN_BUILD_ASGI_APP = sys.version_info[:2] >= (3, 7) -USE_UVICORN = asbool(os.environ.get('GALAXY_TEST_USE_UVICORN', CAN_BUILD_ASGI_APP)) +USE_UVICORN = asbool(os.environ.get('GALAXY_TEST_USE_UVICORN', True)) log = logging.getLogger("test_driver") diff --git a/packages/app/setup.py b/packages/app/setup.py index 78efdd6ebed8..93f9af2344c9 100644 --- a/packages/app/setup.py +++ b/packages/app/setup.py @@ -158,10 +158,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/auth/setup.py b/packages/auth/setup.py index e9c127f99c4f..2eafa480d0a0 100644 --- a/packages/auth/setup.py +++ b/packages/auth/setup.py @@ -88,10 +88,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/containers/setup.py b/packages/containers/setup.py index 754f18332f2a..399aae00a479 100644 --- a/packages/containers/setup.py +++ b/packages/containers/setup.py @@ -89,10 +89,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/data/setup.py b/packages/data/setup.py index 263571e25d53..6eb9d0f66c7e 100644 --- a/packages/data/setup.py +++ b/packages/data/setup.py @@ -109,10 +109,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/files/setup.py b/packages/files/setup.py index e3a0e90e275a..bb48f5fec001 100644 --- a/packages/files/setup.py +++ b/packages/files/setup.py @@ -89,10 +89,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/job_execution/setup.py b/packages/job_execution/setup.py index dbb6ec5e8487..313826fa0948 100644 --- a/packages/job_execution/setup.py +++ b/packages/job_execution/setup.py @@ -91,10 +91,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/job_metrics/setup.py b/packages/job_metrics/setup.py index 34ed91f99818..6aaa5c246405 100644 --- a/packages/job_metrics/setup.py +++ b/packages/job_metrics/setup.py @@ -88,10 +88,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/meta/setup.py b/packages/meta/setup.py index cdaa4c024464..69886c2feb53 100644 --- a/packages/meta/setup.py +++ b/packages/meta/setup.py @@ -116,10 +116,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/objectstore/setup.py b/packages/objectstore/setup.py index 715191e6641b..d7487978fbd6 100644 --- a/packages/objectstore/setup.py +++ b/packages/objectstore/setup.py @@ -87,10 +87,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/selenium/setup.py b/packages/selenium/setup.py index 01aac2114f5e..e0ce6ffb4558 100644 --- a/packages/selenium/setup.py +++ b/packages/selenium/setup.py @@ -90,10 +90,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/test_api/setup.py b/packages/test_api/setup.py index a40273ce1176..05e75f2eba36 100644 --- a/packages/test_api/setup.py +++ b/packages/test_api/setup.py @@ -87,10 +87,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/test_base/setup.py b/packages/test_base/setup.py index cd5148d83a4e..f1e501f2a0ad 100644 --- a/packages/test_base/setup.py +++ b/packages/test_base/setup.py @@ -87,10 +87,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/test_driver/setup.py b/packages/test_driver/setup.py index 0c3b9aec1341..097c348490c8 100644 --- a/packages/test_driver/setup.py +++ b/packages/test_driver/setup.py @@ -87,10 +87,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/test_selenium/setup.py b/packages/test_selenium/setup.py index ff97e7dc707a..ed5946425bcf 100644 --- a/packages/test_selenium/setup.py +++ b/packages/test_selenium/setup.py @@ -88,10 +88,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/tool_util/setup.py b/packages/tool_util/setup.py index c3eced55b8e0..851770cb8480 100644 --- a/packages/tool_util/setup.py +++ b/packages/tool_util/setup.py @@ -126,10 +126,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/util/setup.py b/packages/util/setup.py index 40dc9978d315..e5cbb7ff406e 100644 --- a/packages/util/setup.py +++ b/packages/util/setup.py @@ -96,10 +96,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/web_framework/setup.py b/packages/web_framework/setup.py index 87c19e9d2e81..ebcecb09ff6a 100644 --- a/packages/web_framework/setup.py +++ b/packages/web_framework/setup.py @@ -92,10 +92,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/web_stack/setup.py b/packages/web_stack/setup.py index a69afcb4be18..e3020e83fb62 100644 --- a/packages/web_stack/setup.py +++ b/packages/web_stack/setup.py @@ -87,10 +87,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/packages/webapps/setup.py b/packages/webapps/setup.py index bc8ab520f0ca..2cba3c312936 100644 --- a/packages/webapps/setup.py +++ b/packages/webapps/setup.py @@ -115,10 +115,10 @@ def get_var(var_name): 'Topic :: Software Development :: Testing', 'Natural Language :: English', "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], test_suite=TEST_DIR, tests_require=test_requirements diff --git a/pyproject.toml b/pyproject.toml index 257ea66e861d..137663cf33ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,6 @@ url = "https://wheels.galaxyproject.org/simple" [tool.poetry.dependencies] aiofiles = "*" -async-generator = {version = "*", python = "~3.6"} -async-exit-stack = {version = "*", python = "~3.6"} Babel = "*" bdbag = "*" Beaker = "1.11.0" @@ -29,7 +27,6 @@ celery = "*" Cheetah3 = "!=3.2.6.post2" # yanked release, https://github.com/python-poetry/poetry/issues/2453 cloudauthz = "==0.6.0" cloudbridge = "*" -contextvars = {version = "*", python = "~3.6"} circus = "*" cwltool = "==3.1.20211107152837" dictobj = "*" @@ -42,7 +39,7 @@ fs = "*" future = "*" galaxy_sequence_utils = "*" gxformat2 = "*" -h5py = "==3.1.0" # Last version compatible with python 3.6. Also adding this constraint speeds up poetry resolution immensely +h5py = "*" isa-rwval = "*" kombu = "*" lagom = "*" @@ -63,7 +60,7 @@ pycryptodome = "*" pykwalify = "*" pyparsing = "*" pysam = "*" -python = "^3.6.1" # required by fastapi +python = ">=3.7,<4" python-multipart = "*" # required to support form parsing in FastAPI/Starlette pyuwsgi = "*" PyYAML = "*" @@ -77,9 +74,9 @@ sqlalchemy-migrate = "*" sqlitedict = "*" sqlparse = "*" starlette = "*" -starlette-context = {version = "*", python = ">=3.7"} +starlette-context = "*" svgwrite = "*" -tifffile = "<=2020.9.3" # Last version compatible with python 3.6 +tifffile = "*" tuswsgi = "*" typing-extensions = "*" uvicorn = "*" diff --git a/scripts/check_python.py b/scripts/check_python.py index efb2d3076dce..085f9019122d 100644 --- a/scripts/check_python.py +++ b/scripts/check_python.py @@ -7,14 +7,14 @@ def check_python(): - if sys.version_info[:2] >= (3, 6): + if sys.version_info[:2] >= (3, 7): # supported return else: version_string = '.'.join(str(_) for _ in sys.version_info[:3]) msg = """\ ERROR: Your Python version is: %s -Galaxy is currently supported on Python >=3.6 . +Galaxy is currently supported on Python >=3.7 . To run Galaxy, please install a supported Python version. If a supported version is already installed but is not your default, https://docs.galaxyproject.org/en/latest/admin/python.html contains instructions diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 2e1f0da05ee2..72c22ee5978c 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -116,7 +116,7 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then echo "Creating Conda environment for Galaxy: $GALAXY_CONDA_ENV" echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_CONDA_ENV to an" echo "existing environment before starting Galaxy." - $CONDA_EXE create --yes --override-channels --channel conda-forge --channel defaults --name "$GALAXY_CONDA_ENV" 'python=3.6' 'pip>=19.0' 'virtualenv>=16' + $CONDA_EXE create --yes --override-channels --channel conda-forge --channel defaults --name "$GALAXY_CONDA_ENV" 'python=3.7' 'pip>=19.0' 'virtualenv>=16' unset __CONDA_INFO fi conda_activate diff --git a/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py b/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py index b1632d192fc7..6c619059a742 100644 --- a/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py +++ b/test/unit/webapps/test_request_scoped_sqlalchemy_sessions.py @@ -1,7 +1,6 @@ import asyncio import concurrent.futures import functools -import sys import threading import time import uuid @@ -149,10 +148,7 @@ async def test_request_scoped_sa_session_concurrent_requests_async(): @pytest.mark.asyncio async def test_request_scoped_sa_session_concurrent_requests_and_background_thread(): add_request_id_middleware(app) - if sys.version_info > (3, 6): - loop = asyncio.get_running_loop() - else: - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() target = functools.partial(assert_scoped_session_is_thread_local, GX_APP) with concurrent.futures.ThreadPoolExecutor() as pool: background_pool = loop.run_in_executor(pool, target) diff --git a/test/unit/webapps/test_webapp_base.py b/test/unit/webapps/test_webapp_base.py index f799157e329e..05f93634be9c 100644 --- a/test/unit/webapps/test_webapp_base.py +++ b/test/unit/webapps/test_webapp_base.py @@ -3,7 +3,6 @@ """ import logging import re -import sys import unittest import galaxy.config @@ -60,11 +59,7 @@ def test_parse_allowed_origin_hostnames(self): hostnames = config._parse_allowed_origin_hostnames({ "allowed_origin_hostnames": r"/host\d{2}/,geocities.com,miskatonic.edu" }) - if sys.version_info >= (3, 7): - Pattern = re.Pattern - else: - Pattern = re._pattern_type - self.assertTrue(isinstance(hostnames[0], Pattern)) + self.assertTrue(isinstance(hostnames[0], re.Pattern)) self.assertTrue(isinstance(hostnames[1], str)) self.assertTrue(isinstance(hostnames[2], str)) From 98ecc39b1bd87f9fdf73ad7c4563465f2836d9e7 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 13:26:23 +0000 Subject: [PATCH 165/401] Remove PhantomJS support, dropped in Selenium 4.0 --- doc/source/dev/run_tests_help.txt | 2 +- lib/galaxy/selenium/driver_factory.py | 5 ++--- lib/galaxy_test/selenium/framework.py | 2 +- run_tests.sh | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/source/dev/run_tests_help.txt b/doc/source/dev/run_tests_help.txt index f33850097ceb..176f140e89ba 100644 --- a/doc/source/dev/run_tests_help.txt +++ b/doc/source/dev/run_tests_help.txt @@ -94,7 +94,7 @@ https://sites.google.com/a/chromium.org/chromedriver/. By default Galaxy will check the PATH for these and pick whichever it finds. This can be overridden by setting GALAXY_TEST_SELENIUM_BROWSER to either FIREFOX, CHROME, or something -more esoteric (including OPERA and PHANTOMJS). +more esoteric (including OPERA). If PyVirtualDisplay is installed Galaxy will attempt to run this browser in a headless mode. This can be disabled by setting diff --git a/lib/galaxy/selenium/driver_factory.py b/lib/galaxy/selenium/driver_factory.py index 6a9bdea74de2..6063c1c3b1ea 100644 --- a/lib/galaxy/selenium/driver_factory.py +++ b/lib/galaxy/selenium/driver_factory.py @@ -25,7 +25,7 @@ DEFAULT_SELENIUM_REMOTE_HOST = "127.0.0.1" DEFAULT_WINDOW_WIDTH = 1280 DEFAULT_WINDOW_HEIGHT = 1000 -VALID_LOCAL_BROWSERS = ["CHROME", "FIREFOX", "OPERA", "PHANTOMJS"] +VALID_LOCAL_BROWSERS = ["CHROME", "FIREFOX", "OPERA"] class ConfiguredDriver: @@ -88,7 +88,6 @@ def get_local_driver(browser=DEFAULT_BROWSER, headless=False) -> WebDriver: "CHROME": webdriver.Chrome, "FIREFOX": webdriver.Firefox, "OPERA": webdriver.Opera, - "PHANTOMJS": webdriver.PhantomJS, } driver_class = driver_to_class[browser] if browser == 'CHROME': @@ -120,7 +119,7 @@ def get_remote_driver( # docker run -d -p 4444:4444 -v /dev/shm:/dev/shm selenium/standalone-chrome:3.0.1-aluminum if browser == "auto": browser = "CHROME" - assert browser in ["CHROME", "EDGE", "ANDROID", "FIREFOX", "INTERNETEXPLORER", "IPAD", "IPHONE", "OPERA", "PHANTOMJS", "SAFARI"] + assert browser in ["CHROME", "EDGE", "ANDROID", "FIREFOX", "INTERNETEXPLORER", "IPAD", "IPHONE", "OPERA", "SAFARI"] desired_capabilities = getattr(DesiredCapabilities, browser) desired_capabilities["loggingPrefs"] = LOGGING_PREFS executor = f'http://{host}:{port}/wd/hub' diff --git a/lib/galaxy_test/selenium/framework.py b/lib/galaxy_test/selenium/framework.py index edbbc95000eb..caaf169af152 100644 --- a/lib/galaxy_test/selenium/framework.py +++ b/lib/galaxy_test/selenium/framework.py @@ -51,7 +51,7 @@ TIMEOUT_MULTIPLIER = float(os.environ.get("GALAXY_TEST_TIMEOUT_MULTIPLIER", DEFAULT_TIMEOUT_MULTIPLIER)) GALAXY_TEST_ERRORS_DIRECTORY = os.environ.get("GALAXY_TEST_ERRORS_DIRECTORY", DEFAULT_TEST_ERRORS_DIRECTORY) GALAXY_TEST_SCREENSHOTS_DIRECTORY = os.environ.get("GALAXY_TEST_SCREENSHOTS_DIRECTORY", None) -# Test browser can be ["CHROME", "FIREFOX", "OPERA", "PHANTOMJS"] +# Test browser can be ["CHROME", "FIREFOX", "OPERA"] GALAXY_TEST_SELENIUM_BROWSER = os.environ.get("GALAXY_TEST_SELENIUM_BROWSER", driver_factory.DEFAULT_SELENIUM_BROWSER) GALAXY_TEST_SELENIUM_REMOTE = os.environ.get("GALAXY_TEST_SELENIUM_REMOTE", driver_factory.DEFAULT_SELENIUM_REMOTE) GALAXY_TEST_SELENIUM_REMOTE_PORT = os.environ.get("GALAXY_TEST_SELENIUM_REMOTE_PORT", driver_factory.DEFAULT_SELENIUM_REMOTE_PORT) diff --git a/run_tests.sh b/run_tests.sh index fb801eb6874d..2aea9c687189 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -104,7 +104,7 @@ https://sites.google.com/a/chromium.org/chromedriver/. By default Galaxy will check the PATH for these and pick whichever it finds. This can be overridden by setting GALAXY_TEST_SELENIUM_BROWSER to either FIREFOX, CHROME, or something -more esoteric (including OPERA and PHANTOMJS). +more esoteric (including OPERA). If PyVirtualDisplay is installed Galaxy will attempt to run this browser in a headless mode. This can be disabled by setting From 6b45a474afa683c490a0089ac4c4185e3756bcd4 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 15:13:22 +0000 Subject: [PATCH 166/401] Replace dropped Selenium WebDriver method --- lib/galaxy_test/selenium/test_history_multi_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_test/selenium/test_history_multi_view.py b/lib/galaxy_test/selenium/test_history_multi_view.py index afecf5992f90..15fffe358e91 100644 --- a/lib/galaxy_test/selenium/test_history_multi_view.py +++ b/lib/galaxy_test/selenium/test_history_multi_view.py @@ -86,7 +86,7 @@ def test_purge_history(self): # click on purge button with corresponding history_id self.components.multi_history_view.history_dropdown_menu.purge(history_id=history_id).wait_for_and_click() - self.driver.switch_to_alert().accept() + self.driver.switch_to.alert.accept() self.sleep_for(self.wait_types.UX_RENDER) self.assert_history(history_id, should_exist=False) From 4654662c1498a0661d78c43925d9a333eb559234 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 15:25:35 +0000 Subject: [PATCH 167/401] Fix use of wrong waiting method --- lib/galaxy_test/selenium/test_history_panel_collections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_test/selenium/test_history_panel_collections.py b/lib/galaxy_test/selenium/test_history_panel_collections.py index 84cb197a2465..8377496ba147 100644 --- a/lib/galaxy_test/selenium/test_history_panel_collections.py +++ b/lib/galaxy_test/selenium/test_history_panel_collections.py @@ -162,7 +162,7 @@ def _back_to_history(self): else: back = self.components.history_panel.collection_view.back back.wait_for_and_click() - self.wait(WAIT_TYPES.UX_RENDER) + self.sleep_for(WAIT_TYPES.UX_RENDER) @selenium_test def test_rename_collection(self): From 90c44f56b55bbd8014986076be2df5d2eb79fdf3 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 17:52:04 +0000 Subject: [PATCH 168/401] Make test more precise and remove incorrect comment --- lib/galaxy_test/api/test_workflows.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 381feb747406..78b730ab911d 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -1347,9 +1347,8 @@ def test_run_subworkflow_simple(self) -> None: assert sum(1 for step in steps if step['subworkflow_invocation_id'] is None) == 3 subworkflow_invocation_id = [step['subworkflow_invocation_id'] for step in steps if step['subworkflow_invocation_id']][0] subworkflow_invocation = self.workflow_populator.get_invocation(subworkflow_invocation_id) - # inner_input should be step 0, random_lines should be step 1, but step order gets lost on python < 3.6 - assert [step for step in subworkflow_invocation['steps'] if step['workflow_step_label'] == 'inner_input'] - assert [step for step in subworkflow_invocation['steps'] if step['workflow_step_label'] == 'random_lines'] + assert [step for step in subworkflow_invocation['steps'] if step['order_index'] == 0][0]['workflow_step_label'] == 'inner_input' + assert [step for step in subworkflow_invocation['steps'] if step['order_index'] == 1][0]['workflow_step_label'] == 'random_lines' bco = self.workflow_populator.get_biocompute_object(invocation_id) self.workflow_populator.validate_biocompute_object(bco) From cf1d036b1c17b878fb81370a9a1a6b75acf163bf Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 18:35:24 +0000 Subject: [PATCH 169/401] Remove entry already in `lib/galaxy/config/sample/tool_conf.xml.sample` Cannot be imported from this directory any way. --- test/functional/tools/samples_tool_conf.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index bcf4de8167ae..ef95e737e53a 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -1,7 +1,6 @@ -
From 6c0036fb574734a95aedf5b5c8d182ff5e87fdd4 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 19:50:55 +0000 Subject: [PATCH 170/401] Fix wrong test tool file path --- test/functional/tools/samples_tool_conf.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index ef95e737e53a..4d5cf02519d5 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -136,7 +136,7 @@ - + From 9f89cd4d56b2c130365a86cd73ae104c362582ea Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 20:04:47 +0000 Subject: [PATCH 171/401] Fix errors in loading functional test tools --- test/functional/tools/identifier_multiple_in_conditional.xml | 2 ++ test/functional/tools/sample_datatypes_conf.xml | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/test/functional/tools/identifier_multiple_in_conditional.xml b/test/functional/tools/identifier_multiple_in_conditional.xml index 81d12fd99fa4..c5fb19b6742e 100644 --- a/test/functional/tools/identifier_multiple_in_conditional.xml +++ b/test/functional/tools/identifier_multiple_in_conditional.xml @@ -13,8 +13,10 @@ + + diff --git a/test/functional/tools/sample_datatypes_conf.xml b/test/functional/tools/sample_datatypes_conf.xml index 90c296cad2a6..d115b669ec28 100644 --- a/test/functional/tools/sample_datatypes_conf.xml +++ b/test/functional/tools/sample_datatypes_conf.xml @@ -18,6 +18,9 @@ + + + @@ -35,6 +38,8 @@ + + From 55b690ac991f0610933d1b7042eecbe646ce7fc3 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 23:45:31 +0000 Subject: [PATCH 172/401] Remove missing tool from functional test tool conf --- test/functional/tools/samples_tool_conf.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 4d5cf02519d5..057036b8307e 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -52,7 +52,6 @@ - From 9a0978102eecc53952efe546cd9a3f299508cea0 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Wed, 8 Dec 2021 20:30:56 +1100 Subject: [PATCH 173/401] add tools to IMPLICITLY_REQUIRED_TOOL_FILES in tools/__init__.py --- lib/galaxy/tools/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 12ff0f0b65c6..665b64a764c8 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -205,6 +205,21 @@ # minimum example: # "foobar": {"required": REQUIRE_FULL_DIRECTORY} # if no version is specified, all versions without explicit RequiredFiles will be selected + "circos": {"required": REQUIRE_FULL_DIRECTORY}, + "cp_image_math": {"required": {"includes": [{"path": "cp_common_functions.py", "path_type": "literal"}]}}, + "enumerate_charges": {"required": {"includes": [{"path": "site_substructures.smarts", "path_type": "literal"}]}}, + "fasta_compute_length": {"required": {"includes": [{"path": "utils/*", "path_type": "glob"}]}}, + "fasta_concatenate0": {"required": {"includes": [{"path": "utils/*", "path_type": "glob"}]}}, + "filter_tabular": {"required": {"includes": [{"path": "*.py", "path_type": "glob"}]}}, + "flanking_features_1": {"required": {"includes": [{"path": "utils/*", "path_type": "glob"}]}}, + "gops_intersect_1": {"required": {"includes": [{"path": "utils/*", "path_type": "glob"}]}}, + "gops_subtract_1": {"required": {"includes": [{"path": "utils/*", "path_type": "glob"}]}}, + "maxquant": {"required": {"includes": [{"path": "mqparam.py", "path_type": "literal"}]}}, + "maxquant_mqpar": {"required": {"includes": [{"path": "mqparam.py", "path_type": "literal"}]}}, + "query_tabular": {"required": {"includes": [{"path": "*.py", "path_type": "glob"}]}}, + "shasta": {"required": {"includes": [{"path": "configs/*", "path_type": "glob"}]}}, + "sqlite_to_tabular": {"required": {"includes": [{"path": "*.py", "path_type": "glob"}]}}, + "sucos_max_score": {"required": {"includes": [{"path": "utils.py", "path_type": "literal"}]}}, } From 8c7477cab639a6f8e99c52c989841e4dc494ab0a Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Dec 2021 19:16:43 +0000 Subject: [PATCH 174/401] Test Galaxy release script xref. https://github.com/galaxyproject/galaxy/pull/11641#discussion_r764049621 --- .circleci/config.yml | 9 -------- .github/workflows/test_galaxy_release.yaml | 25 ++++++++++++++++++++++ scripts/release.sh | 7 +++--- test/release.sh | 17 +++++++++------ 4 files changed, 39 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/test_galaxy_release.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index 79e405da3e50..aeb23948fd6d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,15 +93,6 @@ jobs: - run: sudo apt-get install -y libxml2-utils - *install_tox - run: tox -e validate_test_tools - test_galaxy_release: - docker: - - image: circleci/python:3.7 - <<: *set_workdir - steps: - - *restore_repo_cache - - *install_tox - - *install_deb_reqs - - run: tox -e test_galaxy_release test_galaxy_packages: docker: - image: circleci/python:3.7 diff --git a/.github/workflows/test_galaxy_release.yaml b/.github/workflows/test_galaxy_release.yaml new file mode 100644 index 000000000000..84dc6f4f1a80 --- /dev/null +++ b/.github/workflows/test_galaxy_release.yaml @@ -0,0 +1,25 @@ +name: Test Galaxy release script +on: + push: + paths: + - lib/galaxy/version.py + - scripts/release.sh + - test/release.sh + pull_request: + paths: + - lib/galaxy/version.py + - scripts/release.sh + - test/release.sh +concurrency: + group: test-galaxy-release-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Run tests + run: ./test/release.sh diff --git a/scripts/release.sh b/scripts/release.sh index 81150e4ba291..d42a358f01a5 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -19,6 +19,7 @@ shopt -s extglob # Only use this for dev/testing/CI to ignore forward merge conflicts, skip confirmation and package builds, etc. : ${TEST_MODE:=false} +$TEST_MODE && MERGE_STRATEGY_OPTIONS='-X ours' || MERGE_STRATEGY_OPTIONS= VERIFY_PACKAGES=(wheel packaging) @@ -258,7 +259,6 @@ function _test_forward_merge() { local curr_local_branch="__release_merge_test_${curr}" local next_local_branch="__release_merge_test_${next}" local recurse=true - local strategy= log "Testing forward merge of ${curr} to ${next}" if ! branch_exists "$curr_local_branch"; then branch_exists "$curr_branch" || { log_error "No existing branch for merge test: ${curr_branch}"; exit 1; } @@ -270,8 +270,7 @@ function _test_forward_merge() { fi git_checkout_temp "$next_local_branch" "$next_branch" # Test the merge even if ignoring just to test the code path - $TEST_MODE && strategy='-X ours' - log_exec git merge $strategy -m 'test merge; please ignore' "$curr_local_branch" || { + log_exec git merge $MERGE_STRATEGY_OPTIONS -m 'test merge; please ignore' "$curr_local_branch" || { log_error "Merging unmodified ${curr} to ${next} failed, resolve upstream first!"; exit 1; } if $recurse; then _test_forward_merge "$next" @@ -295,7 +294,7 @@ function perform_stable_merge() { local stable_int=$(echo "$stable" | tr -d .) if [ "$curr_int" -ge "$stable_int" ]; then log "Release '${RELEASE_CURR}' >= stable branch release '${stable}', merging 'release_${RELEASE_CURR}' to '${STABLE_BRANCH}'" - log_exec git merge -m "Merge branch 'release_${RELEASE_CURR}' into '${STABLE_BRANCH}'" "__release_${RELEASE_CURR}" + log_exec git merge $MERGE_STRATEGY_OPTIONS -m "Merge branch 'release_${RELEASE_CURR}' into '${STABLE_BRANCH}'" "__release_${RELEASE_CURR}" PUSH_BRANCHES+=("__stable:${STABLE_BRANCH}") else log "Release '${RELEASE_CURR}' < stable branch release '${stable}', skipping merge to '${STABLE_BRANCH}'" diff --git a/test/release.sh b/test/release.sh index fcb6b1f8f999..4277c580bf15 100755 --- a/test/release.sh +++ b/test/release.sh @@ -9,7 +9,6 @@ export TEST_MODE=true REPO_ROOT= FORK_ROOT=$(mktemp -d -t galaxy_release_test_XXXXXXXX) -DEV_BRANCH="$(git branch --show-current)" TEST_RELEASE_PREV= TEST_RELEASE_PREV_MINOR= @@ -20,8 +19,6 @@ TEST_RELEASE_NEXT_NEXT='99.09' : ${VENV:=${FORK_ROOT}/venv} export VENV -[ -n "$DEV_BRANCH" ] || { echo 'ERROR: Cannot determine current branch'; exit 1; } - function trap_handler() { [ -z "$FORK_ROOT" ] || rm -rf "$FORK_ROOT" @@ -84,8 +81,16 @@ function make_forks() { git clone --no-checkout "${REPO_ROOT}" "${FORK_ROOT}/work" ( cd_fork work - log "Checking out ref '${DEV_BRANCH}' as 'dev'" - log_exec git checkout --no-track -b dev "$DEV_BRANCH" + # A username and email address are needed to commit + if [ -z "$(git config --get user.name)" ]; then + log_exec git config user.name "Test User" + fi + if [ -z "$(git config --get user.email)" ]; then + log_exec git config user.email "test@example.org" + fi + CURRENT_COMMIT=$(git rev-parse HEAD) + log "Checking out ref '${CURRENT_COMMIT}' as 'dev'" + log_exec git checkout --no-track -b dev "$CURRENT_COMMIT" ) # Create bare origin and upstream repos @@ -109,7 +114,7 @@ function make_forks() { # Fetch release branches to upstream repo ( cd_fork upstream - log_exec git fetch --no-tags "${REPO_ROOT}" refs/remotes/upstream/${STABLE_BRANCH}:${STABLE_BRANCH} + log_exec git fetch --no-tags "${REPO_ROOT}" refs/remotes/origin/${STABLE_BRANCH}:${STABLE_BRANCH} ) # Set current (previous) stable release from stable branch From 918302c23ecfac80407bd902824d0df25eae990d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 8 Dec 2021 11:32:46 -0500 Subject: [PATCH 175/401] Fix running pytest test/integration/test_remote_files.py::RemoteFilesIntegrationTestCase directly. --- test/functional/tools/export_remote.py | 1 + test/functional/tools/export_remote.xml | 1 + test/functional/tools/samples_tool_conf.xml | 1 + 3 files changed, 3 insertions(+) create mode 120000 test/functional/tools/export_remote.py create mode 120000 test/functional/tools/export_remote.xml diff --git a/test/functional/tools/export_remote.py b/test/functional/tools/export_remote.py new file mode 120000 index 000000000000..bc34a879a525 --- /dev/null +++ b/test/functional/tools/export_remote.py @@ -0,0 +1 @@ +../../../lib/galaxy/tools/bundled/data_export/export_remote.py \ No newline at end of file diff --git a/test/functional/tools/export_remote.xml b/test/functional/tools/export_remote.xml new file mode 120000 index 000000000000..50cb357d717e --- /dev/null +++ b/test/functional/tools/export_remote.xml @@ -0,0 +1 @@ +../../../lib/galaxy/tools/bundled/data_export/export_remote.xml \ No newline at end of file diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 057036b8307e..c5266e354fb9 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -1,6 +1,7 @@ +
From 724d983d467e9a010dd5f57a394eeb1a97efa514 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 6 Dec 2021 20:58:03 +0100 Subject: [PATCH 176/401] Add "diff-format" Makefile target Re-format code changed since the last Git commit using "black" via darker --- Makefile | 4 ++++ lib/galaxy/dependencies/dev-requirements.txt | 9 +++++++-- pyproject.toml | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d807be0af3eb..93e632030d21 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ VENV?=.venv IN_VENV=if [ -f "$(VENV)/bin/activate" ]; then . "$(VENV)/bin/activate"; fi; RELEASE_CURR:=22.01 RELEASE_UPSTREAM:=upstream +TARGET_BRANCH=$(RELEASE_UPSTREAM)/dev CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/config/config_manage.py PROJECT_URL?=https://github.com/galaxyproject/galaxy DOCS_DIR=doc @@ -37,6 +38,9 @@ docs-develop: ## Fast doc generation and more warnings (for development) setup-venv: if [ ! -f $(VENV)/bin/activate ]; then bash scripts/common_startup.sh --dev-wheels; fi +diff-format: + $(IN_VENV) darker -r $(TARGET_BRANCH) + list-dependency-updates: setup-venv $(IN_VENV) pip list --outdated --format=columns diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 793e082e34a9..cab383dc66d3 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -20,6 +20,7 @@ bdbag==1.6.3; (python_version >= "2.7" and python_full_version < "3.0.0") or (py beaker==1.11.0 billiard==3.6.4.0; python_version >= "3.7" bioblend==0.16.0; python_version >= "3.6" +black==21.12b0; python_full_version >= "3.6.2" and python_version >= "3.6" bleach==4.1.0; python_version >= "3.6" boltons==21.0.0 boto==2.49.0 @@ -45,6 +46,7 @@ coverage==6.2; python_version >= "3.6" cryptography==36.0.0; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7" cwltest==2.2.20210901154959; python_version >= "3.6" and python_version < "4" cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4" +darker==1.3.2; python_version >= "3.6" decorator==5.1.0; python_version >= "3.6" and python_version < "4" defusedxml==0.7.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4" deprecated==1.2.13; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" @@ -93,7 +95,7 @@ mirakuru==2.4.1; python_version >= "3.7" mistune==0.8.4; python_version >= "3.6" and python_version < "4" mrcfile==1.3.0 msgpack==1.0.3; python_version >= "3.6" and python_version < "4" -mypy-extensions==0.4.3; python_version >= "3.6" and python_version < "4" +mypy-extensions==0.4.3; python_version >= "3.6" and python_version < "4" and python_full_version >= "3.6.2" networkx==2.5; python_version >= "3.6" and python_version < "4" nodeenv==1.6.0 nose==1.3.7 @@ -107,7 +109,9 @@ paramiko==2.8.1 parsley==1.3 paste==3.5.0 pastedeploy==2.1.1 +pathspec==0.9.0; python_full_version >= "3.6.2" and python_version >= "3.6" pbr==5.8.0; python_version >= "2.6" +platformdirs==2.4.0; python_full_version >= "3.6.2" and python_version >= "3.6" pluggy==1.0.0; python_version >= "3.7" port-for==0.6.1; python_version >= "3.7" prettytable==2.4.0; python_version >= "3.6" @@ -200,7 +204,7 @@ testfixtures==6.18.3 tifffile==2021.11.2; python_version >= "3.7" tinydb==4.5.0; python_version >= "3.5" and python_version < "4.0" toml==0.10.2; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7" -tomli==1.2.2; python_version >= "3.6" +tomli==1.2.2; python_version >= "3.6" and python_full_version >= "3.6.2" tornado==6.1; python_version >= "3.5" tqdm==4.62.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" trio-websocket==0.9.2; python_version >= "3.7" and python_version < "4.0" @@ -208,6 +212,7 @@ trio==0.19.0; python_version >= "3.7" and python_version < "4.0" tuspy==0.2.5 tuswsgi==0.5.4 twill==3.0.1 +typed-ast==1.5.1; python_version < "3.8" and implementation_name == "cpython" and python_full_version >= "3.6.2" and python_version >= "3.6" typing-extensions==4.0.1; python_version >= "3.6" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" ubiquerg==0.6.2 diff --git a/pyproject.toml b/pyproject.toml index 137663cf33ce..d389aab2e31c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ zipstream-new = "*" [tool.poetry.dev-dependencies] cwltest = "2.2.20210901154959" +darker = "*" fluent-logger = "*" gunicorn = "*" httpx = "*" From f7388c48fe57237ab2e8694daeea66f99115a501 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 9 Dec 2021 19:23:14 +0000 Subject: [PATCH 177/401] Fix running `test/release.sh` on the dev branch Fix the following error: ``` # >>>> make_forks() Cloning into '/tmp/galaxy_release_test_cQZHDb9B/work'... done. + cd /tmp/galaxy_release_test_cQZHDb9B/work + git config user.name 'Test User' + git config user.email test@example.org # Checking out ref '9bddbada0d3f7835ecafd12220a76dcfb6e3daca' as 'dev' + git checkout --no-track -b dev 9bddbada0d3f7835ecafd12220a76dcfb6e3daca fatal: A branch named 'dev' already exists. ``` Also: - Add `log_exec` to a few git commands --- test/release.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/release.sh b/test/release.sh index 4277c580bf15..1597fe2aa0e4 100755 --- a/test/release.sh +++ b/test/release.sh @@ -78,7 +78,7 @@ function make_forks() { [ -n "$REPO_ROOT" ] || REPO_ROOT=$(get_repo_root) # Use a "work" clone to prevent modifications to current clone - git clone --no-checkout "${REPO_ROOT}" "${FORK_ROOT}/work" + log_exec git clone --no-checkout "${REPO_ROOT}" "${FORK_ROOT}/work" ( cd_fork work # A username and email address are needed to commit @@ -88,9 +88,11 @@ function make_forks() { if [ -z "$(git config --get user.email)" ]; then log_exec git config user.email "test@example.org" fi - CURRENT_COMMIT=$(git rev-parse HEAD) - log "Checking out ref '${CURRENT_COMMIT}' as 'dev'" - log_exec git checkout --no-track -b dev "$CURRENT_COMMIT" + if [ "$(git rev-parse --abbrev-ref HEAD)" != dev ]; then + CURRENT_COMMIT=$(git rev-parse HEAD) + log "Checking out ref '${CURRENT_COMMIT}' as 'dev'" + log_exec git checkout --no-track -b dev "$CURRENT_COMMIT" + fi ) # Create bare origin and upstream repos @@ -105,7 +107,7 @@ function make_forks() { # Set remotes on work repo ( cd_fork work - git remote remove origin + log_exec git remote remove origin for repo in origin upstream; do log_exec git remote add "$repo" "file://${FORK_ROOT}/${repo}" done @@ -120,7 +122,7 @@ function make_forks() { # Set current (previous) stable release from stable branch ( cd_fork work - git fetch --no-tags upstream + log_exec git fetch --no-tags upstream ) TEST_RELEASE_PREV=$(get_stable_version MAJOR) TEST_RELEASE_PREV_MINOR=$(get_stable_version MINOR) From e7386d52300e1bcb062b193875959fb325289241 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 9 Dec 2021 20:06:23 +0000 Subject: [PATCH 178/401] Fix test_singularity_search unit test in a way that it doesn't break when a new version of the container is released. --- test/unit/tool_util/mulled/test_mulled_search.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/unit/tool_util/mulled/test_mulled_search.py b/test/unit/tool_util/mulled/test_mulled_search.py index 88e4efcee556..9524a5e954db 100644 --- a/test/unit/tool_util/mulled/test_mulled_search.py +++ b/test/unit/tool_util/mulled/test_mulled_search.py @@ -57,7 +57,10 @@ def test_get_package_hash(): @external_dependency_management def test_singularity_search(): sing1 = singularity_search('mulled-v2-0560a8046fc82aa4338588eca29ff18edab2c5aa') + sing1_versions = {result['version'] for result in sing1} + assert { + 'c17ce694dd57ab0ac1a2b86bb214e65fedef760e-0', + 'f471ba33d45697daad10614c5bd25a67693f67f1-0', + 'fc33176431a4b9ef3213640937e641d731db04f1-0'}.issubset(sing1_versions) sing2 = singularity_search('mulled-v2-19fa9431f5863b2be81ff13791f1b00160ed0852') - assert sing1[0]['version'] in ['c17ce694dd57ab0ac1a2b86bb214e65fedef760e-0', 'fc33176431a4b9ef3213640937e641d731db04f1-0'] - assert sing1[1]['version'] in ['c17ce694dd57ab0ac1a2b86bb214e65fedef760e-0', 'fc33176431a4b9ef3213640937e641d731db04f1-0'] assert sing2 == [] From 6671e6f323298e41072fe2e7b252cecf4907f517 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 9 Dec 2021 17:51:50 +0000 Subject: [PATCH 179/401] Don't silently ignore failure to process legacy tool-provided metadata --- lib/galaxy/tool_util/provided_metadata.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tool_util/provided_metadata.py b/lib/galaxy/tool_util/provided_metadata.py index 4dede134a2b5..e279b95c959a 100644 --- a/lib/galaxy/tool_util/provided_metadata.py +++ b/lib/galaxy/tool_util/provided_metadata.py @@ -109,10 +109,9 @@ def __init__(self, meta_file, job_wrapper=None): try: line = stringify_dictionary_keys(json.loads(line)) assert 'type' in line - except Exception: - log.exception(f"({getattr(job_wrapper, 'job_id', None)}) Got JSON data from tool, but data is improperly formatted or no \"type\" key in data") - log.debug(f'Offending data was: {line}') - continue + except Exception as e: + message = f"Got JSON data from tool, but line is improperly formatted or no \"type\" key in: [{line}]" + raise ValueError(message) from e # Set the dataset id if it's a dataset entry and isn't set. # This isn't insecure. We loop the job's output datasets in # the finish method, so if a tool writes out metadata for a From 984a96362914cf51fc4e9a6636a7b88204428d2f Mon Sep 17 00:00:00 2001 From: ic4f Date: Sat, 11 Dec 2021 03:11:21 +0000 Subject: [PATCH 180/401] Update Python dependencies --- lib/galaxy/dependencies/dev-requirements.txt | 8 ++++---- lib/galaxy/dependencies/pinned-lint-requirements.txt | 2 +- lib/galaxy/dependencies/pinned-requirements.txt | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index cab383dc66d3..769efd4aa9aa 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -115,7 +115,7 @@ platformdirs==2.4.0; python_full_version >= "3.6.2" and python_version >= "3.6" pluggy==1.0.0; python_version >= "3.7" port-for==0.6.1; python_version >= "3.7" prettytable==2.4.0; python_version >= "3.6" -prompt-toolkit==3.0.23; python_full_version >= "3.6.2" and python_version >= "3.7" +prompt-toolkit==3.0.24; python_full_version >= "3.6.2" and python_version >= "3.7" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") pulsar-galaxy-lib==0.14.13 @@ -191,7 +191,7 @@ sphinxcontrib-jsmath==1.0.1; python_version >= "3.6" and python_full_version < " sphinxcontrib-qthelp==1.0.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" sphinxcontrib-serializinghtml==1.1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" sqlalchemy-migrate==0.13.0 -sqlalchemy==1.4.27; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") +sqlalchemy==1.4.28; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") sqlitedict==1.7.0 sqlparse==0.4.2; python_version >= "3.5" starlette-context==0.3.3; python_version >= "3.7" @@ -204,7 +204,7 @@ testfixtures==6.18.3 tifffile==2021.11.2; python_version >= "3.7" tinydb==4.5.0; python_version >= "3.5" and python_version < "4.0" toml==0.10.2; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.7" -tomli==1.2.2; python_version >= "3.6" and python_full_version >= "3.6.2" +tomli==1.2.2; python_full_version >= "3.6.2" and python_version >= "3.6" tornado==6.1; python_version >= "3.5" tqdm==4.62.3; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" trio-websocket==0.9.2; python_version >= "3.7" and python_version < "4.0" @@ -217,7 +217,7 @@ typing-extensions==4.0.1; python_version >= "3.6" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" ubiquerg==0.6.2 urllib3==1.26.7; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7" -uvicorn==0.15.0 +uvicorn==0.16.0 vine==5.0.0; python_version >= "3.7" watchdog==2.1.6; python_version >= "3.6" wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7" diff --git a/lib/galaxy/dependencies/pinned-lint-requirements.txt b/lib/galaxy/dependencies/pinned-lint-requirements.txt index aa2c3c99a6cb..6fd5258713f5 100644 --- a/lib/galaxy/dependencies/pinned-lint-requirements.txt +++ b/lib/galaxy/dependencies/pinned-lint-requirements.txt @@ -19,7 +19,7 @@ types-docutils==0.17.1 types-enum34==1.1.1 types-ipaddress==1.0.1 types-Markdown==3.3.9 -types-paramiko==2.8.3 +types-paramiko==2.8.4 types-pkg-resources==0.1.3 types-python-dateutil==2.8.3 types-PyYAML==6.0.1 diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index ed6524a8bb71..8251a210f84d 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -89,7 +89,7 @@ parsley==1.3 paste==3.5.0 pastedeploy==2.1.1 pbr==5.8.0; python_version >= "2.6" -prompt-toolkit==3.0.23; python_full_version >= "3.6.2" and python_version >= "3.7" +prompt-toolkit==3.0.24; python_full_version >= "3.6.2" and python_version >= "3.7" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") pulsar-galaxy-lib==0.14.13 @@ -136,7 +136,7 @@ six==1.16.0; python_version >= "3.7" and python_full_version < "3.0.0" and pytho social-auth-core==4.0.3 sortedcontainers==2.4.0 sqlalchemy-migrate==0.13.0 -sqlalchemy==1.4.27; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") +sqlalchemy==1.4.28; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") sqlitedict==1.7.0 sqlparse==0.4.2; python_version >= "3.5" starlette-context==0.3.3; python_version >= "3.7" @@ -152,7 +152,7 @@ typing-extensions==4.0.1; python_version >= "3.6" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" ubiquerg==0.6.2 urllib3==1.26.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" -uvicorn==0.15.0 +uvicorn==0.16.0 vine==5.0.0; python_version >= "3.7" wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7" webencodings==0.5.1; python_version >= "3.6" From 80bd6e8f3e31a4239e9031a3c4776a47603ea1f6 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Sun, 12 Dec 2021 09:18:19 +0530 Subject: [PATCH 181/401] Rename vault classes and some docs --- lib/galaxy/security/vault.py | 69 ++++++++++++++++++++++++++----- lib/galaxy_test/api/test_vault.py | 2 +- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/security/vault.py b/lib/galaxy/security/vault.py index 61cdf826e441..cf2e3a928de6 100644 --- a/lib/galaxy/security/vault.py +++ b/lib/galaxy/security/vault.py @@ -2,7 +2,7 @@ import logging import os import re -from typing import Optional +from typing import List, Optional import yaml from cryptography.fernet import Fernet, MultiFernet @@ -27,7 +27,7 @@ VAULT_KEY_INVALID_REGEX = re.compile(r"\s\/|\/\s|\/\/") -class UnknownVaultTypeException(Exception): +class InvalidVaultConfigException(Exception): pass @@ -36,30 +36,61 @@ class InvalidVaultKeyException(Exception): class Vault(abc.ABC): + """ + A simple abstraction for reading/writing from external vaults. + """ @abc.abstractmethod def read_secret(self, key: str) -> Optional[str]: + """ + Reads a secret from the vault. + + :param key: The key to read. Typically a hierarchical path such as `/galaxy/user/1/preferences/editor` + :return: The string value stored at the key, such as 'ace_editor'. + """ pass @abc.abstractmethod def write_secret(self, key: str, value: str) -> None: + """ + Write a secret to the vault. + + :param key: The key to write to. Typically a hierarchical path such as `/galaxy/user/1/preferences/editor` + :param value: The value to write, such as 'vscode' + :return: + """ + pass + + @abc.abstractmethod + def list_secrets(self, key: str) -> List[str]: + """ + Lists secrets at a given path. + + :param key: The key prefix to list. e.g. `/galaxy/user/1/preferences`. A trailing slash is optional. + :return: The list of subkeys at path. e.g. + ['/galaxy/user/1/preferences/editor`, '/galaxy/user/1/preferences/storage`] + Note that only immediate subkeys are returned. + """ pass class NullVault(Vault): def read_secret(self, key: str) -> Optional[str]: - raise UnknownVaultTypeException("No vault configured. Make sure the vault_config_file setting is defined in galaxy.yml") + raise InvalidVaultConfigException("No vault configured. Make sure the vault_config_file setting is defined in galaxy.yml") def write_secret(self, key: str, value: str) -> None: - raise UnknownVaultTypeException("No vault configured. Make sure the vault_config_file setting is defined in galaxy.yml") + raise InvalidVaultConfigException("No vault configured. Make sure the vault_config_file setting is defined in galaxy.yml") + + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() class HashicorpVault(Vault): def __init__(self, config): if not hvac: - raise UnknownVaultTypeException("Hashicorp vault library 'hvac' is not available. Make sure hvac is installed.") + raise InvalidVaultConfigException("Hashicorp vault library 'hvac' is not available. Make sure hvac is installed.") self.vault_address = config.get('vault_address') self.vault_token = config.get('vault_token') self.client = hvac.Client(url=self.vault_address, token=self.vault_token) @@ -74,6 +105,9 @@ def read_secret(self, key: str) -> Optional[str]: def write_secret(self, key: str, value: str) -> None: self.client.secrets.kv.v2.create_or_update_secret(path=key, secret={'value': value}) + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + class DatabaseVault(Vault): @@ -114,12 +148,15 @@ def write_secret(self, key: str, value: str) -> None: token = f.encrypt(value.encode('utf-8')) self._update_or_create(key=key, value=token.decode('utf-8')) + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + class CustosVault(Vault): def __init__(self, config): if not custos_sdk_available: - raise UnknownVaultTypeException("Custos sdk library 'custos-sdk' is not available. Make sure the custos-sdk is installed.") + raise InvalidVaultConfigException("Custos sdk library 'custos-sdk' is not available. Make sure the custos-sdk is installed.") custos_settings = CustosServerClientSettings(custos_host=config.get('custos_host'), custos_port=config.get('custos_port'), custos_client_id=config.get('custos_client_id'), @@ -136,6 +173,9 @@ def read_secret(self, key: str) -> Optional[str]: def write_secret(self, key: str, value: str) -> None: self.client.set_kv_credential(key=key, value=value) + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + class UserVaultWrapper(Vault): @@ -149,8 +189,11 @@ def read_secret(self, key: str) -> Optional[str]: def write_secret(self, key: str, value: str) -> None: return self.vault.write_secret(f"user/{self.user.id}/{key}", value) + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() -class VaultKeyValidationDecorator(Vault): + +class VaultKeyValidationWrapper(Vault): """ A decorator to standardize and validate vault key paths """ @@ -181,8 +224,11 @@ def write_secret(self, key: str, value: str) -> None: key = self.normalize_key(key) return self.vault.write_secret(key, value) + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + -class VaultKeyPrefixDecorator(Vault): +class VaultKeyPrefixWrapper(Vault): """ Adds a prefix to all vault keys, such as the galaxy instance id """ @@ -197,6 +243,9 @@ def read_secret(self, key: str) -> Optional[str]: def write_secret(self, key: str, value: str) -> None: return self.vault.write_secret(f"/{self.prefix}/{key}", value) + def list_secrets(self, key: str) -> List[str]: + raise NotImplementedError() + class VaultFactory(object): @@ -217,9 +266,9 @@ def from_vault_type(app, vault_type: Optional[str], cfg: dict) -> Vault: elif vault_type == "custos": vault = CustosVault(cfg) else: - raise UnknownVaultTypeException(f"Unknown vault type: {vault_type}") + raise InvalidVaultConfigException(f"Unknown vault type: {vault_type}") vault_prefix = cfg.get('path_prefix') or "/galaxy" - return VaultKeyValidationDecorator(VaultKeyPrefixDecorator(vault, prefix=vault_prefix)) + return VaultKeyValidationWrapper(VaultKeyPrefixWrapper(vault, prefix=vault_prefix)) @staticmethod def from_app(app) -> Vault: diff --git a/lib/galaxy_test/api/test_vault.py b/lib/galaxy_test/api/test_vault.py index d241edcf669f..7cc2494b0f8d 100644 --- a/lib/galaxy_test/api/test_vault.py +++ b/lib/galaxy_test/api/test_vault.py @@ -12,7 +12,7 @@ TEST_USER_EMAIL = "vault_test_user@bx.psu.edu" -class VaultApiTestCase(ApiTestCase): +class ExtraUserPreferencesApiTestCase(ApiTestCase): @classmethod def handle_galaxy_config_kwds(cls, config): From e9556a99c732d1d39ff98dc4e0800770387ba803 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 22 Sep 2021 19:33:28 +0200 Subject: [PATCH 182/401] Remove unused dependencies Leftovers from previous refactoring. --- lib/galaxy/webapps/galaxy/api/history_contents.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index d137877fb541..96226aaaa170 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -48,12 +48,6 @@ class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): - hda_manager: hdas.HDAManager = depends(hdas.HDAManager) - history_manager: histories.HistoryManager = depends(histories.HistoryManager) - history_contents_manager: history_contents.HistoryContentsManager = depends(history_contents.HistoryContentsManager) - hda_serializer: hdas.HDASerializer = depends(hdas.HDASerializer) - hdca_serializer: hdcas.HDCASerializer = depends(hdcas.HDCASerializer) - history_contents_filters: history_contents.HistoryContentsFilters = depends(history_contents.HistoryContentsFilters) service: HistoriesContentsService = depends(HistoriesContentsService) From 79fe651023dd08fe0c997d053a6707c5256a652c Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 27 Sep 2021 21:13:08 +0200 Subject: [PATCH 183/401] Fix some issues in HDADetailed schema model - Add some missing properties for metadata - Make some properties optional --- lib/galaxy/schema/schema.py | 52 +++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index dce7429cc717..964255ed8638 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -18,7 +18,6 @@ ConstrainedStr, Extra, Field, - FilePath, Json, UUID4, ) @@ -418,13 +417,13 @@ class HDADetailed(HDASummary): hda_ldda: DatasetSourceType = HdaLddaField accessible: bool = AccessibleField genome_build: Optional[str] = GenomeBuildField - misc_info: str = Field( - ..., + misc_info: Optional[str] = Field( + default=None, title="Miscellaneous Information", description="TODO", ) - misc_blurb: str = Field( - ..., + misc_blurb: Optional[str] = Field( + default=None, title="Miscellaneous Blurb", description="TODO", ) @@ -443,13 +442,23 @@ class HDADetailed(HDASummary): title="Resubmitted", description="Whether the job creating this dataset has been resubmitted.", ) - metadata: Any = Field( # TODO: create pydantic model for metadata? - ..., + metadata: Optional[Any] = Field( # TODO: create pydantic model for metadata? + default=None, title="Metadata", description="The metadata associated with this dataset.", ) + metadata_dbkey: Optional[str] = Field( + "?", + title="Metadata DBKey", + description="TODO", + ) + metadata_data_lines: int = Field( + 0, + title="Metadata Data Lines", + description="TODO", + ) meta_files: List[MetadataFile] = Field( - [], + ..., title="Metadata Files", description="Collection of metadata files associated with this dataset.", ) @@ -459,8 +468,8 @@ class HDADetailed(HDASummary): description="The fully qualified name of the class implementing the data type of this dataset.", example="galaxy.datatypes.data.Text" ) - peek: str = Field( - ..., + peek: Optional[str] = Field( + default=None, title="Peek", description="A few lines of contents from the start of the file.", ) @@ -480,24 +489,24 @@ class HDADetailed(HDASummary): title="Permissions", description="Role-based access and manage control permissions for the dataset.", ) - file_name: FilePath = Field( - ..., + file_name: Optional[str] = Field( + default=None, title="File Name", description="The full path to the dataset file.", ) display_apps: List[DisplayApp] = Field( - [], + ..., title="Display Applications", description="Contains new-style display app urls.", ) display_types: List[DisplayApp] = Field( - [], + ..., title="Legacy Display Applications", description="Contains old-style display app urls.", deprecated=False, # TODO: Should this field be deprecated in favor of display_apps? ) visualizations: List[Visualization] = Field( - [], + ..., title="Visualizations", description="The collection of visualizations that can be applied to this dataset.", ) @@ -2470,9 +2479,18 @@ class DeleteHDCAResult(Model): ) -AnyHDA = Union[HDASummary, HDADetailed, HDABeta] +AnyHDA = Union[HDABeta, HDADetailed, HDASummary] AnyHDCA = Union[HDCABeta, HDCADetailed, HDCASummary] -AnyHistoryContentItem = Union[AnyHDA, HDCASummary, HDCADetailed, HDCABeta] +AnyHistoryContentItem = Union[ + AnyHDA, + AnyHDCA, + Any, # Allows custom keys to be specified in the serialization parameters +] + + +class HistoryContentItemList(BaseModel): + __root__: List[AnyHistoryContentItem] + AnyJobStateSummary = Union[ JobStateSummary, From c3d39c84bbc0f5288899a0206ea656326cea8991 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 27 Sep 2021 21:48:55 +0200 Subject: [PATCH 184/401] Refactor HistoryContentsService - Simplify models and move API documentation to dependency functions. This allows to properly display the documentation in OpenAPI when there are multiple models. --- .../webapps/galaxy/api/history_contents.py | 150 ++++++++++++++++-- .../galaxy/services/history_contents.py | 141 ++++------------ 2 files changed, 163 insertions(+), 128 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 96226aaaa170..aa3ca365088f 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -5,19 +5,14 @@ from typing import ( Any, Dict, + Optional, ) import dateutil.parser +from fastapi import Query -from galaxy import ( - util -) -from galaxy.managers import ( - hdas, - hdcas, - histories, - history_contents -) +from galaxy import util +from galaxy.schema import FilterQueryParams from galaxy.schema.schema import ( DatasetPermissionAction, HistoryContentType, @@ -28,25 +23,144 @@ expose_api, expose_api_anonymous, expose_api_raw, - expose_api_raw_anonymous + expose_api_raw_anonymous, ) from galaxy.webapps.base.controller import ( UsesLibraryMixinItems, - UsesTagsMixin + UsesTagsMixin, ) from galaxy.webapps.galaxy.api.common import parse_serialization_params from galaxy.webapps.galaxy.services.history_contents import ( CreateHistoryContentPayload, + DatasetDetailsType, HistoriesContentsService, HistoryContentsFilterList, - HistoryContentsFilterQueryParams, - HistoryContentsIndexLegacyParams + HistoryContentsIndexParams, + LegacyHistoryContentsIndexParams, +) +from . import ( + BaseGalaxyAPIController, + depends, ) -from . import BaseGalaxyAPIController, depends log = logging.getLogger(__name__) +def get_index_query_params( + v: Optional[str] = Query( # Should this be deprecated at some point and directly use the latest version by default? + default=None, + title="Version", + description="Only `dev` value is allowed. Set it to use the latest version of this endpoint.", + example="dev", + ), + dataset_details: Optional[str] = Query( + default=None, + title="Dataset Details", + description=( + "A comma separated list of encoded dataset IDs that will return additional (full) details " + "instead of the *summary* default information." + ), + deprecated=True, # TODO: remove 'dataset_details' when the UI doesn't need it + ), +) -> HistoryContentsIndexParams: + return HistoryContentsIndexParams( + v=v, + dataset_details=dataset_details, + ) + + +def get_legacy_index_query_params( + ids: Optional[str] = Query( + default=None, + title="IDs", + description=( + "A comma separated list of encoded `HDA/HDCA` IDs. If this list is provided, only information about the " + "specific datasets will be returned. Also, setting this value will return `all` details of the content item." + ), + deprecated=True, + ), + types: Optional[str] = Query( + default=None, + title="Types", + description=( + "A list or comma separated list of kinds of contents to return " + "(currently just `dataset` and `dataset_collection` are available). " + "If unset, all types will be returned." + ), + deprecated=True, + ), + type: Optional[str] = Query( + default=None, + title="Type", + description="Legacy name for the `types` parameter.", + deprecated=True, + ), + details: Optional[str] = Query( + default=None, + title="Details", + description=( + "Legacy name for the `dataset_details` parameter." + ), + deprecated=True, + ), + deleted: Optional[bool] = Query( + default=None, + title="Deleted", + description="Whether to return deleted or undeleted datasets only. Leave unset for both.", + deprecated=True, + ), + visible: Optional[bool] = Query( + default=None, + title="Visible", + description="Whether to return visible or hidden datasets only. Leave unset for both.", + deprecated=True, + ), +) -> LegacyHistoryContentsIndexParams: + return parse_legacy_index_query_params( + ids=ids, + types=types, + type=type, + details=details, + deleted=deleted, + visible=visible, + ) + + +def parse_legacy_index_query_params( + ids: Optional[str] = None, + types: Optional[str] = None, + type: Optional[str] = None, + details: Optional[str] = None, + deleted: Optional[bool] = None, + visible: Optional[bool] = None, + **_, # Additional params are ignored +) -> LegacyHistoryContentsIndexParams: + types = types or type + if types: + content_types = util.listify(types) + else: + content_types = [e.value for e in HistoryContentType] + + dataset_details: Optional[DatasetDetailsType] = None + if ids: + ids = util.listify(ids) + # If explicit ids given, always used detailed result. + dataset_details = 'all' + else: + if details and details != 'all': + dataset_details = set(util.listify(details)) + else: # either None or 'all' + dataset_details = details # type: ignore + + return LegacyHistoryContentsIndexParams( + types=content_types, + ids=ids, + deleted=deleted, + visible=visible, + dataset_details=dataset_details, + ) + + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): service: HistoriesContentsService = depends(HistoriesContentsService) @@ -77,11 +191,13 @@ def index(self, trans, history_id, **kwd): :rtype: list :returns: dictionaries containing summary or detailed HDA information """ - legacy_params = HistoryContentsIndexLegacyParams(**kwd) + index_params = HistoryContentsIndexParams(**kwd) + legacy_params = parse_legacy_index_query_params(**kwd) serialization_params = parse_serialization_params(**kwd) - filter_parameters = HistoryContentsFilterQueryParams(**kwd) + filter_parameters = FilterQueryParams(**kwd) return self.service.index( trans, history_id, + index_params, legacy_params, serialization_params, filter_parameters ) @@ -482,7 +598,7 @@ def archive(self, trans, history_id, filename='', format='zip', dry_run=True, ** .. note:: this is a volatile endpoint and settings and behavior may change. """ dry_run = util.string_as_bool(dry_run) - filter_parameters = HistoryContentsFilterQueryParams(**kwd) + filter_parameters = FilterQueryParams(**kwd) return self.service.archive(trans, history_id, filter_parameters, filename, dry_run) @expose_api_raw_anonymous diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 77180f3536e9..8ec6f26a8884 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -56,7 +56,6 @@ ) from galaxy.schema.fields import ( EncodedDatabaseIdField, - OrderParamField, ) from galaxy.schema.schema import ( AnyHDA, @@ -89,80 +88,21 @@ class DirectionOptions(str, Enum): after = "after" -class HistoryContentsFilterQueryParams(FilterQueryParams): - order: Optional[str] = OrderParamField(default_order="hid-asc") - - HistoryContentFilter = List[Any] # Lists with [attribute:str, operator:str, value:Any] HistoryContentsFilterList = List[HistoryContentFilter] -class HistoryContentsIndexLegacyParamsBase(Model): - deleted: Optional[bool] = Field( - default=None, - title="Deleted", - description="Whether to return deleted or undeleted datasets only. Leave unset for both.", - ) - visible: Optional[bool] = Field( - default=None, - title="Visible", - description="Whether to return visible or hidden datasets only. Leave unset for both.", - ) - - -class HistoryContentsIndexLegacyParams(HistoryContentsIndexLegacyParamsBase): - v: Optional[str] = Field( # Should this be deprecated at some point and directly use the latest version by default? - default=None, - title="Version", - description="Only `dev` value is allowed. Set it to use the latest version of this endpoint.", - ) - ids: Optional[str] = Field( - default=None, - title="IDs", - description=( - "A comma separated list of encoded `HDA` IDs. If this list is provided, only information about the " - "specific datasets will be provided. Also, setting this value will set `dataset_details` to `all`." - ), - ) - types: Optional[Union[List[HistoryContentType], str]] = Field( - default=None, - alias="type", # Legacy alias - title="Types", - description=( - "A list or comma separated list of kinds of contents to return " - "(currently just `dataset` and `dataset_collection` are available)." - ), - ) - dataset_details: Optional[str] = Field( - default=None, - alias="details", # Legacy alias - title="Dataset Details", - description=( - "A comma separated list of encoded dataset IDs that will return additional (full) details " - "instead of the *summary* default information." - ), - ) +class HistoryContentsIndexParams(Model): + v: Optional[str] + dataset_details: Optional[DatasetDetailsType] -class ParsedHistoryContentsLegacyParams(HistoryContentsIndexLegacyParamsBase): - ids: Optional[List[int]] = Field( - default=None, - title="IDs", - description="A list of (decoded) `HDA` IDs to return detailed information. Only these items will be returned.", - ) - types: List[HistoryContentType] = Field( - default=[], - title="Types", - description="A list with the types of contents to return.", - ) - dataset_details: Optional[DatasetDetailsType] = Field( - default=None, - title="Dataset Details", - description=( - "A set of encoded IDs for the datasets that will be returned with detailed " - "information or the value `all` to return (full) details for all datasets." - ), - ) +class LegacyHistoryContentsIndexParams(Model): + ids: Optional[List[EncodedDatabaseIdField]] + types: List[HistoryContentType] + dataset_details: Optional[DatasetDetailsType] + deleted: Optional[bool] + visible: Optional[bool] class CreateHistoryContentPayloadBase(Model): @@ -251,18 +191,19 @@ def index( self, trans, history_id: EncodedDatabaseIdField, - legacy_params: HistoryContentsIndexLegacyParams, + params: HistoryContentsIndexParams, + legacy_params: LegacyHistoryContentsIndexParams, serialization_params: SerializationParams, - filter_query_params: HistoryContentsFilterQueryParams, + filter_query_params: FilterQueryParams, ) -> List[AnyHistoryContentItem]: """ - Return a list of contents (HDAs and HDCAs) for the history with the given ``id`` + Return a list of contents (HDAs and HDCAs) for the history with the given ``ID``. - .. note:: Anonymous users are allowed to get their current history contents + .. note:: Anonymous users are allowed to get their current history contents. """ - if legacy_params.v == 'dev': + if params.v == 'dev': return self.__index_v2( - trans, history_id, legacy_params, serialization_params, filter_query_params + trans, history_id, params, serialization_params, filter_query_params ) return self.__index_legacy(trans, history_id, legacy_params) @@ -271,7 +212,7 @@ def show( trans, id: EncodedDatabaseIdField, serialization_params: SerializationParams, - contents_type: HistoryContentType = HistoryContentType.dataset, + contents_type: Optional[HistoryContentType], fuzzy_count: Optional[int] = None, ) -> AnyHistoryContentItem: """ @@ -304,6 +245,7 @@ def show( :returns: dictionary containing detailed HDA or HDCA information """ + contents_type = contents_type or HistoryContentType.dataset if contents_type == HistoryContentType.dataset: return self.__show_dataset(trans, id, serialization_params) elif contents_type == HistoryContentType.dataset_collection: @@ -586,7 +528,7 @@ def delete( def archive( self, trans, history_id: EncodedDatabaseIdField, - filter_query_params: HistoryContentsFilterQueryParams, + filter_query_params: FilterQueryParams, filename: str = '', dry_run: bool = True, ): @@ -944,14 +886,17 @@ def __deserialize_dataset(self, trans, hda, payload: Dict[str, Any]): def __index_legacy( self, trans, history_id: EncodedDatabaseIdField, - legacy_params: HistoryContentsIndexLegacyParams, + legacy_params: LegacyHistoryContentsIndexParams, ) -> List[AnyHistoryContentItem]: """Legacy implementation of the `index` action.""" history = self._get_history(trans, history_id) - parsed_legacy_params = self._parse_legacy_contents_params(legacy_params) - contents = history.contents_iter(**parsed_legacy_params) + legacy_params_dict = legacy_params.dict(exclude_defaults=True) + ids = legacy_params_dict.get("ids") + if ids: + legacy_params_dict["ids"] = self.decode_ids(ids) + contents = history.contents_iter(**legacy_params_dict) return [ - self._serialize_legacy_content_item(trans, content, parsed_legacy_params.get("dataset_details")) + self._serialize_legacy_content_item(trans, content, legacy_params_dict.get("dataset_details")) for content in contents ] @@ -959,9 +904,9 @@ def __index_v2( self, trans, history_id: EncodedDatabaseIdField, - legacy_params: HistoryContentsIndexLegacyParams, + params: HistoryContentsIndexParams, serialization_params: SerializationParams, - filter_query_params: HistoryContentsFilterQueryParams, + filter_query_params: FilterQueryParams, ) -> List[AnyHistoryContentItem]: """ Latests implementation of the `index` action. @@ -969,11 +914,9 @@ def __index_v2( """ history = self._get_history(trans, history_id) filters = self.history_contents_filters.parse_query_filters(filter_query_params) + filter_query_params.order = filter_query_params.order or "hid-asc" order_by = self.build_order_by(self.history_contents_manager, filter_query_params.order) - # TODO: > 16.04: remove these - # TODO: remove 'dataset_details' and the following section when the UI doesn't need it - parsed_legacy_params = self._parse_legacy_contents_params(legacy_params) contents = self.history_contents_manager.contents( history, filters=filters, @@ -985,7 +928,7 @@ def __index_v2( return [ self._serialize_content_item( trans, content, - dataset_details=parsed_legacy_params.get("dataset_details"), + dataset_details=params.dataset_details, serialization_params=serialization_params, ) for content in contents @@ -1036,30 +979,6 @@ def _serialize_content_item( content, user=trans.user, trans=trans, view=view, **serialization_params_dict ) - def _parse_legacy_contents_params(self, params: HistoryContentsIndexLegacyParams): - if params.types: - types = util.listify(params.types) - else: - types = [e.value for e in HistoryContentType] - - details: Any = params.dataset_details - ids = None - if params.ids: - ids = [self.decode_id(EncodedDatabaseIdField(id)) for id in params.ids.split(',')] - # If explicit ids given, always used detailed result. - details = 'all' - else: - if details and details != 'all': - details = set(util.listify(details)) - - return ParsedHistoryContentsLegacyParams( - types=types, - ids=ids, - deleted=params.deleted, - visible=params.visible, - dataset_details=details, - ).dict(exclude_defaults=True) - def __collection_dict(self, trans, dataset_collection_instance, **kwds): return dictify_dataset_collection_instance(dataset_collection_instance, security=trans.security, parent=dataset_collection_instance.history, **kwds) From 8f94aca51f78b47755d797740d470fa785246bff Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 28 Sep 2021 13:02:49 +0200 Subject: [PATCH 185/401] Allow custom model properties to be serialized --- lib/galaxy/schema/schema.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 964255ed8638..3ee31f6ccca5 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -358,8 +358,11 @@ class HistoryItemBase(Model): class HistoryItemCommon(HistoryItemBase): """Common information provided by items contained in a History.""" - type_id: Optional[str] = Field( - default=None, + class Config: + extra = Extra.allow + + type_id: str = Field( + ..., title="Type - ID", description="The type and the encoded ID of this item. Used for caching.", example="dataset-616e371b2cc6c62e", @@ -2479,19 +2482,25 @@ class DeleteHDCAResult(Model): ) +class CustomHistoryItem(Model): + """Can contain any serializable property of the item. + + Allows arbitrary custom keys to be specified in the serialization + parameters without a particular view (predefined set of keys). + """ + class Config: + extra = Extra.allow + + AnyHDA = Union[HDABeta, HDADetailed, HDASummary] AnyHDCA = Union[HDCABeta, HDCADetailed, HDCASummary] AnyHistoryContentItem = Union[ AnyHDA, AnyHDCA, - Any, # Allows custom keys to be specified in the serialization parameters + CustomHistoryItem, ] -class HistoryContentItemList(BaseModel): - __root__: List[AnyHistoryContentItem] - - AnyJobStateSummary = Union[ JobStateSummary, ImplicitCollectionJobsStateSummary, From 1f89295a1403be9ebc2a4f71cba26f3346f3a235 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 28 Sep 2021 13:11:23 +0200 Subject: [PATCH 186/401] Add FastAPI route for index operation The query parameters are retrieved by dependency functions to be able to render the documentation correctly. Directly using the models as dependecies will work but the documentation is not shown in OpenAPI for some reason. --- .../webapps/galaxy/api/history_contents.py | 72 ++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index aa3ca365088f..9281db8055ea 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -5,15 +5,26 @@ from typing import ( Any, Dict, + List, Optional, ) import dateutil.parser -from fastapi import Query +from fastapi import ( + Depends, + Path, + Query, +) from galaxy import util -from galaxy.schema import FilterQueryParams +from galaxy.managers.context import ProvidesHistoryContext +from galaxy.schema import ( + FilterQueryParams, + SerializationParams, +) +from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.schema.schema import ( + AnyHistoryContentItem, DatasetPermissionAction, HistoryContentType, UpdateDatasetPermissionsPayload, @@ -29,7 +40,11 @@ UsesLibraryMixinItems, UsesTagsMixin, ) -from galaxy.webapps.galaxy.api.common import parse_serialization_params +from galaxy.webapps.galaxy.api.common import ( + get_filter_query_params, + parse_serialization_params, + query_serialization_params, +) from galaxy.webapps.galaxy.services.history_contents import ( CreateHistoryContentPayload, DatasetDetailsType, @@ -41,11 +56,22 @@ from . import ( BaseGalaxyAPIController, depends, + DependsOnTrans, + Router, ) log = logging.getLogger(__name__) +router = Router(tags=['histories']) + +HistoryIDPathParam: EncodedDatabaseIdField = Path( + ..., + title='History ID', + description='The ID of the History.' +) + + def get_index_query_params( v: Optional[str] = Query( # Should this be deprecated at some point and directly use the latest version by default? default=None, @@ -161,6 +187,46 @@ def parse_legacy_index_query_params( ) +@router.cbv +class FastAPIHistoryContents: + service: HistoriesContentsService = depends(HistoriesContentsService) + + @router.get( + '/api/histories/{history_id}/contents', + summary='Returns the contents of the given history.', + ) + @router.get( + '/api/histories/{history_id}/contents/{type}s', + summary='Returns the contents of the given history filtered by type.', + ) + def index( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + index_params: HistoryContentsIndexParams = Depends(get_index_query_params), + legacy_params: LegacyHistoryContentsIndexParams = Depends(get_legacy_index_query_params), + serialization_params: SerializationParams = Depends(query_serialization_params), + filter_query_params: FilterQueryParams = Depends(get_filter_query_params), + ) -> List[AnyHistoryContentItem]: + """ + Return a list of `HDA`/`HDCA` data for the history with the given ``ID``. + + - The contents can be filtered and queried using the appropriate parameters. + - The amount of information returned for each item can be customized. + + .. note:: Anonymous users are allowed to get their current history contents. + """ + items = self.service.index( + trans, + history_id, + index_params, + legacy_params, + serialization_params, + filter_query_params, + ) + return items + + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): service: HistoriesContentsService = depends(HistoriesContentsService) From 6c134a56b956da9245601400aec6a3434aadeda3 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 28 Sep 2021 13:12:48 +0200 Subject: [PATCH 187/401] Add FastAPI route for show operation --- .../webapps/galaxy/api/history_contents.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 9281db8055ea..97ae4efdc9ad 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -71,6 +71,12 @@ description='The ID of the History.' ) +HistoryItemIDPathParam: EncodedDatabaseIdField = Path( + ..., + title='History Item ID', + description='The ID of the item (`HDA`/`HDCA`) contained in the history.' +) + def get_index_query_params( v: Optional[str] = Query( # Should this be deprecated at some point and directly use the latest version by default? @@ -226,6 +232,60 @@ def index( ) return items + @router.get( + '/api/histories/{history_id}/contents/{id}', + summary='Return detailed information about an HDA within a history.', + deprecated=True, + ) + @router.get( + '/api/histories/{history_id}/contents/{type}s/{id}', + summary='Return detailed information about a specific HDA or HDCA with the given `ID` within a history.', + ) + def show( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + id: EncodedDatabaseIdField = HistoryItemIDPathParam, + type: Optional[HistoryContentType] = Query( + default=None, + title="Content Type", + description="The type of the history element to show.", + example=HistoryContentType.dataset, + ), + fuzzy_count: Optional[int] = Query( + default=None, + title="Fuzzy Count", + description=( + 'This value can be used to broadly restrict the magnitude ' + 'of the number of elements returned via the API for large ' + 'collections. The number of actual elements returned may ' + 'be "a bit" more than this number or "a lot" less - varying ' + 'on the depth of nesting, balance of nesting at each level, ' + 'and size of target collection. The consumer of this API should ' + 'not expect a stable number or pre-calculable number of ' + 'elements to be produced given this parameter - the only ' + 'promise is that this API will not respond with an order ' + 'of magnitude more elements estimated with this value. ' + 'The UI uses this parameter to fetch a "balanced" concept of ' + 'the "start" of large collections at every depth of the ' + 'collection.' + ) + ), + serialization_params: SerializationParams = Depends(query_serialization_params), + ) -> AnyHistoryContentItem: + """ + Return detailed information about an `HDA` or `HDCA` within a history. + + .. note:: Anonymous users are allowed to get their current history contents. + """ + return self.service.show( + trans, + id=id, + serialization_params=serialization_params, + contents_type=type, + fuzzy_count=fuzzy_count, + ) + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): From 97417bdc9cc971dd3ded6b77346114dcb4fcadad Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 29 Sep 2021 15:58:36 +0200 Subject: [PATCH 188/401] Small refactoring + docstrings --- .../webapps/galaxy/api/history_contents.py | 40 +++++++++++++++---- .../galaxy/services/history_contents.py | 4 +- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 97ae4efdc9ad..7e49ab8bdab4 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -95,12 +95,27 @@ def get_index_query_params( deprecated=True, # TODO: remove 'dataset_details' when the UI doesn't need it ), ) -> HistoryContentsIndexParams: - return HistoryContentsIndexParams( + """This function is meant to be used as a dependency to render the OpenAPI documentation + correctly when multiple model are used in the same route.""" + return parse_index_query_params( v=v, dataset_details=dataset_details, ) +def parse_index_query_params( + v: Optional[str] = None, + dataset_details: Optional[str] = None, + **_, # Additional params are ignored +) -> HistoryContentsIndexParams: + """Parses query parameters for the history contents `index` operation + and returns a model containing the values in the correct type.""" + return HistoryContentsIndexParams( + v=v, + dataset_details=parse_dataset_details(dataset_details), + ) + + def get_legacy_index_query_params( ids: Optional[str] = Query( default=None, @@ -148,6 +163,8 @@ def get_legacy_index_query_params( deprecated=True, ), ) -> LegacyHistoryContentsIndexParams: + """This function is meant to be used as a dependency to render the OpenAPI documentation + correctly when multiple model are used in the same route.""" return parse_legacy_index_query_params( ids=ids, types=types, @@ -167,22 +184,20 @@ def parse_legacy_index_query_params( visible: Optional[bool] = None, **_, # Additional params are ignored ) -> LegacyHistoryContentsIndexParams: + """Parses (legacy) query parameters for the history contents `index` operation + and returns a model containing the values in the correct type.""" types = types or type if types: content_types = util.listify(types) else: content_types = [e.value for e in HistoryContentType] - dataset_details: Optional[DatasetDetailsType] = None if ids: ids = util.listify(ids) # If explicit ids given, always used detailed result. dataset_details = 'all' else: - if details and details != 'all': - dataset_details = set(util.listify(details)) - else: # either None or 'all' - dataset_details = details # type: ignore + dataset_details = parse_dataset_details(details) return LegacyHistoryContentsIndexParams( types=content_types, @@ -193,6 +208,17 @@ def parse_legacy_index_query_params( ) +def parse_dataset_details(details: Optional[str]): + """Parses the different values that the `dataset_details` parameter + can have from a string.""" + dataset_details: Optional[DatasetDetailsType] = None + if details and details != 'all': + dataset_details = set(util.listify(details)) + else: # either None or 'all' + dataset_details = details # type: ignore + return dataset_details + + @router.cbv class FastAPIHistoryContents: service: HistoriesContentsService = depends(HistoriesContentsService) @@ -317,7 +343,7 @@ def index(self, trans, history_id, **kwd): :rtype: list :returns: dictionaries containing summary or detailed HDA information """ - index_params = HistoryContentsIndexParams(**kwd) + index_params = parse_index_query_params(**kwd) legacy_params = parse_legacy_index_query_params(**kwd) serialization_params = parse_serialization_params(**kwd) filter_parameters = FilterQueryParams(**kwd) diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 8ec6f26a8884..d8edafa927d9 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -93,11 +93,13 @@ class DirectionOptions(str, Enum): class HistoryContentsIndexParams(Model): - v: Optional[str] + """Query parameters exclusively used by the *new version* of `index` operation.""" + v: Optional[Literal['dev']] dataset_details: Optional[DatasetDetailsType] class LegacyHistoryContentsIndexParams(Model): + """Query parameters exclusively used by the *legacy version* of `index` operation.""" ids: Optional[List[EncodedDatabaseIdField]] types: List[HistoryContentType] dataset_details: Optional[DatasetDetailsType] From 2c101621883223d748cbc883b5339366dfbc66e1 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 29 Sep 2021 17:45:43 +0200 Subject: [PATCH 189/401] Add FastAPI route for `index_jobs_summary` operation --- .../webapps/galaxy/api/history_contents.py | 74 +++++++++++++++++-- .../galaxy/services/history_contents.py | 18 ++--- 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 7e49ab8bdab4..0d8595085e12 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -25,6 +25,7 @@ from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.schema.schema import ( AnyHistoryContentItem, + AnyJobStateSummary, DatasetPermissionAction, HistoryContentType, UpdateDatasetPermissionsPayload, @@ -50,6 +51,7 @@ DatasetDetailsType, HistoriesContentsService, HistoryContentsFilterList, + HistoryContentsIndexJobsSummaryParams, HistoryContentsIndexParams, LegacyHistoryContentsIndexParams, ) @@ -89,14 +91,14 @@ def get_index_query_params( default=None, title="Dataset Details", description=( - "A comma separated list of encoded dataset IDs that will return additional (full) details " + "A comma-separated list of encoded dataset IDs that will return additional (full) details " "instead of the *summary* default information." ), deprecated=True, # TODO: remove 'dataset_details' when the UI doesn't need it ), ) -> HistoryContentsIndexParams: """This function is meant to be used as a dependency to render the OpenAPI documentation - correctly when multiple model are used in the same route.""" + correctly""" return parse_index_query_params( v=v, dataset_details=dataset_details, @@ -121,7 +123,7 @@ def get_legacy_index_query_params( default=None, title="IDs", description=( - "A comma separated list of encoded `HDA/HDCA` IDs. If this list is provided, only information about the " + "A comma-separated list of encoded `HDA/HDCA` IDs. If this list is provided, only information about the " "specific datasets will be returned. Also, setting this value will return `all` details of the content item." ), deprecated=True, @@ -130,7 +132,7 @@ def get_legacy_index_query_params( default=None, title="Types", description=( - "A list or comma separated list of kinds of contents to return " + "A list or comma-separated list of kinds of contents to return " "(currently just `dataset` and `dataset_collection` are available). " "If unset, all types will be returned." ), @@ -164,7 +166,7 @@ def get_legacy_index_query_params( ), ) -> LegacyHistoryContentsIndexParams: """This function is meant to be used as a dependency to render the OpenAPI documentation - correctly when multiple model are used in the same route.""" + correctly""" return parse_legacy_index_query_params( ids=ids, types=types, @@ -219,6 +221,45 @@ def parse_dataset_details(details: Optional[str]): return dataset_details +def get_index_jobs_summary_params( + ids: Optional[str] = Query( + default=None, + title="IDs", + description=( + "A comma-separated list of encoded ids of job summary objects to return - if `ids` " + "is specified types must also be specified and have same length." + ), + ), + types: Optional[str] = Query( + default=None, + title="Types", + description=( + "A comma-separated list of type of object represented by elements in the ids array - any of " + "`Job`, `ImplicitCollectionJob`, or `WorkflowInvocation`." + ), + ), +) -> HistoryContentsIndexJobsSummaryParams: + """This function is meant to be used as a dependency to render the OpenAPI documentation + correctly""" + return parse_index_jobs_summary_params( + ids=ids, + types=types, + ) + + +def parse_index_jobs_summary_params( + ids: Optional[str] = None, + types: Optional[str] = None, + **_, # Additional params are ignored +) -> HistoryContentsIndexJobsSummaryParams: + """Parses query parameters for the history contents `index_jobs_summary` operation + and returns a model containing the values in the correct type.""" + return HistoryContentsIndexJobsSummaryParams( + ids=util.listify(ids), + types=util.listify(types) + ) + + @router.cbv class FastAPIHistoryContents: service: HistoriesContentsService = depends(HistoriesContentsService) @@ -312,6 +353,24 @@ def show( fuzzy_count=fuzzy_count, ) + @router.get( + '/api/histories/{history_id}/jobs_summary', + summary='Return job state summary info for jobs, implicit groups jobs for collections or workflow invocations.', + ) + def index_jobs_summary( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + params: HistoryContentsIndexJobsSummaryParams = Depends(get_index_jobs_summary_params), + ) -> List[AnyJobStateSummary]: + """Return job state summary info for jobs, implicit groups jobs for collections or workflow invocations. + + **Warning**: We allow anyone to fetch job state information about any object they + can guess an encoded ID for - it isn't considered protected data. This keeps + polling IDs as part of state calculation for large histories and collections as + efficient as possible. + """ + return self.service.index_jobs_summary(trans, params) + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): @@ -426,9 +485,8 @@ def index_jobs_summary(self, trans, history_id, **kwd): :rtype: dict[] :returns: an array of job summary object dictionaries. """ - ids = util.listify(kwd.get("ids", None)) - types = util.listify(kwd.get("types", None)) - return self.service.index_jobs_summary(trans, ids, types) + params = parse_index_jobs_summary_params(**kwd) + return self.service.index_jobs_summary(trans, params) @expose_api_anonymous def show_jobs_summary(self, trans, id, history_id, **kwd): diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index d8edafa927d9..efb9a17bb463 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -107,6 +107,12 @@ class LegacyHistoryContentsIndexParams(Model): visible: Optional[bool] +class HistoryContentsIndexJobsSummaryParams(Model): + """Query parameters exclusively used by the `index_jobs_summary` operation.""" + ids: List[EncodedDatabaseIdField] = [] + types: List[JobSourceType] = [] + + class CreateHistoryContentPayloadBase(Model): type: Optional[HistoryContentType] = Field( HistoryContentType.dataset, @@ -256,8 +262,7 @@ def show( def index_jobs_summary( self, trans, - ids: List[EncodedDatabaseIdField], - types: List[JobSourceType], + params: HistoryContentsIndexJobsSummaryParams, ) -> List[AnyJobStateSummary]: """ Return job state summary info for jobs, implicit groups jobs for collections or workflow invocations @@ -266,14 +271,9 @@ def index_jobs_summary( can guess an encoded ID for - it isn't considered protected data. This keeps polling IDs as part of state calculation for large histories and collections as efficient as possible. - - :param ids: the encoded ids of job summary objects to return - if ids - is specified types must also be specified and have same length. - :param types: type of object represented by elements in the ids array - any of - Job, ImplicitCollectionJob, or WorkflowInvocation. - - :returns: an array of job summary object dictionaries. """ + ids = params.ids + types = params.types if len(ids) != len(types): raise exceptions.RequestParameterInvalidException( f"The number of ids ({len(ids)}) and types ({len(types)}) must match." From 928a24f2cf1dd3a57cf3e79e91757e5cb99cbf83 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 29 Sep 2021 18:03:01 +0200 Subject: [PATCH 190/401] Add FastAPI route for `show_jobs_summary` operation --- .../webapps/galaxy/api/history_contents.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 0d8595085e12..64f56bf105e4 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -234,7 +234,7 @@ def get_index_jobs_summary_params( default=None, title="Types", description=( - "A comma-separated list of type of object represented by elements in the ids array - any of " + "A comma-separated list of type of object represented by elements in the `ids` array - any of " "`Job`, `ImplicitCollectionJob`, or `WorkflowInvocation`." ), ), @@ -371,6 +371,30 @@ def index_jobs_summary( """ return self.service.index_jobs_summary(trans, params) + @router.get( + '/api/histories/{history_id}/contents/{type}s/{id}/jobs_summary', + summary='Return detailed information about an `HDA` or `HDCAs` jobs.', + ) + def show_jobs_summary( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + id: EncodedDatabaseIdField = HistoryItemIDPathParam, + type: HistoryContentType = Path( + default=None, + title="Content Type", + description="The type of the history element to show.", + example=HistoryContentType.dataset, + ), + ) -> AnyJobStateSummary: + """Return detailed information about an `HDA` or `HDCAs` jobs. + + **Warning**: We allow anyone to fetch job state information about any object they + can guess an encoded ID for - it isn't considered protected data. This keeps + polling IDs as part of state calculation for large histories and collections as + efficient as possible. + """ + return self.service.show_jobs_summary(trans, id, contents_type=type) + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): From 30721b702ce3a810b60998d65d092ed36ffdea53 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 30 Sep 2021 11:28:14 +0200 Subject: [PATCH 191/401] Refactor `download_dataset_collection` - Rename method and return ZipstreamWrapper to adapt response depending on API framework (legacy or FastAPI) - Move response related code from service to controller - Raise proper exception and do not expose internal exception on API response --- lib/galaxy/webapps/galaxy/api/history_contents.py | 4 +++- .../webapps/galaxy/services/history_contents.py | 14 ++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 64f56bf105e4..7f3194169b01 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -550,7 +550,9 @@ def download_dataset_collection(self, trans, id, history_id=None, **kwd): :param id: encoded HistoryDatasetCollectionAssociation (HDCA) id :param history_id: encoded id string of the HDCA's History """ - return self.service.download_dataset_collection(trans, id) + archive = self.service.get_dataset_collection_archive_for_download(trans, id) + trans.response.headers.update(archive.get_headers()) + return archive.response() @expose_api_anonymous def create(self, trans, history_id, payload, **kwd): diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index efb9a17bb463..90e5147dd2e6 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -21,7 +21,6 @@ from galaxy import ( exceptions, - util ) from galaxy.managers import ( folders, @@ -319,7 +318,7 @@ def show_jobs_summary( assert job is None or implicit_collection_jobs is None return self.encode_all_ids(summarize_jobs_to_dict(trans.sa_session, job or implicit_collection_jobs)) - def download_dataset_collection(self, trans, id: EncodedDatabaseIdField): + def get_dataset_collection_archive_for_download(self, trans, id: EncodedDatabaseIdField): """ Download the content of a HistoryDatasetCollection as a tgz archive while maintaining approximate collection structure. @@ -329,15 +328,14 @@ def download_dataset_collection(self, trans, id: EncodedDatabaseIdField): try: dataset_collection_instance = self.__get_accessible_collection(trans, id) return self.__stream_dataset_collection(trans, dataset_collection_instance) - except Exception as e: - log.exception("Error in API while creating dataset collection archive") - trans.response.status = 500 - return {'error': util.unicodify(e)} + except Exception: + error_message = "Error in API while creating dataset collection archive" + log.exception(error_message) + raise exceptions.InternalServerError(error_message) def __stream_dataset_collection(self, trans, dataset_collection_instance): archive = hdcas.stream_dataset_collection(dataset_collection_instance=dataset_collection_instance, upstream_mod_zip=trans.app.config.upstream_mod_zip) - trans.response.headers.update(archive.get_headers()) - return archive.response() + return archive def create( self, trans, From b1ae4252f8207e4a574e3c110dd93484d33b2b2d Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 1 Oct 2021 18:07:35 +0200 Subject: [PATCH 192/401] Add FastAPI route for `download_dataset_collection` - Temporarily adding here the '/api/dataset_collection/{id}/download' route. It can be moved to the dataset_collections API and reuse this service later. - I managed to make the StreamingResponse here work with minimal changes in ZipstreamWrapper, but I'm not sure if this is the best solution or we need to think of a different solution when the legacy controllers are removed. --- lib/galaxy/util/zipstream.py | 3 ++ .../webapps/galaxy/api/history_contents.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/lib/galaxy/util/zipstream.py b/lib/galaxy/util/zipstream.py index f6d706508681..e467d5ab5860 100644 --- a/lib/galaxy/util/zipstream.py +++ b/lib/galaxy/util/zipstream.py @@ -22,6 +22,9 @@ def response(self): else: yield iter(self.archive) + def get_iterator(self): + return iter(self.archive) + def get_headers(self): headers = {} if self.archive_name: diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 7f3194169b01..dc8851616d0d 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -15,6 +15,7 @@ Path, Query, ) +from starlette.responses import StreamingResponse from galaxy import util from galaxy.managers.context import ProvidesHistoryContext @@ -79,6 +80,12 @@ description='The ID of the item (`HDA`/`HDCA`) contained in the history.' ) +HistoryHDCAIDPathParam: EncodedDatabaseIdField = Path( + ..., + title='History Dataset Collection ID', + description='The ID of the `HDCA` contained in the history.' +) + def get_index_query_params( v: Optional[str] = Query( # Should this be deprecated at some point and directly use the latest version by default? @@ -360,6 +367,7 @@ def show( def index_jobs_summary( self, trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, params: HistoryContentsIndexJobsSummaryParams = Depends(get_index_jobs_summary_params), ) -> List[AnyJobStateSummary]: """Return job state summary info for jobs, implicit groups jobs for collections or workflow invocations. @@ -378,6 +386,7 @@ def index_jobs_summary( def show_jobs_summary( self, trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, id: EncodedDatabaseIdField = HistoryItemIDPathParam, type: HistoryContentType = Path( default=None, @@ -395,6 +404,29 @@ def show_jobs_summary( """ return self.service.show_jobs_summary(trans, id, contents_type=type) + @router.get( + '/api/histories/{history_id}/contents/dataset_collections/{id}/download', + summary='Download the content of a dataset collection as a `zip` archive.', + response_class=StreamingResponse, + ) + @router.get( # TODO: Move to dataset_collections API? + '/api/dataset_collection/{id}/download', + summary='Download the content of a dataset collection as a `zip` archive.', + response_class=StreamingResponse, + tags=["dataset_collections"], + ) + def download_dataset_collection( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + id: EncodedDatabaseIdField = HistoryHDCAIDPathParam, + ): + """Download the content of a history dataset collection as a `zip` archive + while maintaining approximate collection structure. + """ + archive = self.service.get_dataset_collection_archive_for_download(trans, id) + return StreamingResponse(archive.get_iterator(), headers=archive.get_headers()) + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): From 004a666c2b9eb379a4717d00d2b78933a598f711 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 5 Oct 2021 12:39:39 +0200 Subject: [PATCH 193/401] Adapt history contents API tests to use JSON payload See https://github.com/galaxyproject/galaxy/pull/12152 for details --- lib/galaxy_test/api/test_history_contents.py | 31 ++++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py index 428e20091334..9014a34e0a8a 100644 --- a/lib/galaxy_test/api/test_history_contents.py +++ b/lib/galaxy_test/api/test_history_contents.py @@ -1,4 +1,3 @@ -import json import time import urllib from datetime import datetime @@ -170,7 +169,7 @@ def test_hda_copy(self): ) second_history_id = self.dataset_populator.new_history() assert self.__count_contents(second_history_id) == 0 - create_response = self._post(f"histories/{second_history_id}/contents", create_data) + create_response = self._post(f"histories/{second_history_id}/contents", create_data, json=True) self._assert_status_code_is(create_response, 200) assert self.__count_contents(second_history_id) == 1 @@ -181,7 +180,7 @@ def test_library_copy(self): content=ld["id"], ) assert self.__count_contents(self.history_id) == 0 - create_response = self._post(f"histories/{self.history_id}/contents", create_data) + create_response = self._post(f"histories/{self.history_id}/contents", create_data, json=True) self._assert_status_code_is(create_response, 200) assert self.__count_contents(self.history_id) == 1 @@ -342,18 +341,18 @@ def test_dataset_collection_create_from_exisiting_datasets_with_new_tags(self): assert update_response['tags'] == ['existing:tag'] creation_payload = {'collection_type': 'list', 'history_id': history_id, - 'element_identifiers': json.dumps([{'id': hda_id, - 'src': 'hda', - 'name': 'element_id1', - 'tags': ['my_new_tag']}, - {'id': hda2_id, - 'src': 'hda', - 'name': 'element_id2', - 'tags': ['another_new_tag']} - ]), + 'element_identifiers': [{'id': hda_id, + 'src': 'hda', + 'name': 'element_id1', + 'tags': ['my_new_tag']}, + {'id': hda2_id, + 'src': 'hda', + 'name': 'element_id2', + 'tags': ['another_new_tag']} + ], 'type': 'dataset_collection', 'copy_elements': True} - r = self._post(f"histories/{self.history_id}/contents", creation_payload).json() + r = self._post(f"histories/{self.history_id}/contents", creation_payload, json=True).json() assert r['elements'][0]['object']['id'] != hda_id, "HDA has not been copied" assert len(r['elements'][0]['object']['tags']) == 1 assert r['elements'][0]['object']['tags'][0] == 'my_new_tag' @@ -489,7 +488,7 @@ def test_hdca_copy(self): content=hdca_id, ) assert len(self._get(f"histories/{second_history_id}/contents/dataset_collections").json()) == 0 - create_response = self._post(f"histories/{second_history_id}/contents/dataset_collections", create_data) + create_response = self._post(f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True) self.__check_create_collection_response(create_response) contents = self._get(f"histories/{second_history_id}/contents/dataset_collections").json() assert len(contents) == 1 @@ -503,7 +502,7 @@ def test_hdca_copy_with_new_dbkey(self): assert hdca["elements"][0]["object"]["metadata_dbkey"] == "?" assert hdca["elements"][0]["object"]["genome_build"] == "?" create_data = {'source': 'hdca', 'content': hdca_id, 'dbkey': 'hg19'} - create_response = self._post(f"histories/{self.history_id}/contents/dataset_collections", create_data) + create_response = self._post(f"histories/{self.history_id}/contents/dataset_collections", create_data, json=True) collection = self.__check_create_collection_response(create_response) new_forward = collection['elements'][0]['object'] assert new_forward["metadata_dbkey"] == "hg19" @@ -519,7 +518,7 @@ def test_hdca_copy_and_elements(self): copy_elements=True, ) assert len(self._get(f"histories/{second_history_id}/contents/dataset_collections").json()) == 0 - create_response = self._post(f"histories/{second_history_id}/contents/dataset_collections", create_data) + create_response = self._post(f"histories/{second_history_id}/contents/dataset_collections", create_data, json=True) self.__check_create_collection_response(create_response) contents = self._get(f"histories/{second_history_id}/contents/dataset_collections").json() From 45bc7bfbf932aba6403dd0b4e43d9dec17874533 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 5 Oct 2021 12:47:57 +0200 Subject: [PATCH 194/401] Fix `create` endpoint and related models - Make some fields optional as they depend on the type of request - Explicitly get history content type from payload or path param --- .../galaxy/services/history_contents.py | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 90e5147dd2e6..083a1e3b822d 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -60,6 +60,7 @@ AnyHDA, AnyHistoryContentItem, AnyJobStateSummary, + ColletionSourceType, DatasetAssociationRoles, DeleteHDCAResult, HistoryContentSource, @@ -139,6 +140,52 @@ class CreateHistoryContentPayloadFromCopy(CreateHistoryContentPayloadBase): ) +class CollectionElementIdentifier(Model): + name: Optional[str] = Field( + None, + title="Name", + description="The name of the element.", + ) + src: ColletionSourceType = Field( + ..., + title="Source", + description="The source of the element.", + ) + id: EncodedDatabaseIdField = Field( + ..., + title="ID", + description="The encoded ID of the element.", + ) + tags: List[str] = Field( + default=[], + title="Tags", + description="The list of tags associated with the element.", + ) + + +class CreateNewCollectionPayload(Model): + collection_type: Optional[str] = Field( + default=None, + title="Collection Type", + description="The type of the collection. For example, `list`, `paired`, `list:paired`.", + ) + element_identifiers: Optional[List[CollectionElementIdentifier]] = Field( + default=None, + title="Element Identifiers", + description="List of elements that should be in the new collection.", + ) + name: Optional[str] = Field( + default=None, + title="Name", + description="The name of the new collection.", + ) + hide_source_items: Optional[bool] = Field( + default=False, + title="Hide Source Items", + description="Whether to mark the original HDAs as hidden.", + ) + + class CreateHistoryContentPayloadFromCollection(CreateHistoryContentPayloadFromCopy): dbkey: Optional[str] = Field( default=None, @@ -156,7 +203,7 @@ class CreateHistoryContentPayloadFromCollection(CreateHistoryContentPayloadFromC ) -class CreateHistoryContentPayload(CreateHistoryContentPayloadFromCollection): +class CreateHistoryContentPayload(CreateHistoryContentPayloadFromCollection, CreateNewCollectionPayload): class Config: extra = Extra.allow From aee536dfd63b75379773a3773e139bf1c292b3a0 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 5 Oct 2021 12:51:23 +0200 Subject: [PATCH 195/401] Add FastAPI route for `create` operation - Also marks `/api/histories/{history_id}/contents` as deprecated because explicit sounds better than implicit, but we can always remove the deprecation flag if that's ok. --- .../webapps/galaxy/api/history_contents.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index dc8851616d0d..d1776a630a8f 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -11,6 +11,7 @@ import dateutil.parser from fastapi import ( + Body, Depends, Path, Query, @@ -427,6 +428,32 @@ def download_dataset_collection( archive = self.service.get_dataset_collection_archive_for_download(trans, id) return StreamingResponse(archive.get_iterator(), headers=archive.get_headers()) + @router.post( + '/api/histories/{history_id}/contents/{type}s', + summary='Create a new `HDA` or `HDCA` in the given History.', + ) + @router.post( + '/api/histories/{history_id}/contents', + summary='Create a new `HDA` or `HDCA` in the given History.', + deprecated=True, + ) + def create( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + type: Optional[HistoryContentType] = Query( + default=None, + title="Content Type", + description="The type of the history element to create.", + example=HistoryContentType.dataset, + ), + serialization_params: SerializationParams = Depends(query_serialization_params), + payload: CreateHistoryContentPayload = Body(...), + ) -> AnyHistoryContentItem: + """Create a new `HDA` or `HDCA` in the given History.""" + payload.type = type or payload.type + return self.service.create(trans, history_id, payload, serialization_params) + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): From 85395d045ceb95e15252b13a74c8111a2987eaa0 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 5 Oct 2021 17:10:05 +0200 Subject: [PATCH 196/401] Adapt more API tests around history contents to use JSON payload See https://github.com/galaxyproject/galaxy/pull/12152 for details --- lib/galaxy_test/api/test_jobs.py | 4 ++-- lib/galaxy_test/api/test_libraries.py | 12 ++++++------ lib/galaxy_test/api/test_workflow_extraction.py | 4 ++-- lib/galaxy_test/api/test_workflows.py | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 78decfefb1ce..851fc140c598 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -547,7 +547,7 @@ def test_search(self, history_id): # We first copy the datasets, so that the update time is lower than the job creation time new_history_id = self.dataset_populator.new_history() copy_payload = {"content": dataset_id, "source": "hda", "type": "dataset"} - copy_response = self._post(f"histories/{new_history_id}/contents", data=copy_payload) + copy_response = self._post(f"histories/{new_history_id}/contents", data=copy_payload, json=True) self._assert_status_code_is(copy_response, 200) inputs = json.dumps({ 'input1': {'src': 'hda', 'id': dataset_id} @@ -654,7 +654,7 @@ def test_search_with_hdca_pair_input(self, history_id): # We test that a job can be found even if the collection has been copied to another history new_history_id = self.dataset_populator.new_history() copy_payload = {"content": list_id_a, "source": "hdca", "type": "dataset_collection"} - copy_response = self._post(f"histories/{new_history_id}/contents", data=copy_payload) + copy_response = self._post(f"histories/{new_history_id}/contents", data=copy_payload, json=True) self._assert_status_code_is(copy_response, 200) new_list_a = copy_response.json()['id'] copied_inputs = json.dumps({ diff --git a/lib/galaxy_test/api/test_libraries.py b/lib/galaxy_test/api/test_libraries.py index bc77335d8d11..059721cc7bdf 100644 --- a/lib/galaxy_test/api/test_libraries.py +++ b/lib/galaxy_test/api/test_libraries.py @@ -424,7 +424,7 @@ def test_import_paired_collection(self): 'name': collection_name, 'collection_type': 'list:paired', "type": "dataset_collection", - 'element_identifiers': json.dumps([ + 'element_identifiers': [ { 'src': 'new_collection', 'name': 'pair1', @@ -432,9 +432,9 @@ def test_import_paired_collection(self): 'element_identifiers': [{'name': 'forward', 'src': 'ldda', 'id': ld['id']}, {'name': 'reverse', 'src': 'ldda', 'id': ld['id']}] } - ]) + ] } - new_collection = self._post(url, payload).json() + new_collection = self._post(url, payload, json=True).json() assert new_collection['name'] == collection_name pair = new_collection['elements'][0] assert pair['element_identifier'] == 'pair1' @@ -453,14 +453,14 @@ def _import_to_history(self, visible=True): "history_id": history_id, "name": collection_name, "hide_source_items": not visible, - "element_identifiers": json.dumps([{ + "element_identifiers": [{ "id": ld['id'], "name": element_identifer, - "src": "ldda"}]), + "src": "ldda"}], "type": "dataset_collection", "elements": [] } - new_collection = self._post(url, payload).json() + new_collection = self._post(url, payload, json=True).json() assert new_collection['name'] == collection_name assert new_collection['element_count'] == 1 element = new_collection['elements'][0] diff --git a/lib/galaxy_test/api/test_workflow_extraction.py b/lib/galaxy_test/api/test_workflow_extraction.py index 9c623b34a040..a3f0849e2a22 100644 --- a/lib/galaxy_test/api/test_workflow_extraction.py +++ b/lib/galaxy_test/api/test_workflow_extraction.py @@ -454,14 +454,14 @@ def __copy_content_to_history(self, history_id, content): source="hda", content=content["id"] ) - response = self._post(f"histories/{history_id}/contents/datasets", payload) + response = self._post(f"histories/{history_id}/contents/datasets", payload, json=True) else: payload = dict( source="hdca", content=content["id"] ) - response = self._post(f"histories/{history_id}/contents/dataset_collections", payload) + response = self._post(f"histories/{history_id}/contents/dataset_collections", payload, json=True) self._assert_status_code_is(response, 200) return response.json() diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index cc1ae2ef9947..153e7864ff51 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -2845,7 +2845,7 @@ def test_workflow_rerun_with_use_cached_job(self): new_ds_map = json.loads(new_workflow_request['ds_map']) for key, input_values in invocation_1['inputs'].items(): copy_payload = {"content": input_values['id'], "source": "hda", "type": "dataset"} - copy_response = self._post(f"histories/{history_id_two}/contents", data=copy_payload).json() + copy_response = self._post(f"histories/{history_id_two}/contents", data=copy_payload, json=True).json() new_ds_map[key]['id'] = copy_response['id'] new_workflow_request['ds_map'] = json.dumps(new_ds_map, sort_keys=True) new_workflow_request['history'] = f"hist_id={history_id_two}" @@ -2882,7 +2882,7 @@ def test_nested_workflow_rerun_with_use_cached_job(self): dataset_type = inputs['outer_input']['src'] dataset_id = inputs['outer_input']['id'] copy_payload = {"content": dataset_id, "source": dataset_type, "type": "dataset"} - copy_response = self._post(f"histories/{history_id_two}/contents", data=copy_payload) + copy_response = self._post(f"histories/{history_id_two}/contents", data=copy_payload, json=True) self._assert_status_code_is(copy_response, 200) new_dataset_id = copy_response.json()['id'] inputs['outer_input']['id'] = new_dataset_id From 91017b3fbcf303bef313eef316ad2538a87649d7 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 5 Oct 2021 17:11:06 +0200 Subject: [PATCH 197/401] Add support for nested collections in `CreateNewCollectionPayload` --- .../galaxy/services/history_contents.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 083a1e3b822d..6327a980f856 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -151,8 +151,8 @@ class CollectionElementIdentifier(Model): title="Source", description="The source of the element.", ) - id: EncodedDatabaseIdField = Field( - ..., + id: Optional[EncodedDatabaseIdField] = Field( + None, title="ID", description="The encoded ID of the element.", ) @@ -161,6 +161,21 @@ class CollectionElementIdentifier(Model): title="Tags", description="The list of tags associated with the element.", ) + element_identifiers: Optional[List['CollectionElementIdentifier']] = Field( + default=None, + title="Element Identifiers", + description="List of elements that should be in the new nested collection.", + ) + collection_type: Optional[str] = Field( + default=None, + title="Collection Type", + description="The type of the nested collection. For example, `list`, `paired`, `list:paired`.", + ) + + +# Required for self-referencing models +# See https://pydantic-docs.helpmanual.io/usage/postponed_annotations/#self-referencing-models +CollectionElementIdentifier.update_forward_refs() class CreateNewCollectionPayload(Model): From 8bfae6f3723a0765cfb16dc9057dded88c8ae6de Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 6 Oct 2021 14:09:22 +0200 Subject: [PATCH 198/401] Add FastAPI route for `update_permissions` operation - Make `set_permissions` the default action in payload --- .../webapps/galaxy/api/history_contents.py | 52 +++++++++++++------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index d1776a630a8f..77187f642bf7 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -28,7 +28,7 @@ from galaxy.schema.schema import ( AnyHistoryContentItem, AnyJobStateSummary, - DatasetPermissionAction, + DatasetAssociationRoles, HistoryContentType, UpdateDatasetPermissionsPayload, UpdateHistoryContentsBatchPayload, @@ -268,6 +268,20 @@ def parse_index_jobs_summary_params( ) +def get_update_permission_payload(payload: Dict[str, Any]) -> UpdateDatasetPermissionsPayload: + """Coverts the generic payload dictionary into a UpdateDatasetPermissionsPayload model with custom parsing. + + This is an attempt on supporting multiple aliases for the permissions params.""" + # There are several allowed names for the same role list parameter, i.e.: `access`, `access_ids`, `access_ids[]` + # The `access_ids[]` name is not pydantic friendly, so this will be modelled as an alias but we can only set one alias + # TODO: Maybe we should choose only one way/naming and deprecate the others? + payload["access_ids"] = payload.get("access_ids[]") or payload.get("access") + payload["manage_ids"] = payload.get("manage_ids[]") or payload.get("manage") + payload["modify_ids"] = payload.get("modify_ids[]") or payload.get("modify") + update_payload = UpdateDatasetPermissionsPayload(**payload) + return update_payload + + @router.cbv class FastAPIHistoryContents: service: HistoriesContentsService = depends(HistoriesContentsService) @@ -454,6 +468,25 @@ def create( payload.type = type or payload.type return self.service.create(trans, history_id, payload, serialization_params) + @router.put( + '/api/histories/{history_id}/contents/{dataset_id}/permissions', + summary='Set permissions of the given history dataset to the given role ids.', + ) + def update_permissions( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + dataset_id: EncodedDatabaseIdField = HistoryItemIDPathParam, + # Using a generic Dict here as an attempt on supporting multiple aliases for the permissions params. + payload: Dict[str, Any] = Body( + default=..., + example=UpdateDatasetPermissionsPayload(), + ), + ) -> DatasetAssociationRoles: + """Set permissions of the given history dataset to the given role ids.""" + update_payload = get_update_permission_payload(payload) + return self.service.update_permissions(trans, dataset_id, update_payload) + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): @@ -749,24 +782,9 @@ def update_permissions(self, trans, history_id, history_content_id, payload: Dic """ if payload: kwd.update(payload) - update_payload = self._get_update_permission_payload(kwd) + update_payload = get_update_permission_payload(kwd) return self.service.update_permissions(trans, history_content_id, update_payload) - def _get_update_permission_payload(self, payload: Dict[str, Any]) -> UpdateDatasetPermissionsPayload: - """Coverts the payload dictionary into a UpdateDatasetPermissionsPayload model with custom parsing. - - This is an attempt on supporting multiple aliases for the permissions params.""" - # There are several allowed names for the same role list parameter, i.e.: `access`, `access_ids`, `access_ids[]` - # The `access_ids[]` name is not pydantic friendly, so this will be modelled as an alias but we can only set one alias - # TODO: Maybe we should choose only one way and deprecate the others? - payload["access_ids[]"] = payload.get("access_ids[]") or payload.get("access") - payload["manage_ids[]"] = payload.get("manage_ids[]") or payload.get("manage") - payload["modify_ids[]"] = payload.get("modify_ids[]") or payload.get("modify") - # The action is required, so the default will be used if none is specified - payload["action"] = payload.get("action", DatasetPermissionAction.set_permissions) - update_payload = UpdateDatasetPermissionsPayload(**payload) - return update_payload - @expose_api_anonymous def update_batch(self, trans, history_id, payload, **kwd): """ From 2959d581e308da586e597b789367b616896633fb Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 6 Oct 2021 17:59:07 +0200 Subject: [PATCH 199/401] Simplify UpdateHistoryContentsBatchPayload model Also provide example in the schema. --- lib/galaxy/schema/schema.py | 66 ++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 3ee31f6ccca5..9f227f7b8896 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -31,7 +31,6 @@ from galaxy.schema.fields import ( EncodedDatabaseIdField, ModelClassField, - optional, ) from galaxy.schema.types import RelativeUrl @@ -570,12 +569,6 @@ class HDABeta(HDADetailed): # TODO: change HDABeta name to a more appropriate o pass -@optional -class UpdateHDAPayload(HDABeta): - """Used for updating a particular HDA. All fields are optional.""" - pass - - class DCSummary(Model): """Dataset Collection summary information.""" model_class: str = ModelClassField(DC_MODEL_CLASS_NAME) @@ -678,45 +671,44 @@ class HDCADetailed(HDCASummary): elements: List[DCESummary] = ElementsField -@optional -class UpdateHDCAPayload(HDCADetailed): - """Used for updating a particular HDCA. All fields are optional.""" - pass +class HistoryBase(BaseModel): + """Provides basic configuration for all the History models.""" + class Config: + use_enum_values = True # When using .dict() + extra = Extra.allow # Allow any other extra fields -class UpdateHistoryContentsBatchPayload(BaseModel): - class Config: - use_enum_values = True # when using .dict() - allow_population_by_field_name = True - extra = Extra.allow # Allow any additional field +class UpdateContentItem(HistoryBase): + """Used for updating a particular HDA. All fields are optional.""" + history_content_type: HistoryContentType = Field( + ..., + title="Content Type", + description="The type of this item.", + ) + id: EncodedDatabaseIdField = EncodedEntityIdField + - items: List[Union[UpdateHDAPayload, UpdateHDCAPayload]] = Field( +class UpdateHistoryContentsBatchPayload(HistoryBase): + """Contains property values that will be updated for all the history`items` provided.""" + + items: List[UpdateContentItem] = Field( ..., title="Items", description="A list of content items to update with the changes.", ) - deleted: Optional[bool] = Field( - default=False, - title="Deleted", - description=( - "This will check the uploading state if not deleting (i.e: deleted=False), " - "otherwise cannot delete uploading files, so it will raise an error." - ), - ) - visible: Optional[bool] = Field( - default=False, - title="Visible", - description=( - "Show or hide history contents" - ), - ) - -class HistoryBase(BaseModel): - """Provides basic configuration for all the History models.""" class Config: - use_enum_values = True # When using .dict() - extra = Extra.allow # Allow any other extra fields + schema_extra = { + "example": { + "items": [ + { + "history_content_type": "dataset", + "id": "string" + } + ], + "visible": False, + } + } class HistorySummary(HistoryBase): From 619a8e8b7344b5e6c58b16a94cd3642b06ea7904 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 6 Oct 2021 18:01:56 +0200 Subject: [PATCH 200/401] Add FastAPI route for `update_batch` operation --- .../webapps/galaxy/api/history_contents.py | 18 ++++++++++++++++++ .../galaxy/services/history_contents.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 77187f642bf7..fec168ea1a4e 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -487,6 +487,24 @@ def update_permissions( update_payload = get_update_permission_payload(payload) return self.service.update_permissions(trans, dataset_id, update_payload) + @router.put( + '/api/histories/{history_id}/contents', + summary='Batch update specific properties of a set items contained in the given History.', + ) + def update_batch( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + serialization_params: SerializationParams = Depends(query_serialization_params), + payload: UpdateHistoryContentsBatchPayload = Body(...), + ) -> List[AnyHistoryContentItem]: + """Batch update specific properties of a set items contained in the given History. + + If you provide an invalid/unknown property key the request will not fail, but no changes + will be made to the items. + """ + return self.service.update_batch(trans, history_id, payload, serialization_params) + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 6327a980f856..06406fefc8cd 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -486,7 +486,7 @@ def update_batch( history_id: EncodedDatabaseIdField, payload: UpdateHistoryContentsBatchPayload, serialization_params: SerializationParams, - ): + ) -> List[AnyHistoryContentItem]: """ PUT /api/histories/{history_id}/contents From 6fe71941141d7744c3c1e4e58c71d7e417963dad Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 7 Oct 2021 12:11:22 +0200 Subject: [PATCH 201/401] Unify put and delete requests in some API tests Replace raw requests by custom `_put` and `_delete` --- lib/galaxy_test/api/test_history_contents.py | 44 ++++++++------------ 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py index 9014a34e0a8a..604420722ba8 100644 --- a/lib/galaxy_test/api/test_history_contents.py +++ b/lib/galaxy_test/api/test_history_contents.py @@ -2,8 +2,6 @@ import urllib from datetime import datetime -from requests import delete, put - from galaxy.webapps.galaxy.services.history_contents import DirectionOptions from galaxy_test.base.populators import ( DatasetCollectionPopulator, @@ -187,24 +185,24 @@ def test_library_copy(self): def test_update(self): hda1 = self._wait_for_new_hda() assert str(hda1["deleted"]).lower() == "false" - update_response = self._raw_update(hda1["id"], dict(deleted=True)) + update_response = self._update(hda1["id"], dict(deleted=True)) self._assert_status_code_is(update_response, 200) show_response = self.__show(hda1) assert str(show_response.json()["deleted"]).lower() == "true" - update_response = self._raw_update(hda1["id"], dict(name="Updated Name")) + update_response = self._update(hda1["id"], dict(name="Updated Name")) assert self.__show(hda1).json()["name"] == "Updated Name" - update_response = self._raw_update(hda1["id"], dict(name="Updated Name")) + update_response = self._update(hda1["id"], dict(name="Updated Name")) assert self.__show(hda1).json()["name"] == "Updated Name" unicode_name = 'ржевский сапоги' - update_response = self._raw_update(hda1["id"], dict(name=unicode_name)) + update_response = self._update(hda1["id"], dict(name=unicode_name)) updated_hda = self.__show(hda1).json() assert updated_hda["name"] == unicode_name, updated_hda quoted_name = '"Mooo"' - update_response = self._raw_update(hda1["id"], dict(name=quoted_name)) + update_response = self._update(hda1["id"], dict(name=quoted_name)) updated_hda = self.__show(hda1).json() assert updated_hda["name"] == quoted_name, quoted_name @@ -230,28 +228,28 @@ def test_update_batch(self): # update deleted flag => true payload = dict(items=[{"history_content_type": "dataset", "id": hda1["id"]}], deleted=True) - update_response = self._raw_update_batch(payload) + update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is True assert objects[0]["visible"] is True # update visibility flag => false payload = dict(items=[{"history_content_type": "dataset", "id": hda1["id"]}], visible=False) - update_response = self._raw_update_batch(payload) + update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is True assert objects[0]["visible"] is False # update both flags payload = dict(items=[{"history_content_type": "dataset", "id": hda1["id"]}], deleted=False, visible=True) - update_response = self._raw_update_batch(payload) + update_response = self._update_batch(payload) objects = update_response.json() assert objects[0]["deleted"] is False assert objects[0]["visible"] is True def test_update_type_failures(self): hda1 = self._wait_for_new_hda() - update_response = self._raw_update(hda1["id"], dict(deleted='not valid')) + update_response = self._update(hda1["id"], dict(deleted='not valid')) self._assert_status_code_is(update_response, 400) def _wait_for_new_hda(self): @@ -259,27 +257,21 @@ def _wait_for_new_hda(self): self.dataset_populator.wait_for_history(self.history_id) return hda1 - def _set_edit_update(self, json): - set_edit_url = f"{self.url}/dataset/set_edit" - update_response = put(set_edit_url, json=json) + def _set_edit_update(self, data): + update_response = self._put(f"{self.url}/dataset/set_edit", data=data, json=True) return update_response - def _raw_update(self, item_id, data, admin=False, history_id=None): + def _update(self, item_id, data, admin=False, history_id=None): history_id = history_id or self.history_id - key_param = "use_admin_key" if admin else "use_key" - update_url = self._api_url(f"histories/{history_id}/contents/{item_id}", **{key_param: True}) - update_response = put(update_url, json=data) + update_response = self._put(f"histories/{history_id}/contents/{item_id}", data=data, json=True, admin=admin) return update_response def _update_permissions(self, url, data, admin=False): - key_param = "use_admin_key" if admin else "use_key" - update_url = self._api_url(url, **{key_param: True}) - update_response = put(update_url, json=data) + update_response = self._put(url, data=data, json=True, admin=admin) return update_response - def _raw_update_batch(self, data): - update_url = self._api_url(f"histories/{self.history_id}/contents", use_key=True) - update_response = put(update_url, json=data) + def _update_batch(self, data): + update_response = self._put(f"histories/{self.history_id}/contents", data=data, json=True) return update_response def test_delete(self): @@ -337,7 +329,7 @@ def test_dataset_collection_create_from_exisiting_datasets_with_new_tags(self): with self.dataset_populator.test_history() as history_id: hda_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] hda2_id = self.dataset_populator.new_dataset(history_id, content="1 2 3")['id'] - update_response = self._raw_update(hda2_id, dict(tags=['existing:tag']), history_id=history_id).json() + update_response = self._update(hda2_id, dict(tags=['existing:tag']), history_id=history_id).json() assert update_response['tags'] == ['existing:tag'] creation_payload = {'collection_type': 'list', 'history_id': history_id, @@ -389,7 +381,7 @@ def _check_pair_creation(self, endpoint, payload): assert not dataset_collection["deleted"] - delete_response = delete(self._api_url(collection_url, use_key=True)) + delete_response = self._delete(collection_url) self._assert_status_code_is(delete_response, 200) show_response = self._get(collection_url) From b69c67ab2bc7b5fbd5e69c91f54d609ccc8f6f7e Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 7 Oct 2021 12:14:49 +0200 Subject: [PATCH 202/401] Unify content type query parameter handling Use `dataset` as default value and reuse schema definition. --- .../webapps/galaxy/api/history_contents.py | 21 ++++++++----------- .../galaxy/services/history_contents.py | 9 ++++---- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index fec168ea1a4e..9d4ecd16c3be 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -87,6 +87,13 @@ description='The ID of the `HDCA` contained in the history.' ) +ContentTypeQueryParam = Query( + default=HistoryContentType.dataset, + title="Content Type", + description="The type of the history element to show.", + example=HistoryContentType.dataset, +) + def get_index_query_params( v: Optional[str] = Query( # Should this be deprecated at some point and directly use the latest version by default? @@ -335,12 +342,7 @@ def show( trans: ProvidesHistoryContext = DependsOnTrans, history_id: EncodedDatabaseIdField = HistoryIDPathParam, id: EncodedDatabaseIdField = HistoryItemIDPathParam, - type: Optional[HistoryContentType] = Query( - default=None, - title="Content Type", - description="The type of the history element to show.", - example=HistoryContentType.dataset, - ), + type: HistoryContentType = ContentTypeQueryParam, fuzzy_count: Optional[int] = Query( default=None, title="Fuzzy Count", @@ -403,12 +405,7 @@ def show_jobs_summary( trans: ProvidesHistoryContext = DependsOnTrans, history_id: EncodedDatabaseIdField = HistoryIDPathParam, id: EncodedDatabaseIdField = HistoryItemIDPathParam, - type: HistoryContentType = Path( - default=None, - title="Content Type", - description="The type of the history element to show.", - example=HistoryContentType.dataset, - ), + type: HistoryContentType = ContentTypeQueryParam, ) -> AnyJobStateSummary: """Return detailed information about an `HDA` or `HDCAs` jobs. diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 06406fefc8cd..427a92e45ce8 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -281,7 +281,7 @@ def show( trans, id: EncodedDatabaseIdField, serialization_params: SerializationParams, - contents_type: Optional[HistoryContentType], + contents_type: HistoryContentType, fuzzy_count: Optional[int] = None, ) -> AnyHistoryContentItem: """ @@ -314,7 +314,6 @@ def show( :returns: dictionary containing detailed HDA or HDCA information """ - contents_type = contents_type or HistoryContentType.dataset if contents_type == HistoryContentType.dataset: return self.__show_dataset(trans, id, serialization_params) elif contents_type == HistoryContentType.dataset_collection: @@ -348,7 +347,7 @@ def index_jobs_summary( def show_jobs_summary( self, trans, id: EncodedDatabaseIdField, - contents_type: HistoryContentType = HistoryContentType.dataset, + contents_type: HistoryContentType, ) -> AnyJobStateSummary: """ Return detailed information about an HDA or HDCAs jobs @@ -461,7 +460,7 @@ def update( id: EncodedDatabaseIdField, payload: Dict[str, Any], serialization_params: SerializationParams, - contents_type: HistoryContentType = HistoryContentType.dataset, + contents_type: HistoryContentType, ): """ Updates the values for the history content item with the given ``id`` @@ -558,7 +557,7 @@ def delete( self, trans, id, serialization_params: SerializationParams, - contents_type: HistoryContentType = HistoryContentType.dataset, + contents_type: HistoryContentType, purge: bool = False, recursive: bool = False, ) -> Union[AnyHDA, DeleteHDCAResult]: From 76eb39c5f493c16b9c003538f67d36cc9a6907cd Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 7 Oct 2021 12:15:21 +0200 Subject: [PATCH 203/401] Add FastAPI route for `update` operation --- lib/galaxy/schema/schema.py | 11 +++++++++++ .../webapps/galaxy/api/history_contents.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 9f227f7b8896..667773440ffc 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -711,6 +711,17 @@ class Config: } +class UpdateHistoryContentsPayload(HistoryBase): + """Contains arbitrary property values that will be updated for a particular history item.""" + class Config: + schema_extra = { + "example": { + "visible": False, + "annotation": "Test", + } + } + + class HistorySummary(HistoryBase): """History summary information.""" model_class: str = ModelClassField(HISTORY_MODEL_CLASS_NAME) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 9d4ecd16c3be..cb5c21fcb015 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -32,6 +32,7 @@ HistoryContentType, UpdateDatasetPermissionsPayload, UpdateHistoryContentsBatchPayload, + UpdateHistoryContentsPayload, ) from galaxy.web import ( expose_api, @@ -502,6 +503,22 @@ def update_batch( """ return self.service.update_batch(trans, history_id, payload, serialization_params) + @router.put( + '/api/histories/{history_id}/contents/{id}', + summary='Updates the values for the history content item with the given ``ID``.', + ) + def update( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + id: EncodedDatabaseIdField = HistoryItemIDPathParam, + type: HistoryContentType = ContentTypeQueryParam, + serialization_params: SerializationParams = Depends(query_serialization_params), + payload: UpdateHistoryContentsPayload = Body(...), + ) -> AnyHistoryContentItem: + """Updates the values for the history content item with the given ``ID``.""" + return self.service.update(trans, history_id, id, payload.dict(), serialization_params, contents_type=type) + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): From de7512d89f0ad76d9a2a4870c22002219b6665f0 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 7 Oct 2021 15:56:36 +0200 Subject: [PATCH 204/401] Adapt more API tests around history contents to use JSON payload --- lib/galaxy_test/api/test_datasets.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy_test/api/test_datasets.py b/lib/galaxy_test/api/test_datasets.py index fb81cbfd458c..35ec78d6bb96 100644 --- a/lib/galaxy_test/api/test_datasets.py +++ b/lib/galaxy_test/api/test_datasets.py @@ -47,7 +47,7 @@ def test_search_by_tag(self): } updated_hda = self._put( f"histories/{self.history_id}/contents/{hda_id}", - update_payload).json() + update_payload, json=True).json() assert 'cool:new_tag' in updated_hda['tags'] assert 'cool:another_tag' in updated_hda['tags'] payload = {'limit': 10, 'offset': 0, 'q': ['history_content_type', 'tag'], 'qv': ['dataset', 'cool:new_tag']} @@ -169,19 +169,19 @@ def test_update_datatype(self): update_while_incomplete_response = self._put( # try updating datatype while used as output of a running job f"histories/{self.history_id}/contents/{queued_id}", - {'datatype': 'tabular'}) + data={'datatype': 'tabular'}, json=True) self._assert_status_code_is(update_while_incomplete_response, 400) self.dataset_populator.wait_for_history_jobs(self.history_id) # now wait for upload to complete successful_updated_hda_response = self._put( f"histories/{self.history_id}/contents/{hda_id}", - {'datatype': 'tabular'}).json() + data={'datatype': 'tabular'}, json=True).json() assert successful_updated_hda_response['extension'] == 'tabular' assert successful_updated_hda_response['data_type'] == 'galaxy.datatypes.tabular.Tabular' assert 'scatterplot' in [viz['name'] for viz in successful_updated_hda_response['visualizations']] invalidly_updated_hda_response = self._put( # try updating with invalid datatype f"histories/{self.history_id}/contents/{hda_id}", - {'datatype': 'invalid'}) + data={'datatype': 'invalid'}, json=True) self._assert_status_code_is(invalidly_updated_hda_response, 400) From 9e280690fc76d8017ed97916cc51f183502a6539 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 7 Oct 2021 16:46:18 +0200 Subject: [PATCH 205/401] Fix dataset_details not being populated in `dev` version index operation Sometimes the endpoint get called with the parameter alias `details` instead of `dataset_details`. --- lib/galaxy/webapps/galaxy/api/history_contents.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index cb5c21fcb015..2d1d40311b97 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -105,6 +105,7 @@ def get_index_query_params( ), dataset_details: Optional[str] = Query( default=None, + alias="details", title="Dataset Details", description=( "A comma-separated list of encoded dataset IDs that will return additional (full) details " @@ -552,6 +553,8 @@ def index(self, trans, history_id, **kwd): """ index_params = parse_index_query_params(**kwd) legacy_params = parse_legacy_index_query_params(**kwd) + # Sometimes the `v=dev` version is called with `details` or `dataset_details` + index_params.dataset_details = index_params.dataset_details or legacy_params.dataset_details serialization_params = parse_serialization_params(**kwd) filter_parameters = FilterQueryParams(**kwd) return self.service.index( From baeb1e0dfb94f6ed6e854c92d7cb869ea8b8d6be Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 7 Oct 2021 17:25:55 +0200 Subject: [PATCH 206/401] Add FastAPI route for `validate` operation --- lib/galaxy/webapps/galaxy/api/history_contents.py | 13 +++++++++++++ .../webapps/galaxy/services/history_contents.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 2d1d40311b97..15a350526f5a 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -520,6 +520,19 @@ def update( """Updates the values for the history content item with the given ``ID``.""" return self.service.update(trans, history_id, id, payload.dict(), serialization_params, contents_type=type) + @router.put( + '/api/histories/{history_id}/contents/{id}/validate', + summary='Validates the metadata associated with a dataset within a History.', + ) + def validate( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + id: EncodedDatabaseIdField = HistoryItemIDPathParam, + ) -> dict: # TODO: define a response? + """Validates the metadata associated with a dataset within a History.""" + return self.service.validate(trans, history_id, id) + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 427a92e45ce8..b5b0fd181e20 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -534,7 +534,7 @@ def validate( history_content_id: EncodedDatabaseIdField ): """ - Updates the values for the history content item with the given ``id`` + Validates the metadata associated with a dataset within a History. :type history_id: str :param history_id: encoded id string of the items's History From 33f9794d3a367b6001f94d8d43af27d1c9a1f171 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 8 Oct 2021 14:51:38 +0200 Subject: [PATCH 207/401] Add FastAPI route for `delete` operation - Refactor payload and response model --- lib/galaxy/schema/schema.py | 47 +++++++++++----- .../webapps/galaxy/api/history_contents.py | 55 ++++++++++++++++++- .../galaxy/services/history_contents.py | 29 +++------- lib/galaxy_test/api/test_history_contents.py | 2 +- 4 files changed, 97 insertions(+), 36 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 667773440ffc..df02fc970a0b 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -2472,19 +2472,6 @@ class UpdateDatasetPermissionsPayload(Model): ) -class DeleteHDCAResult(Model): - id: EncodedDatabaseIdField = Field( - ..., - title="ID", - description="The encoded ID of the collection.", - ) - deleted: bool = Field( - ..., - title="Deleted", - description="True if the collection was successfully deleted.", - ) - - class CustomHistoryItem(Model): """Can contain any serializable property of the item. @@ -2512,6 +2499,40 @@ class Config: HistoryArchiveExportResult = Union[JobExportHistoryArchiveModel, JobIdResponse] + +class DeleteHistoryContentPayload(BaseModel): + purge: bool = Field( + default=False, + title="Purge", + description="Whether to remove from disk the target HDA or child HDAs of the target HDCA.", + ) + recursive: bool = Field( + default=False, + title="Recursive", + description="When deleting a dataset collection, whether to also delete containing datasets.", + ) + + +class DeleteHistoryContentResult(CustomHistoryItem): + """Contains minimum information about the deletion state of a history item. + + Can also contain any other properties of the item.""" + id: EncodedDatabaseIdField = Field( + ..., + title="ID", + description="The encoded ID of the history item.", + ) + deleted: bool = Field( + ..., + title="Deleted", + description="True if the item was successfully deleted.", + ) + purged: Optional[bool] = Field( + default=None, + title="Purged", + description="True if the item was successfully removed from disk.", + ) + # Sharing ----------------------------------------------------------------- diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 15a350526f5a..f99b09176659 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -29,6 +29,8 @@ AnyHistoryContentItem, AnyJobStateSummary, DatasetAssociationRoles, + DeleteHistoryContentPayload, + DeleteHistoryContentResult, HistoryContentType, UpdateDatasetPermissionsPayload, UpdateHistoryContentsBatchPayload, @@ -369,7 +371,7 @@ def show( """ Return detailed information about an `HDA` or `HDCA` within a history. - .. note:: Anonymous users are allowed to get their current history contents. + **Note**: Anonymous users are allowed to get their current history contents. """ return self.service.show( trans, @@ -533,6 +535,54 @@ def validate( """Validates the metadata associated with a dataset within a History.""" return self.service.validate(trans, history_id, id) + @router.delete( + '/api/histories/{history_id}/contents/{id}', + summary='Delete the history dataset with the given ``ID``.', + ) + @router.delete( + '/api/histories/{history_id}/contents/{type}s/{id}', + summary='Delete the history content with the given ``ID`` and specified type.', + ) + def delete( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + id: EncodedDatabaseIdField = HistoryItemIDPathParam, + type: HistoryContentType = ContentTypeQueryParam, + serialization_params: SerializationParams = Depends(query_serialization_params), + purge: Optional[bool] = Query( + default=False, + title="Purge", + description="Whether to remove from disk the target HDA or child HDAs of the target HDCA.", + deprecated=True, + ), + recursive: Optional[bool] = Query( + default=False, + title="Recursive", + description="When deleting a dataset collection, whether to also delete containing datasets.", + deprecated=True, + ), + payload: DeleteHistoryContentPayload = Body(None), + ) -> DeleteHistoryContentResult: + """ + Delete the history content with the given ``ID`` and specified type (defaults to dataset). + + **Note**: Currently does not stop any active jobs for which this dataset is an output. + """ + # TODO: should we just use the default payload and deprecate the query params? + if payload is None: + payload = DeleteHistoryContentPayload() + payload.purge = payload.purge or purge is True + payload.recursive = payload.recursive or recursive is True + rval = self.service.delete( + trans, + id=id, + serialization_params=serialization_params, + contents_type=type, + payload=payload, + ) + return rval + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): @@ -938,7 +988,8 @@ def delete(self, trans, history_id, id, purge=False, recursive=False, **kwd): # payload takes priority purge = util.string_as_bool(kwd['payload'].get('purge', purge)) recursive = util.string_as_bool(kwd['payload'].get('recursive', recursive)) - return self.service.delete(trans, id, serialization_params, contents_type, purge, recursive) + delete_payload = DeleteHistoryContentPayload(purge=purge, recursive=recursive) + return self.service.delete(trans, id, serialization_params, contents_type, delete_payload) @expose_api_raw def archive(self, trans, history_id, filename='', format='zip', dry_run=True, **kwd): diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index b5b0fd181e20..59d236e477ce 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -57,12 +57,12 @@ EncodedDatabaseIdField, ) from galaxy.schema.schema import ( - AnyHDA, AnyHistoryContentItem, AnyJobStateSummary, ColletionSourceType, DatasetAssociationRoles, - DeleteHDCAResult, + DeleteHistoryContentPayload, + DeleteHistoryContentResult, HistoryContentSource, HistoryContentType, JobSourceType, @@ -558,31 +558,20 @@ def delete( id, serialization_params: SerializationParams, contents_type: HistoryContentType, - purge: bool = False, - recursive: bool = False, - ) -> Union[AnyHDA, DeleteHDCAResult]: + payload: DeleteHistoryContentPayload, + ) -> DeleteHistoryContentResult: """ Delete the history content with the given ``id`` and specified type (defaults to dataset) .. note:: Currently does not stop any active jobs for which this dataset is an output. - - :param id: the encoded id of the history item to delete - :type recursive: bool - :param recursive: if True, and deleted an HDCA also delete containing HDAs - :type purge: bool - :param purge: if True, purge the target HDA or child HDAs of the target HDCA - - :rtype: dict - :returns: an error object if an error occurred or a dictionary containing: - * id: the encoded id of the history, - * deleted: if the history content was marked as deleted, - * purged: if the history content was purged """ if contents_type == HistoryContentType.dataset: - return self.__delete_dataset(trans, id, purge, serialization_params) + return self.__delete_dataset(trans, id, payload.purge, serialization_params) elif contents_type == HistoryContentType.dataset_collection: - self.dataset_collection_manager.delete(trans, "history", id, recursive=recursive, purge=purge) - return DeleteHDCAResult(id=id, deleted=True) + self.dataset_collection_manager.delete( + trans, "history", id, recursive=payload.recursive, purge=payload.purge + ) + return DeleteHistoryContentResult(id=id, deleted=True) else: raise exceptions.UnknownContentsType(f'Unknown contents type: {contents_type}') diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py index 604420722ba8..bf2da187685d 100644 --- a/lib/galaxy_test/api/test_history_contents.py +++ b/lib/galaxy_test/api/test_history_contents.py @@ -305,7 +305,7 @@ def test_purge(self): assert str(self.__show(hda1).json()["deleted"]).lower() == "false" assert str(self.__show(hda1).json()["purged"]).lower() == "false" data = {'purge': True} - delete_response = self._delete(f"histories/{self.history_id}/contents/{hda1['id']}", data=data) + delete_response = self._delete(f"histories/{self.history_id}/contents/{hda1['id']}", data=data, json=True) assert delete_response.status_code < 300 # Something in the 200s :). assert str(self.__show(hda1).json()["deleted"]).lower() == "true" assert str(self.__show(hda1).json()["purged"]).lower() == "true" From e4a4c7734098b91c09869d84f66227ab6b4aff70 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Oct 2021 11:14:52 +0200 Subject: [PATCH 208/401] Move direct request access outside of service function in `archive` operation This will allow to reuse the archive service function in FastAPI mode too. Also add model for dry run result. --- lib/galaxy/schema/schema.py | 7 +++++++ lib/galaxy/webapps/galaxy/api/history_contents.py | 7 ++++++- .../webapps/galaxy/services/history_contents.py | 15 ++++++--------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index df02fc970a0b..e9431dd69305 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -2533,6 +2533,13 @@ class DeleteHistoryContentResult(CustomHistoryItem): description="True if the item was successfully removed from disk.", ) + +class HistoryContentsArchiveDryRunResult(BaseModel): + """The structure of the archive for debugging. + + Contains pairs of filepath/filename.""" + __root__: List[Tuple[str, str]] + # Sharing ----------------------------------------------------------------- diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index f99b09176659..17367dd56a5a 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -31,6 +31,7 @@ DatasetAssociationRoles, DeleteHistoryContentPayload, DeleteHistoryContentResult, + HistoryContentsArchiveDryRunResult, HistoryContentType, UpdateDatasetPermissionsPayload, UpdateHistoryContentsBatchPayload, @@ -1011,7 +1012,11 @@ def archive(self, trans, history_id, filename='', format='zip', dry_run=True, ** """ dry_run = util.string_as_bool(dry_run) filter_parameters = FilterQueryParams(**kwd) - return self.service.archive(trans, history_id, filter_parameters, filename, dry_run) + archive = self.service.archive(trans, history_id, filter_parameters, filename, dry_run) + if not isinstance(archive, HistoryContentsArchiveDryRunResult): + trans.response.headers.update(archive.get_headers()) + return archive.response + return archive @expose_api_raw_anonymous def contents_near(self, trans, history_id, direction, hid, limit, **kwd): diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 59d236e477ce..ad709cf99d4c 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -63,6 +63,7 @@ DatasetAssociationRoles, DeleteHistoryContentPayload, DeleteHistoryContentResult, + HistoryContentsArchiveDryRunResult, HistoryContentSource, HistoryContentType, JobSourceType, @@ -71,7 +72,6 @@ UpdateHistoryContentsBatchPayload, ) from galaxy.security.idencoding import IdEncodingHelper -from galaxy.util.json import safe_dumps from galaxy.util.zipstream import ZipstreamWrapper from galaxy.webapps.galaxy.api.common import parse_serialization_params from galaxy.webapps.galaxy.services.base import ServiceBase @@ -579,9 +579,9 @@ def archive( self, trans, history_id: EncodedDatabaseIdField, filter_query_params: FilterQueryParams, - filename: str = '', - dry_run: bool = True, - ): + filename: Optional[str] = '', + dry_run: Optional[bool] = True, + ) -> Union[HistoryContentsArchiveDryRunResult, ZipstreamWrapper]: """ Build and return a compressed archive of the selected history contents @@ -672,8 +672,7 @@ def build_archive_files_and_paths(content, *parents): # if dry_run, return the structure as json for debugging if dry_run: - trans.response.headers['Content-Type'] = 'application/json' - return safe_dumps(paths_and_files) + return HistoryContentsArchiveDryRunResult.parse_obj(paths_and_files) # create the archive, add the dataset files, then stream the archive as a download archive = ZipstreamWrapper( @@ -683,9 +682,7 @@ def build_archive_files_and_paths(content, *parents): ) for file_path, archive_path in paths_and_files: archive.write(file_path, archive_path) - - trans.response.headers.update(archive.get_headers()) - return archive.response() + return archive def contents_near( self, trans, From 5444baa6c6af7d875cbc0269db66610bc451828d Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Oct 2021 11:15:25 +0200 Subject: [PATCH 209/401] Add FastAPI route for `archive` operation --- .../webapps/galaxy/api/history_contents.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 17367dd56a5a..5eb1c883a797 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -584,6 +584,41 @@ def delete( ) return rval + @router.get( + '/api/histories/{history_id}/contents/archive/{id}', + summary='Build and return a compressed archive of the selected history contents.', + ) + @router.get( + '/api/histories/{history_id}/contents/archive/{filename}.{format}', + summary='Build and return a compressed archive of the selected history contents.', + ) + def archive( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + filename: Optional[str] = Query( + default=None, + description="The name that the Archive will have (defaults to history name).", + ), + format: Optional[str] = Query( + default="zip", + description="Output format of the archive.", + deprecated=True, # Looks like is not really used? + ), + dry_run: Optional[bool] = Query( + default=True, + description="Whether to return the archive and file paths only (as JSON) and not an actual archive file.", + ), + filter_query_params: FilterQueryParams = Depends(get_filter_query_params), + ): + """Build and return a compressed archive of the selected history contents. + + **Note**: this is a volatile endpoint and settings and behavior may change.""" + archive = self.service.archive(trans, history_id, filter_query_params, filename, dry_run) + if isinstance(archive, HistoryContentsArchiveDryRunResult): + return archive + return StreamingResponse(archive.get_iterator(), headers=archive.get_headers(), media_type="application/zip") + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): From 29ae9d2394bcadef700bb3069f035f22fecab392 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Oct 2021 14:55:34 +0200 Subject: [PATCH 210/401] Refactor `contents_near` operation - Move direct response manipulation from service to endpoint (FastAPI compatibility) - Add models for stats and results - Expose legacy endpoint as JSON (not raw) --- lib/galaxy/schema/schema.py | 23 ++++++++++++++++ .../webapps/galaxy/api/history_contents.py | 8 +++++- .../galaxy/services/history_contents.py | 27 +++++++++---------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index e9431dd69305..814f6ec27558 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -2540,6 +2540,29 @@ class HistoryContentsArchiveDryRunResult(BaseModel): Contains pairs of filepath/filename.""" __root__: List[Tuple[str, str]] + +class ContentsNearStats(BaseModel): + matches: int + matches_up: int + matches_down: int + total_matches: int + total_matches_up: int + total_matches_down: int + max_hid: Optional[int] = None + min_hid: Optional[int] = None + history_size: str + history_empty: bool + + +class HistoryContentsResult(Model): + __root__: List[AnyHistoryContentItem] + + +class ContentsNearResult(BaseModel): + contents: HistoryContentsResult + stats: ContentsNearStats + + # Sharing ----------------------------------------------------------------- diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 5eb1c883a797..8734e0c67a95 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -1087,9 +1087,15 @@ def contents_near(self, trans, history_id, direction, hid, limit, **kwd): hid = int(hid) limit = int(limit) - return self.service.contents_near( + result = self.service.contents_near( trans, history_id, serialization_params, filter_params, direction, hid, limit, since, ) + if result is None: + trans.response.status = 204 + return + # Put stats in http headers + trans.response.headers.update(result.stats.dict()) + return result.contents # Parsing query string according to REST standards. def _parse_rest_params(self, qdict: Dict[str, Any]) -> HistoryContentsFilterList: diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index ad709cf99d4c..7e42a8129569 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -60,6 +60,8 @@ AnyHistoryContentItem, AnyJobStateSummary, ColletionSourceType, + ContentsNearResult, + ContentsNearStats, DatasetAssociationRoles, DeleteHistoryContentPayload, DeleteHistoryContentResult, @@ -693,7 +695,7 @@ def contents_near( hid: int, limit: int, since: Optional[datetime.datetime] = None, - ): + ) -> Optional[ContentsNearResult]: """ Return {limit} history items "near" the {hid}. The {direction} determines what items are selected: @@ -715,8 +717,7 @@ def contents_near( # If a timezone is provided (since.tzinfo is not None) we convert to UTC and remove tzinfo so that comparison with history.update_time is correct. since = since if since.tzinfo is None else since.astimezone(datetime.timezone.utc).replace(tzinfo=None) if history.update_time <= since: - trans.response.status = 204 - return + return None order_by_dsc = self.build_order_by(self.history_contents_manager, 'hid-dsc') order_by_asc = self.build_order_by(self.history_contents_manager, 'hid-asc') @@ -755,18 +756,14 @@ def contents_near( matches_up=len(up_matches), matches_down=len(down_matches), total_matches_up=up_total_count, total_matches_down=down_total_count) - trans.response.headers['matches'] = item_counts['matches'] - trans.response.headers['matches_up'] = item_counts['matches_up'] - trans.response.headers['matches_down'] = item_counts['matches_down'] - trans.response.headers['total_matches'] = item_counts['total_matches'] - trans.response.headers['total_matches_up'] = item_counts['total_matches_up'] - trans.response.headers['total_matches_down'] = item_counts['total_matches_down'] - trans.response.headers['max_hid'] = max_hid - trans.response.headers['min_hid'] = min_hid - trans.response.headers['history_size'] = str(history.disk_size) - trans.response.headers['history_empty'] = json.dumps(history.empty) # convert to proper bool - - return json.dumps(expanded) + stats = ContentsNearStats( + max_hid=max_hid, + min_hid=min_hid, + history_size=history.disk_size, + history_empty=history.empty, + **item_counts, + ) + return ContentsNearResult(contents=expanded, stats=stats) def _get_limits(self, limit): q, r = divmod(limit, 2) From 4681a611c4badd1d24dfa840b0b25e3d51662d63 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Oct 2021 19:52:39 +0200 Subject: [PATCH 211/401] Add FastAPI route for `contents_near` operation This endpoint handles the filter query parameters differently and the only way I could make it work in FastAPI mode is by directly parsing the query parameters from the requests. Since the name of the parameters can be anything there is no way of displaying this information in OpenAPI. --- lib/galaxy/schema/schema.py | 17 ++- .../webapps/galaxy/api/history_contents.py | 112 ++++++++++++++---- .../galaxy/services/history_contents.py | 1 - 3 files changed, 106 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 814f6ec27558..240316308069 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -1,5 +1,6 @@ """This module contains general pydantic models and common schema field annotations for them.""" +import json import re from datetime import datetime from enum import Enum @@ -2542,6 +2543,7 @@ class HistoryContentsArchiveDryRunResult(BaseModel): class ContentsNearStats(BaseModel): + """Stats used by the `contents_near` endpoint.""" matches: int matches_up: int matches_down: int @@ -2550,11 +2552,24 @@ class ContentsNearStats(BaseModel): total_matches_down: int max_hid: Optional[int] = None min_hid: Optional[int] = None - history_size: str + history_size: int history_empty: bool + def to_headers(self) -> Dict[str, str]: + """Converts all field values to json strings. + + The headers values need to be json strings or updating the response + headers will raise encoding errors.""" + headers = {} + for key, val in self: + headers[key] = json.dumps(val) + return headers + class HistoryContentsResult(Model): + """Collection of history content items. + Can contain different views and kinds of items. + """ __root__: List[AnyHistoryContentItem] diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 8734e0c67a95..d44ec0fce249 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -2,6 +2,7 @@ API operations on the contents of a history. """ import logging +from datetime import datetime from typing import ( Any, Dict, @@ -15,8 +16,10 @@ Depends, Path, Query, + Request, ) -from starlette.responses import StreamingResponse +from starlette import status +from starlette.responses import Response, StreamingResponse from galaxy import util from galaxy.managers.context import ProvidesHistoryContext @@ -32,6 +35,7 @@ DeleteHistoryContentPayload, DeleteHistoryContentResult, HistoryContentsArchiveDryRunResult, + HistoryContentsResult, HistoryContentType, UpdateDatasetPermissionsPayload, UpdateHistoryContentsBatchPayload, @@ -280,6 +284,28 @@ def parse_index_jobs_summary_params( ) +def parse_content_filter_params(params: Dict[str, Any]) -> HistoryContentsFilterList: + """Alternative way of parsing query parameter for filtering history contents. + + Parses parameters like: ?[field]-[operator]=[value] + Example: ?update_time-gt=2015-01-29 + + Currently used by the `contents_near` endpoint. + """ + DEFAULT_OP = 'eq' + splitchar = '-' + + result = [] + for key, val in params.items(): + attr = key + op = DEFAULT_OP + if splitchar in key: + attr, op = key.rsplit(splitchar, 1) + result.append([attr, op, val]) + + return result + + def get_update_permission_payload(payload: Dict[str, Any]) -> UpdateDatasetPermissionsPayload: """Coverts the generic payload dictionary into a UpdateDatasetPermissionsPayload model with custom parsing. @@ -314,14 +340,14 @@ def index( legacy_params: LegacyHistoryContentsIndexParams = Depends(get_legacy_index_query_params), serialization_params: SerializationParams = Depends(query_serialization_params), filter_query_params: FilterQueryParams = Depends(get_filter_query_params), - ) -> List[AnyHistoryContentItem]: + ): """ Return a list of `HDA`/`HDCA` data for the history with the given ``ID``. - The contents can be filtered and queried using the appropriate parameters. - The amount of information returned for each item can be customized. - .. note:: Anonymous users are allowed to get their current history contents. + **Note**: Anonymous users are allowed to get their current history contents. """ items = self.service.index( trans, @@ -499,13 +525,14 @@ def update_batch( history_id: EncodedDatabaseIdField = HistoryIDPathParam, serialization_params: SerializationParams = Depends(query_serialization_params), payload: UpdateHistoryContentsBatchPayload = Body(...), - ) -> List[AnyHistoryContentItem]: + ) -> HistoryContentsResult: """Batch update specific properties of a set items contained in the given History. If you provide an invalid/unknown property key the request will not fail, but no changes will be made to the items. """ - return self.service.update_batch(trans, history_id, payload, serialization_params) + result = self.service.update_batch(trans, history_id, payload, serialization_params) + return HistoryContentsResult.parse_obj(result) @router.put( '/api/histories/{history_id}/contents/{id}', @@ -619,6 +646,62 @@ def archive( return archive return StreamingResponse(archive.get_iterator(), headers=archive.get_headers(), media_type="application/zip") + @router.get( + '/api/histories/{history_id}/contents/near/{hid}/{limit}', + summary='Get content items around (above and below) a particular `HID`.', + ) + def contents_near( + self, + request: Request, + response: Response, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + hid: int = Path( + ..., + title="Target HID", + description="The target `HID` to get content around it.", + ), + limit: int = Path( + ..., + description="The maximum number of content items to return above and below the target `HID`.", + ), + since: Optional[datetime] = Query( + default=None, + description=( + "A timestamp in ISO format to check if the history has changed since this particular date/time. " + "If it hasn't changed, no additional processing will be done and 204 status code will be returned." + ), + ), + serialization_params: SerializationParams = Depends(query_serialization_params), + ) -> HistoryContentsResult: + """ + This endpoint provides random access to a large history without having + to know exactly how many pages are in the final query. Pick a target HID + and filters, and the endpoint will get LIMIT counts above and below that + target regardless of how many gaps may exist in the HID due to + filtering. + + It does 2 queries, one up and one down from the target hid with a + result size of limit. Additional counts for total matches of both seeks + provided in the http headers. + + **Note**: This endpoint uses slightly different filter params syntax. Instead of using `q`/`qv` parameters + it uses the following syntax for query parameters: + ?[field]-[operator]=[value] + Example: + ?update_time-gt=2015-01-29 + """ + serialization_params.default_view = serialization_params.default_view or "betawebclient" + # Needed to parse arbitrary query parameter names + filter_params = parse_content_filter_params(request.query_params) + result = self.service.contents_near( + trans, history_id, serialization_params, filter_params, hid, limit, since, + ) + if result is None: + return Response(status_code=status.HTTP_204_NO_CONTENT) + response.headers.update(result.stats.to_headers()) + return result.contents + class HistoryContentsController(BaseGalaxyAPIController, UsesLibraryMixinItems, UsesTagsMixin): @@ -1083,7 +1166,7 @@ def contents_near(self, trans, history_id, direction, hid, limit, **kwd): else: since = None - filter_params = self._parse_rest_params(kwd) + filter_params = parse_content_filter_params(kwd) hid = int(hid) limit = int(limit) @@ -1094,20 +1177,5 @@ def contents_near(self, trans, history_id, direction, hid, limit, **kwd): trans.response.status = 204 return # Put stats in http headers - trans.response.headers.update(result.stats.dict()) + trans.response.headers.update(result.stats.to_headers()) return result.contents - - # Parsing query string according to REST standards. - def _parse_rest_params(self, qdict: Dict[str, Any]) -> HistoryContentsFilterList: - DEFAULT_OP = 'eq' - splitchar = '-' - - result = [] - for key, val in qdict.items(): - attr = key - op = DEFAULT_OP - if splitchar in key: - attr, op = key.rsplit(splitchar, 1) - result.append([attr, op, val]) - - return result diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 7e42a8129569..8348a55e2773 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -1,5 +1,4 @@ import datetime -import json import logging import os import re From bef34df74bff6d192bed3ec13bbea074da479cd2 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Oct 2021 20:59:40 +0200 Subject: [PATCH 212/401] Fix mypy I wonder why `tox -e mypy` does not catch this... maybe uses a different version? --- lib/galaxy/webapps/galaxy/api/history_contents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index d44ec0fce249..e5396ff73988 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -693,7 +693,7 @@ def contents_near( """ serialization_params.default_view = serialization_params.default_view or "betawebclient" # Needed to parse arbitrary query parameter names - filter_params = parse_content_filter_params(request.query_params) + filter_params = parse_content_filter_params(request.query_params._dict) result = self.service.contents_near( trans, history_id, serialization_params, filter_params, hid, limit, since, ) From 60675c5519d19b0dd9f90745fbafd30693868ff7 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Oct 2021 13:43:43 +0200 Subject: [PATCH 213/401] Exclude known query params from `contents_near` filter params Since we are directly accessing the request's query_params we also need to exclude the known params that are already parsed by FastAPI or they may be treated as filter params too. --- .../webapps/galaxy/api/history_contents.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index e5396ff73988..e9844f636700 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -8,6 +8,7 @@ Dict, List, Optional, + Set, ) import dateutil.parser @@ -284,19 +285,26 @@ def parse_index_jobs_summary_params( ) -def parse_content_filter_params(params: Dict[str, Any]) -> HistoryContentsFilterList: +def parse_content_filter_params( + params: Dict[str, Any], + exclude: Optional[Set[str]] = None, +) -> HistoryContentsFilterList: """Alternative way of parsing query parameter for filtering history contents. Parses parameters like: ?[field]-[operator]=[value] Example: ?update_time-gt=2015-01-29 - Currently used by the `contents_near` endpoint. + Currently used by the `contents_near` endpoint. The `exclude` set can contain + names of parameters that will be ignored and not added to the filters. """ DEFAULT_OP = 'eq' splitchar = '-' + exclude = exclude or set() result = [] for key, val in params.items(): + if key in exclude: + continue attr = key op = DEFAULT_OP if splitchar in key: @@ -692,8 +700,15 @@ def contents_near( ?update_time-gt=2015-01-29 """ serialization_params.default_view = serialization_params.default_view or "betawebclient" - # Needed to parse arbitrary query parameter names - filter_params = parse_content_filter_params(request.query_params._dict) + + # Needed to parse arbitrary query parameter names for the filters. + # Since we are directly accessing the request's query_params we also need to exclude the + # known params that are already parsed by FastAPI or they may be treated as filter params too. + # This looks a bit hacky... + exclude_params = set(["since"]) + exclude_params.update(SerializationParams.__fields__.keys()) + filter_params = parse_content_filter_params(request.query_params._dict, exclude=exclude_params) + result = self.service.contents_near( trans, history_id, serialization_params, filter_params, hid, limit, since, ) From 255135b54f9381df5d490a2f0b5442e531f70d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20L=C3=B3pez?= <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Oct 2021 09:39:26 +0200 Subject: [PATCH 214/401] Apply suggestions from code review Co-authored-by: Marius van den Beek --- lib/galaxy/schema/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 240316308069..312bc34ee2c8 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -690,7 +690,7 @@ class UpdateContentItem(HistoryBase): class UpdateHistoryContentsBatchPayload(HistoryBase): - """Contains property values that will be updated for all the history`items` provided.""" + """Contains property values that will be updated for all the history `items` provided.""" items: List[UpdateContentItem] = Field( ..., @@ -2505,7 +2505,7 @@ class DeleteHistoryContentPayload(BaseModel): purge: bool = Field( default=False, title="Purge", - description="Whether to remove from disk the target HDA or child HDAs of the target HDCA.", + description="Whether to remove the dataset from storage. Datasets will only be removed from storage once all HDAs or LDDAs that refer to this datasets are deleted.", ) recursive: bool = Field( default=False, From d29e606753f9861c47b1b1e87e914b346775df7f Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Oct 2021 10:04:55 +0200 Subject: [PATCH 215/401] Update HistoryContentsArchiveDryRunResult docstring --- lib/galaxy/schema/schema.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 312bc34ee2c8..b3ac9c78d5a1 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -2536,9 +2536,14 @@ class DeleteHistoryContentResult(CustomHistoryItem): class HistoryContentsArchiveDryRunResult(BaseModel): - """The structure of the archive for debugging. + """ + Contains a collection of filepath/filename entries that represent + the contents that would have been included in the archive. + This is returned when the `dry_run` flag is active when + creating an archive with the contents of the history. - Contains pairs of filepath/filename.""" + This is used for debugging purposes. + """ __root__: List[Tuple[str, str]] From 1b237c3296b5d5220d157141226fc7a8e2f4d8c6 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 14 Oct 2021 10:55:38 +0200 Subject: [PATCH 216/401] Fix `archive` operation to support `upstream_mod_zip` --- lib/galaxy/webapps/galaxy/api/history_contents.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index e9844f636700..d03db2d20ce9 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -652,6 +652,8 @@ def archive( archive = self.service.archive(trans, history_id, filter_query_params, filename, dry_run) if isinstance(archive, HistoryContentsArchiveDryRunResult): return archive + if archive.upstream_mod_zip: + return StreamingResponse(archive.response(), headers=archive.get_headers()) return StreamingResponse(archive.get_iterator(), headers=archive.get_headers(), media_type="application/zip") @router.get( @@ -1148,7 +1150,7 @@ def archive(self, trans, history_id, filename='', format='zip', dry_run=True, ** archive = self.service.archive(trans, history_id, filter_parameters, filename, dry_run) if not isinstance(archive, HistoryContentsArchiveDryRunResult): trans.response.headers.update(archive.get_headers()) - return archive.response + return archive.response() return archive @expose_api_raw_anonymous From 6fa9bff4709f95927ff2b261cc33b9215957eec2 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 21 Oct 2021 12:25:24 +0200 Subject: [PATCH 217/401] Add fallback to legacy url_for in UrlBuilder --- lib/galaxy/webapps/galaxy/api/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index f74ce2e0d09d..eb0cb3c96f07 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -22,6 +22,7 @@ from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter from pydantic.main import BaseModel +from starlette.routing import NoMatchFound try: from starlette_context import context as request_context except ImportError: @@ -31,6 +32,7 @@ from galaxy import ( app as galaxy_app, model, + web, ) from galaxy.exceptions import ( AdminRequiredException, @@ -147,9 +149,13 @@ def __init__(self, request: Request): def __call__(self, name: str, **path_params): qualified = path_params.pop("qualified", False) - if qualified: - return self.request.url_for(name, **path_params) - return self.request.app.url_path_for(name, **path_params) + try: + if qualified: + return self.request.url_for(name, **path_params) + return self.request.app.url_path_for(name, **path_params) + except NoMatchFound: + # Fallback to legacy url_for + return web.url_for(name, **path_params) class GalaxyASGIRequest(GalaxyAbstractRequest): From a9dde649a608ad11c177fac12aa33cb77f9121ae Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 21 Oct 2021 13:37:18 +0200 Subject: [PATCH 218/401] Add names to routes for generating url matching --- lib/galaxy/webapps/galaxy/api/history_contents.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index d03db2d20ce9..972c7e239427 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -334,6 +334,7 @@ class FastAPIHistoryContents: @router.get( '/api/histories/{history_id}/contents', + name='history_contents', summary='Returns the contents of the given history.', ) @router.get( @@ -369,11 +370,13 @@ def index( @router.get( '/api/histories/{history_id}/contents/{id}', + name='history_content', summary='Return detailed information about an HDA within a history.', deprecated=True, ) @router.get( '/api/histories/{history_id}/contents/{type}s/{id}', + name='history_content_typed', summary='Return detailed information about a specific HDA or HDCA with the given `ID` within a history.', ) def show( From d31dab80ab04e158620fc04e8383817f61600800 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 21 Oct 2021 13:37:54 +0200 Subject: [PATCH 219/401] Override URL serialization using UrlBuilder --- .../webapps/galaxy/services/history_contents.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 8348a55e2773..3225b426a064 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -1018,9 +1018,21 @@ def _serialize_content_item( if serializer is None: raise exceptions.UnknownContentsType(f'Unknown contents type: {content.content_type}') - return serializer.serialize_to_view( + rval = serializer.serialize_to_view( content, user=trans.user, trans=trans, view=view, **serialization_params_dict ) + # Override URL generation to use UrlBuilder + if trans.url_builder: + if rval.get("url"): + rval["url"] = trans.url_builder('history_content_typed', + history_id=rval["history_id"], id=rval["id"], type=rval["history_content_type"] + ) + if rval.get("contents_url"): + rval["contents_url"] = trans.url_builder('contents_dataset_collection', + hdca_id=rval["id"], + parent_id=self.encode_id(content.collection_id) + ) + return rval def __collection_dict(self, trans, dataset_collection_instance, **kwds): return dictify_dataset_collection_instance(dataset_collection_instance, From 08bf998714c03a0cff6cf41c24799fcd715c8e5a Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 29 Oct 2021 16:27:46 +0200 Subject: [PATCH 220/401] Fix merge error The type_id should be optional --- lib/galaxy/schema/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index b3ac9c78d5a1..30353b193fb7 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -361,8 +361,8 @@ class HistoryItemCommon(HistoryItemBase): class Config: extra = Extra.allow - type_id: str = Field( - ..., + type_id: Optional[str] = Field( + default=None, title="Type - ID", description="The type and the encoded ID of this item. Used for caching.", example="dataset-616e371b2cc6c62e", From 1a47a737e6138a9e8f3512630dfea585038411d1 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 29 Oct 2021 16:28:27 +0200 Subject: [PATCH 221/401] Fix `contents_url` generation for collections --- lib/galaxy/managers/hdcas.py | 4 +++- lib/galaxy/webapps/galaxy/api/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/managers/hdcas.py b/lib/galaxy/managers/hdcas.py index 0c21479bafc4..062f0a1e69dc 100644 --- a/lib/galaxy/managers/hdcas.py +++ b/lib/galaxy/managers/hdcas.py @@ -320,7 +320,9 @@ def add_serializers(self): def generate_contents_url(self, item, key, **context): encode_id = self.app.security.encode_id - contents_url = self.url_for('contents_dataset_collection', + trans = context.get("trans") + url_for = trans.url_builder if trans and trans.url_builder else self.url_for + contents_url = url_for('contents_dataset_collection', hdca_id=encode_id(item.id), parent_id=encode_id(item.collection_id)) return contents_url diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index eb0cb3c96f07..4709dae0699d 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -16,6 +16,7 @@ Form, Header, Query, + Request, Response, ) from fastapi.params import Depends @@ -27,7 +28,6 @@ from starlette_context import context as request_context except ImportError: request_context = None -from starlette.requests import Request from galaxy import ( app as galaxy_app, From ba013aebd59358c0e7f099d7f6e8581aba3c3d3e Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 31 Oct 2021 23:38:56 +0100 Subject: [PATCH 222/401] Add missing alternative update route --- lib/galaxy/webapps/galaxy/api/history_contents.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 972c7e239427..5600d320c39a 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -463,11 +463,11 @@ def show_jobs_summary( summary='Download the content of a dataset collection as a `zip` archive.', response_class=StreamingResponse, ) - @router.get( # TODO: Move to dataset_collections API? + @router.get( '/api/dataset_collection/{id}/download', summary='Download the content of a dataset collection as a `zip` archive.', response_class=StreamingResponse, - tags=["dataset_collections"], + tags=["dataset collections"], ) def download_dataset_collection( self, @@ -545,9 +545,14 @@ def update_batch( result = self.service.update_batch(trans, history_id, payload, serialization_params) return HistoryContentsResult.parse_obj(result) + @router.put( + '/api/histories/{history_id}/contents/{type}s/{id}', + summary='Updates the values for the history content item with the given ``ID``.', + ) @router.put( '/api/histories/{history_id}/contents/{id}', summary='Updates the values for the history content item with the given ``ID``.', + deprecated=True, ) def update( self, From 3a126bee9c426d9c0965a4a7483fe3f008197113 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 1 Nov 2021 01:20:01 +0100 Subject: [PATCH 223/401] Reorder routes to avoid masking The order of the routes declaration is important. The route with the most specific path must be declared before the more general one. --- .../webapps/galaxy/api/history_contents.py | 92 +++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 5600d320c39a..451898ef1518 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -332,15 +332,15 @@ def get_update_permission_payload(payload: Dict[str, Any]) -> UpdateDatasetPermi class FastAPIHistoryContents: service: HistoriesContentsService = depends(HistoriesContentsService) + @router.get( + '/api/histories/{history_id}/contents/{type}s', + summary='Returns the contents of the given history filtered by type.', + ) @router.get( '/api/histories/{history_id}/contents', name='history_contents', summary='Returns the contents of the given history.', ) - @router.get( - '/api/histories/{history_id}/contents/{type}s', - summary='Returns the contents of the given history filtered by type.', - ) def index( self, trans: ProvidesHistoryContext = DependsOnTrans, @@ -369,16 +369,36 @@ def index( return items @router.get( - '/api/histories/{history_id}/contents/{id}', - name='history_content', - summary='Return detailed information about an HDA within a history.', - deprecated=True, + '/api/histories/{history_id}/contents/{type}s/{id}/jobs_summary', + summary='Return detailed information about an `HDA` or `HDCAs` jobs.', ) + def show_jobs_summary( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + id: EncodedDatabaseIdField = HistoryItemIDPathParam, + type: HistoryContentType = ContentTypeQueryParam, + ) -> AnyJobStateSummary: + """Return detailed information about an `HDA` or `HDCAs` jobs. + + **Warning**: We allow anyone to fetch job state information about any object they + can guess an encoded ID for - it isn't considered protected data. This keeps + polling IDs as part of state calculation for large histories and collections as + efficient as possible. + """ + return self.service.show_jobs_summary(trans, id, contents_type=type) + @router.get( '/api/histories/{history_id}/contents/{type}s/{id}', name='history_content_typed', summary='Return detailed information about a specific HDA or HDCA with the given `ID` within a history.', ) + @router.get( + '/api/histories/{history_id}/contents/{id}', + name='history_content', + summary='Return detailed information about an HDA within a history.', + deprecated=True, + ) def show( self, trans: ProvidesHistoryContext = DependsOnTrans, @@ -438,26 +458,6 @@ def index_jobs_summary( """ return self.service.index_jobs_summary(trans, params) - @router.get( - '/api/histories/{history_id}/contents/{type}s/{id}/jobs_summary', - summary='Return detailed information about an `HDA` or `HDCAs` jobs.', - ) - def show_jobs_summary( - self, - trans: ProvidesHistoryContext = DependsOnTrans, - history_id: EncodedDatabaseIdField = HistoryIDPathParam, - id: EncodedDatabaseIdField = HistoryItemIDPathParam, - type: HistoryContentType = ContentTypeQueryParam, - ) -> AnyJobStateSummary: - """Return detailed information about an `HDA` or `HDCAs` jobs. - - **Warning**: We allow anyone to fetch job state information about any object they - can guess an encoded ID for - it isn't considered protected data. This keeps - polling IDs as part of state calculation for large histories and collections as - efficient as possible. - """ - return self.service.show_jobs_summary(trans, id, contents_type=type) - @router.get( '/api/histories/{history_id}/contents/dataset_collections/{id}/download', summary='Download the content of a dataset collection as a `zip` archive.', @@ -545,6 +545,19 @@ def update_batch( result = self.service.update_batch(trans, history_id, payload, serialization_params) return HistoryContentsResult.parse_obj(result) + @router.put( + '/api/histories/{history_id}/contents/{id}/validate', + summary='Validates the metadata associated with a dataset within a History.', + ) + def validate( + self, + trans: ProvidesHistoryContext = DependsOnTrans, + history_id: EncodedDatabaseIdField = HistoryIDPathParam, + id: EncodedDatabaseIdField = HistoryItemIDPathParam, + ) -> dict: # TODO: define a response? + """Validates the metadata associated with a dataset within a History.""" + return self.service.validate(trans, history_id, id) + @router.put( '/api/histories/{history_id}/contents/{type}s/{id}', summary='Updates the values for the history content item with the given ``ID``.', @@ -566,27 +579,14 @@ def update( """Updates the values for the history content item with the given ``ID``.""" return self.service.update(trans, history_id, id, payload.dict(), serialization_params, contents_type=type) - @router.put( - '/api/histories/{history_id}/contents/{id}/validate', - summary='Validates the metadata associated with a dataset within a History.', + @router.delete( + '/api/histories/{history_id}/contents/{type}s/{id}', + summary='Delete the history content with the given ``ID`` and specified type.', ) - def validate( - self, - trans: ProvidesHistoryContext = DependsOnTrans, - history_id: EncodedDatabaseIdField = HistoryIDPathParam, - id: EncodedDatabaseIdField = HistoryItemIDPathParam, - ) -> dict: # TODO: define a response? - """Validates the metadata associated with a dataset within a History.""" - return self.service.validate(trans, history_id, id) - @router.delete( '/api/histories/{history_id}/contents/{id}', summary='Delete the history dataset with the given ``ID``.', ) - @router.delete( - '/api/histories/{history_id}/contents/{type}s/{id}', - summary='Delete the history content with the given ``ID`` and specified type.', - ) def delete( self, trans: ProvidesHistoryContext = DependsOnTrans, @@ -628,11 +628,11 @@ def delete( return rval @router.get( - '/api/histories/{history_id}/contents/archive/{id}', + '/api/histories/{history_id}/contents/archive/{filename}.{format}', summary='Build and return a compressed archive of the selected history contents.', ) @router.get( - '/api/histories/{history_id}/contents/archive/{filename}.{format}', + '/api/histories/{history_id}/contents/archive/{id}', summary='Build and return a compressed archive of the selected history contents.', ) def archive( From c69f72150b41da8b7a188a29db4933148b6ba2e0 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 1 Nov 2021 02:53:12 +0100 Subject: [PATCH 224/401] Fix response model issues Enabling this flag will avoid returning fields with default values. The `type_id` field was not returned by the services dictionary response but the pydantic model was adding it with the default value of None. This was causing the client to behave in a strange way when editing the name of a collection, making a POST request to create a new collection instead of making a PUT request with the name change. --- lib/galaxy/webapps/galaxy/api/history_contents.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 451898ef1518..128d856e7c48 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -392,11 +392,13 @@ def show_jobs_summary( '/api/histories/{history_id}/contents/{type}s/{id}', name='history_content_typed', summary='Return detailed information about a specific HDA or HDCA with the given `ID` within a history.', + response_model_exclude_unset=True, ) @router.get( '/api/histories/{history_id}/contents/{id}', name='history_content', summary='Return detailed information about an HDA within a history.', + response_model_exclude_unset=True, deprecated=True, ) def show( @@ -484,10 +486,12 @@ def download_dataset_collection( @router.post( '/api/histories/{history_id}/contents/{type}s', summary='Create a new `HDA` or `HDCA` in the given History.', + response_model_exclude_unset=True, ) @router.post( '/api/histories/{history_id}/contents', summary='Create a new `HDA` or `HDCA` in the given History.', + response_model_exclude_unset=True, deprecated=True, ) def create( @@ -561,10 +565,12 @@ def validate( @router.put( '/api/histories/{history_id}/contents/{type}s/{id}', summary='Updates the values for the history content item with the given ``ID``.', + response_model_exclude_unset=True, ) @router.put( '/api/histories/{history_id}/contents/{id}', summary='Updates the values for the history content item with the given ``ID``.', + response_model_exclude_unset=True, deprecated=True, ) def update( From 13aebb313aed5a350ac826f2e88c9b6769b228d0 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 1 Nov 2021 16:00:01 +0100 Subject: [PATCH 225/401] Inject UrlBuilder when serializing dataset collection instances --- lib/galaxy/managers/collections_util.py | 10 +++++----- lib/galaxy/webapps/galaxy/api/library_contents.py | 4 +++- lib/galaxy/webapps/galaxy/api/tools.py | 8 ++++++-- .../webapps/galaxy/services/dataset_collections.py | 3 ++- lib/galaxy/webapps/galaxy/services/history_contents.py | 9 +++++++-- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/managers/collections_util.py b/lib/galaxy/managers/collections_util.py index 84e2a2ca39e7..12b047063438 100644 --- a/lib/galaxy/managers/collections_util.py +++ b/lib/galaxy/managers/collections_util.py @@ -1,7 +1,7 @@ import logging import math -from galaxy import exceptions, model, web +from galaxy import exceptions, model from galaxy.util import string_as_bool log = logging.getLogger(__name__) @@ -103,20 +103,20 @@ def get_collection_elements(collection, name=""): return names, hdas -def dictify_dataset_collection_instance(dataset_collection_instance, parent, security, view="element", fuzzy_count=None): +def dictify_dataset_collection_instance(dataset_collection_instance, parent, security, url_builder, view="element", fuzzy_count=None): hdca_view = "element" if view in ["element", "element-reference"] else "collection" dict_value = dataset_collection_instance.to_dict(view=hdca_view) encoded_id = security.encode_id(dataset_collection_instance.id) if isinstance(parent, model.History): encoded_history_id = security.encode_id(parent.id) - dict_value['url'] = web.url_for('history_content_typed', history_id=encoded_history_id, id=encoded_id, type="dataset_collection") + dict_value['url'] = url_builder('history_content_typed', history_id=encoded_history_id, id=encoded_id, type="dataset_collection") elif isinstance(parent, model.LibraryFolder): encoded_library_id = security.encode_id(parent.library_root.id) encoded_folder_id = security.encode_id(parent.id) # TODO: Work in progress - this end-point is not right yet... - dict_value['url'] = web.url_for('library_content', library_id=encoded_library_id, id=encoded_id, folder_id=encoded_folder_id) + dict_value['url'] = url_builder('library_content', library_id=encoded_library_id, id=encoded_id, folder_id=encoded_folder_id) - dict_value['contents_url'] = web.url_for( + dict_value['contents_url'] = url_builder( 'contents_dataset_collection', hdca_id=encoded_id, parent_id=security.encode_id(dataset_collection_instance.collection_id) diff --git a/lib/galaxy/webapps/galaxy/api/library_contents.py b/lib/galaxy/webapps/galaxy/api/library_contents.py index f9c4b0924499..af8a68c92ee7 100644 --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -263,7 +263,9 @@ def create(self, trans, library_id, payload, **kwd): create_params['parent'] = parent dataset_collection_manager = trans.app.dataset_collection_manager dataset_collection_instance = dataset_collection_manager.create(**create_params) - return [dictify_dataset_collection_instance(dataset_collection_instance, security=trans.security, parent=parent)] + return [dictify_dataset_collection_instance( + dataset_collection_instance, security=trans.security, url_builder=trans.url_builder, parent=parent + )] if status != 200: trans.response.status = status return output diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 594a3a964c04..47fe01229e49 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -672,13 +672,17 @@ def _handle_inputs_output_to_api_response(self, trans, tool, target_history, var for output_name, collection_instance in vars.get('output_collections', []): history = target_history or trans.history - output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history) + output_dict = dictify_dataset_collection_instance( + collection_instance, security=trans.security, url_builder=trans.url_builder, parent=history, + ) output_dict['output_name'] = output_name rval['output_collections'].append(output_dict) for output_name, collection_instance in vars.get('implicit_collections', {}).items(): history = target_history or trans.history - output_dict = dictify_dataset_collection_instance(collection_instance, security=trans.security, parent=history) + output_dict = dictify_dataset_collection_instance( + collection_instance, security=trans.security, url_builder=trans.url_builder, parent=history, + ) output_dict['output_name'] = output_name rval['implicit_collections'].append(output_dict) diff --git a/lib/galaxy/webapps/galaxy/services/dataset_collections.py b/lib/galaxy/webapps/galaxy/services/dataset_collections.py index acf07110c007..2e38a52d0f21 100644 --- a/lib/galaxy/webapps/galaxy/services/dataset_collections.py +++ b/lib/galaxy/webapps/galaxy/services/dataset_collections.py @@ -136,7 +136,7 @@ def create(self, trans: ProvidesHistoryContext, payload: CreateNewCollectionPayl dataset_collection_instance = self.collection_manager.create(trans=trans, **create_params) rval = dictify_dataset_collection_instance( - dataset_collection_instance, security=trans.security, parent=create_params["parent"] + dataset_collection_instance, security=trans.security, url_builder=trans.url_builder, parent=create_params["parent"] ) return rval @@ -203,6 +203,7 @@ def show( rval = dictify_dataset_collection_instance( dataset_collection_instance, security=trans.security, + url_builder=trans.url_builder, parent=parent, view='element' ) diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py index 3225b426a064..4d70c9e58697 100644 --- a/lib/galaxy/webapps/galaxy/services/history_contents.py +++ b/lib/galaxy/webapps/galaxy/services/history_contents.py @@ -1035,8 +1035,13 @@ def _serialize_content_item( return rval def __collection_dict(self, trans, dataset_collection_instance, **kwds): - return dictify_dataset_collection_instance(dataset_collection_instance, - security=trans.security, parent=dataset_collection_instance.history, **kwds) + return dictify_dataset_collection_instance( + dataset_collection_instance, + security=trans.security, + url_builder=trans.url_builder, + parent=dataset_collection_instance.history, + **kwds + ) def _get_history(self, trans, history_id: EncodedDatabaseIdField) -> History: """Retrieves the History with the given ID or raises an error if the current user cannot access it.""" From 62a163559ea77fde226190f3068cd048b0674398 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 1 Nov 2021 17:02:57 +0100 Subject: [PATCH 226/401] Remove test case for datetime format not supported by `datetime.fromisoformat` --- lib/galaxy_test/api/test_history_contents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py index bf2da187685d..86c1a8a8af5f 100644 --- a/lib/galaxy_test/api/test_history_contents.py +++ b/lib/galaxy_test/api/test_history_contents.py @@ -676,7 +676,7 @@ def test_history_contents_near_since_with_standard_iso8601_date(self): assert history_contents.status_code == 204 # test parsing for other standard is08601 formats - sample_formats = ['2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '20210826T155302Z', '2002-10-10T12:00:00-05:00'] + sample_formats = ['2021-08-26T15:53:02+00:00', '2021-08-26T15:53:02Z', '2002-10-10T12:00:00-05:00'] for date_str in sample_formats: encoded_date = urllib.parse.quote_plus(date_str) # handles pluses, minuses history_contents = self._get(f"/api/histories/{history_id}/contents/near/100/100?since={encoded_date}") From 6f8a2e12b82d917f2b1d258b2e9cbe87fe831f77 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 2 Nov 2021 11:44:14 +0100 Subject: [PATCH 227/401] Force explicit JSON payload in tests When deleting a dataset through the history contents API --- lib/galaxy_test/base/populators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index dcb71f320cb0..008c27356b10 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -576,7 +576,7 @@ def delete_history(self, history_id: str) -> None: delete_response.raise_for_status() def delete_dataset(self, history_id: str, content_id: str, purge: bool = False) -> Response: - delete_response = self._delete(f"histories/{history_id}/contents/{content_id}", {'purge': purge}) + delete_response = self._delete(f"histories/{history_id}/contents/{content_id}", {'purge': purge}, json=True) return delete_response def create_tool_from_path(self, tool_path: str) -> Dict[str, Any]: From b899fa38b1a360b7494534e256703d5a7535ea95 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 12 Nov 2021 19:32:13 +0100 Subject: [PATCH 228/401] Adapt contents_near endpoint to use new direction parameter --- .../webapps/galaxy/api/history_contents.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 128d856e7c48..48c0b06d4620 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -671,7 +671,7 @@ def archive( return StreamingResponse(archive.get_iterator(), headers=archive.get_headers(), media_type="application/zip") @router.get( - '/api/histories/{history_id}/contents/near/{hid}/{limit}', + '/api/histories/{history_id}/contents/{direction}/{hid}/{limit}', summary='Get content items around (above and below) a particular `HID`.', ) def contents_near( @@ -685,6 +685,10 @@ def contents_near( title="Target HID", description="The target `HID` to get content around it.", ), + direction: DirectionOptions = Path( + ..., + description="Determines what items are selected before, after or near the target `hid`.", + ), limit: int = Path( ..., description="The maximum number of content items to return above and below the target `HID`.", @@ -701,13 +705,13 @@ def contents_near( """ This endpoint provides random access to a large history without having to know exactly how many pages are in the final query. Pick a target HID - and filters, and the endpoint will get LIMIT counts above and below that - target regardless of how many gaps may exist in the HID due to - filtering. + and filters, and the endpoint will get a maximum of `limit` history items "near" the `hid`. + The `direction` determines what items are selected: + - before: select items with hid < {hid} + - after: select items with hid > {hid} + - near: select items "around" {hid}, so that |before| <= limit // 2, |after| <= limit // 2 + 1 - It does 2 queries, one up and one down from the target hid with a - result size of limit. Additional counts for total matches of both seeks - provided in the http headers. + Additional counts are provided in the HTTP headers. **Note**: This endpoint uses slightly different filter params syntax. Instead of using `q`/`qv` parameters it uses the following syntax for query parameters: @@ -726,7 +730,7 @@ def contents_near( filter_params = parse_content_filter_params(request.query_params._dict, exclude=exclude_params) result = self.service.contents_near( - trans, history_id, serialization_params, filter_params, hid, limit, since, + trans, history_id, serialization_params, filter_params, direction, hid, limit, since, ) if result is None: return Response(status_code=status.HTTP_204_NO_CONTENT) From 9c63a611255fe65e3d10af75339f77bb02261303 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Nov 2021 11:34:14 +0100 Subject: [PATCH 229/401] Adapt changes from #12914 --- lib/galaxy/schema/schema.py | 3 ++- .../webapps/galaxy/api/history_contents.py | 26 ++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index 30353b193fb7..d20c5d15e272 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -2544,7 +2544,8 @@ class HistoryContentsArchiveDryRunResult(BaseModel): This is used for debugging purposes. """ - __root__: List[Tuple[str, str]] + # TODO: Use Tuple again when https://github.com/tiangolo/fastapi/issues/3665 is fixed upstream + __root__: List[List[str]] # List[Tuple[str, str]] class ContentsNearStats(BaseModel): diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 48c0b06d4620..90c2ee51fafe 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -60,6 +60,7 @@ from galaxy.webapps.galaxy.services.history_contents import ( CreateHistoryContentPayload, DatasetDetailsType, + DirectionOptions, HistoriesContentsService, HistoryContentsFilterList, HistoryContentsIndexJobsSummaryParams, @@ -672,7 +673,7 @@ def archive( @router.get( '/api/histories/{history_id}/contents/{direction}/{hid}/{limit}', - summary='Get content items around (above and below) a particular `HID`.', + summary='Get content items around a particular `HID`.', ) def contents_near( self, @@ -703,16 +704,29 @@ def contents_near( serialization_params: SerializationParams = Depends(query_serialization_params), ) -> HistoryContentsResult: """ + **Warning**: For internal use to support the scroller functionality. + This endpoint provides random access to a large history without having to know exactly how many pages are in the final query. Pick a target HID - and filters, and the endpoint will get a maximum of `limit` history items "near" the `hid`. - The `direction` determines what items are selected: - - before: select items with hid < {hid} - - after: select items with hid > {hid} - - near: select items "around" {hid}, so that |before| <= limit // 2, |after| <= limit // 2 + 1 + and filters, and the endpoint will get a maximum of `limit` history items "around" the `hid`. Additional counts are provided in the HTTP headers. + The `direction` determines what items are selected: + + a) item counts: + - total matches-up: hid < {hid} + - total matches-down: hid > {hid} + - total matches: total matches-up + total matches-down + 1 (+1 for hid == {hid}) + - displayed matches-up: hid <= {hid} (hid == {hid} is included) + - displayed matches-down: hid > {hid} + - displayed matches: displayed matches-up + displayed matches-down + + b) {limit} history items: + - if direction == before: hid <= {hid} + - if direction == after: hid > {hid} + - if direction == near: "near" {hid}, so that |before| <= limit // 2, |after| <= limit // 2 + 1. + **Note**: This endpoint uses slightly different filter params syntax. Instead of using `q`/`qv` parameters it uses the following syntax for query parameters: ?[field]-[operator]=[value] From b0dbec87232498228990d0f85aa240367d0e8431 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Nov 2021 11:42:05 +0100 Subject: [PATCH 230/401] Fix mypy --- lib/galaxy/webapps/galaxy/api/history_contents.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index 90c2ee51fafe..e9445b4ec156 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -220,8 +220,9 @@ def parse_legacy_index_query_params( else: content_types = [e.value for e in HistoryContentType] + id_list: Optional[List[EncodedDatabaseIdField]] = None if ids: - ids = util.listify(ids) + id_list = util.listify(ids) # If explicit ids given, always used detailed result. dataset_details = 'all' else: @@ -229,7 +230,7 @@ def parse_legacy_index_query_params( return LegacyHistoryContentsIndexParams( types=content_types, - ids=ids, + ids=id_list, deleted=deleted, visible=visible, dataset_details=dataset_details, From 8067c9af376d36126a15b7c4a5cf5898d35c8621 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Nov 2021 11:52:49 +0100 Subject: [PATCH 231/401] Remove duplicated code --- lib/galaxy/webapps/galaxy/api/history_contents.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index e9445b4ec156..e54ef911c7e3 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -54,6 +54,7 @@ ) from galaxy.webapps.galaxy.api.common import ( get_filter_query_params, + get_update_permission_payload, parse_serialization_params, query_serialization_params, ) @@ -316,20 +317,6 @@ def parse_content_filter_params( return result -def get_update_permission_payload(payload: Dict[str, Any]) -> UpdateDatasetPermissionsPayload: - """Coverts the generic payload dictionary into a UpdateDatasetPermissionsPayload model with custom parsing. - - This is an attempt on supporting multiple aliases for the permissions params.""" - # There are several allowed names for the same role list parameter, i.e.: `access`, `access_ids`, `access_ids[]` - # The `access_ids[]` name is not pydantic friendly, so this will be modelled as an alias but we can only set one alias - # TODO: Maybe we should choose only one way/naming and deprecate the others? - payload["access_ids"] = payload.get("access_ids[]") or payload.get("access") - payload["manage_ids"] = payload.get("manage_ids[]") or payload.get("manage") - payload["modify_ids"] = payload.get("modify_ids[]") or payload.get("modify") - update_payload = UpdateDatasetPermissionsPayload(**payload) - return update_payload - - @router.cbv class FastAPIHistoryContents: service: HistoriesContentsService = depends(HistoriesContentsService) From f31a008059f360a8e3d06d63d80ee25ba25c8923 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 19 Nov 2021 14:38:17 +0100 Subject: [PATCH 232/401] Remove raw API decorators from legacy endpoints The return pydantic models/dicts now, not raw JSON --- lib/galaxy/webapps/galaxy/api/history_contents.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index e54ef911c7e3..a9c9ff61277a 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -45,7 +45,6 @@ from galaxy.web import ( expose_api, expose_api_anonymous, - expose_api_raw, expose_api_raw_anonymous, ) from galaxy.webapps.base.controller import ( @@ -1147,7 +1146,7 @@ def delete(self, trans, history_id, id, purge=False, recursive=False, **kwd): delete_payload = DeleteHistoryContentPayload(purge=purge, recursive=recursive) return self.service.delete(trans, id, serialization_params, contents_type, delete_payload) - @expose_api_raw + @expose_api def archive(self, trans, history_id, filename='', format='zip', dry_run=True, **kwd): """ archive( self, trans, history_id, filename='', format='zip', dry_run=True, **kwd ) @@ -1173,7 +1172,7 @@ def archive(self, trans, history_id, filename='', format='zip', dry_run=True, ** return archive.response() return archive - @expose_api_raw_anonymous + @expose_api_anonymous def contents_near(self, trans, history_id, direction, hid, limit, **kwd): """ Returns the following data: From b4a29f2600bbf9bbd5586e802fcf41c9052f4afa Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 13 Dec 2021 12:49:47 +0100 Subject: [PATCH 233/401] Only check profile version in Galaxy Should fix https://github.com/galaxyproject/galaxy/issues/13008 --- lib/galaxy/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 6425690fe212..4cfc8fa87547 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -772,7 +772,7 @@ def parse(self, tool_source, guid=None, dynamic=False): raise Exception("Missing tool 'id' for tool at '%s'" % tool_source) profile = packaging.version.parse(str(self.profile)) - if profile >= packaging.version.parse("16.04") and packaging.version.parse(VERSION_MAJOR) < profile: + if self.app.name == 'galaxy' and profile >= packaging.version.parse("16.04") and packaging.version.parse(VERSION_MAJOR) < profile: template = "The tool %s targets version %s of Galaxy, you should upgrade Galaxy to ensure proper functioning of this tool." message = template % (self.id, self.profile) raise Exception(message) From 77835edde45fe35412045c0fd7a53bd5c39ffef4 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Thu, 18 Nov 2021 16:56:05 +0100 Subject: [PATCH 234/401] delete legacy code, unify library breadcrumb --- client/src/bundleEntries.js | 5 - .../LibraryFolder/LibraryBreadcrumb.vue | 35 + .../Libraries/LibraryFolder/LibraryFolder.vue | 5 +- .../LibraryFolderDataset/LibraryDataset.vue | 74 ++ .../LibraryFolder/TopToolbar/FolderTopBar.vue | 25 +- .../Libraries/LibraryFolder/services.js | 11 + ...ibraryFolderRouter.js => LibraryRouter.js} | 9 +- client/src/components/Libraries/index.js | 2 +- client/src/galaxy.library.js | 67 -- .../src/mvc/library/library-dataset-view.js | 963 ------------------ 10 files changed, 139 insertions(+), 1057 deletions(-) create mode 100644 client/src/components/Libraries/LibraryFolder/LibraryBreadcrumb.vue create mode 100644 client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue rename client/src/components/Libraries/{LibraryFolderRouter.js => LibraryRouter.js} (84%) delete mode 100644 client/src/galaxy.library.js delete mode 100644 client/src/mvc/library/library-dataset-view.js diff --git a/client/src/bundleEntries.js b/client/src/bundleEntries.js index 42ba563d8153..26240b1a5ee3 100644 --- a/client/src/bundleEntries.js +++ b/client/src/bundleEntries.js @@ -17,7 +17,6 @@ export { TracksterUI } from "viz/trackster"; import Circster from "viz/circster"; export { PhylovizView as phyloviz } from "viz/phyloviz"; export { SweepsterVisualization, SweepsterVisualizationView } from "viz/sweepster"; -import GalaxyLibrary from "galaxy.library"; export { createTabularDatasetChunkedView } from "mvc/dataset/data"; import { HistoryCollection } from "mvc/history/history-model"; export { History } from "mvc/history/history-model"; @@ -44,10 +43,6 @@ export function circster(options) { new Circster.GalaxyApp(options); } -export function library(options) { - new GalaxyLibrary.GalaxyApp(options); -} - export function multiHistory(options) { const histories = new HistoryCollection([], { includeDeleted: options.includingDeleted, diff --git a/client/src/components/Libraries/LibraryFolder/LibraryBreadcrumb.vue b/client/src/components/Libraries/LibraryFolder/LibraryBreadcrumb.vue new file mode 100644 index 000000000000..e86f458327c1 --- /dev/null +++ b/client/src/components/Libraries/LibraryFolder/LibraryBreadcrumb.vue @@ -0,0 +1,35 @@ + + diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue index 709c951e7ac5..bd557e30f273 100644 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue @@ -89,9 +89,10 @@ :to="{ name: `LibraryFolder`, params: { folder_id: `${row.item.id}` } }" >{{ row.item.name }} - {{ + + {{ row.item.name - }} + }}
diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue new file mode 100644 index 000000000000..45e8f20ce49d --- /dev/null +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue @@ -0,0 +1,74 @@ + + + diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue index d71aca032d86..20a412b15938 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/FolderTopBar.vue @@ -118,21 +118,11 @@
- - - Libraries - - - + diff --git a/client/src/components/Libraries/LibraryFolder/services.js b/client/src/components/Libraries/LibraryFolder/services.js index 8df6bcf91bb4..5637b6b3af68 100644 --- a/client/src/components/Libraries/LibraryFolder/services.js +++ b/client/src/components/Libraries/LibraryFolder/services.js @@ -110,4 +110,17 @@ export class Services { rethrowSimple(e); } } + async detectDatatype(datasetID, data, onSucess, onError) { + const url = `${this.root}api/libraries/datasets/${datasetID}`; + try { + await axios + .patch(url, data) + .then(() => onSucess()) + .catch((error) => { + onError(error.err_msg); + }); + } catch (e) { + rethrowSimple(e); + } + } } From ddc77e9374e911807f8f174001454a3b18f85749 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Fri, 19 Nov 2021 17:37:13 +0100 Subject: [PATCH 236/401] add table itself --- .../LibraryFolderDataset/LibraryDataset.vue | 104 +++++++++++++----- .../LibraryFolderDataset/fields.js | 12 ++ .../import-to-history/import-dataset.js | 2 + 3 files changed, 90 insertions(+), 28 deletions(-) create mode 100644 client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/fields.js diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue index b0fe01fd67a5..dcf3775f7be5 100644 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue @@ -1,32 +1,41 @@ @@ -43,14 +62,22 @@ import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome"; import { faUsers, faRedo, faPencilAlt, faBook, faDownload } from "@fortawesome/free-solid-svg-icons"; import { library } from "@fortawesome/fontawesome-svg-core"; +import mod_import_dataset from "components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-dataset"; import { Services } from "components/Libraries/LibraryFolder/services"; import LibraryBreadcrumb from "components/Libraries/LibraryFolder/LibraryBreadcrumb"; import download from "components/Libraries/LibraryFolder/TopToolbar/download"; import CopyToClipboard from "components/CopyToClipboard"; import { Toast } from "ui/toast"; +import { fieldsTitles } from "./fields"; library.add(faUsers, faRedo, faBook, faDownload, faPencilAlt); + +function buildFields(fieldTitles, data) { + return Object.entries(fieldTitles).flatMap(([property, title]) => + data[property] ? { name: title, value: data[property] } : [] + ); +} export default { props: { dataset_id: { @@ -69,16 +96,29 @@ export default { }, data() { return { + fieldTitles: fieldsTitles, dataset: undefined, currentRouteName: window.location.href, datasetDownloadFormat: "uncompressed", download: download, + table_items: [], + isEditMode: false, + fields: [ + { + key: "name", + }, + { key: "value" }, + ], }; }, created() { this.services = new Services({ root: this.root }); this.services.getDataset(this.dataset_id).then((response) => { this.dataset = response; + this.table_items = buildFields(this.fieldTitles, this.dataset); + + console.log(this.table_items); + console.log(this.dataset); }); }, methods: { @@ -91,6 +131,14 @@ export default { (error) => Toast.success(error) ); }, + importToHistory() { + new mod_import_dataset.ImportDatasetModal({ + selected: { dataset_ids: [this.dataset_id] }, + }); + }, + debug(e) { + console.log(e); + }, }, }; diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/fields.js b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/fields.js new file mode 100644 index 000000000000..1891e0b5899f --- /dev/null +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/fields.js @@ -0,0 +1,12 @@ +export const fieldsTitles = { + name: "Name", + data_type: "Data type", + genome_build: "Genome build", + size: "Size", + update_time: "Date last updated (UTC)", + date_uploaded: "Date uploaded (UTC)", + uploaded_by: "Uploaded by", + misc_blurb: "Misc. blurb", + misc_info: "Misc. info", + uuid: "UUID", +}; diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-dataset.js b/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-dataset.js index f9d17e703e7b..9c4582a9cfd7 100644 --- a/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-dataset.js +++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/import-to-history/import-dataset.js @@ -108,6 +108,8 @@ var ImportDatasetModal = Backbone.View.extend({ historyItem.source = "library"; items_to_import.push(historyItem); } + + checked_items.folder_ids = checked_items.folder_ids ? checked_items.folder_ids : []; // prepare the folder objects to be imported for (let i = checked_items.folder_ids.length - 1; i >= 0; i--) { const library_folder_id = checked_items.folder_ids[i]; From 7b7ed955010486afdbc6c88522ca351776936cc4 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Fri, 19 Nov 2021 20:10:18 +0100 Subject: [PATCH 237/401] implement save and type select --- client/src/components/CopyToClipboard.vue | 2 +- .../LibraryFolderDataset/LibraryDataset.vue | 93 ++++++++++++++----- .../LibraryFolderDataset/constants.js | 29 ++++++ .../LibraryFolderDataset/fields.js | 12 --- .../Libraries/LibraryFolder/services.js | 14 ++- 5 files changed, 112 insertions(+), 38 deletions(-) create mode 100644 client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/constants.js delete mode 100644 client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/fields.js diff --git a/client/src/components/CopyToClipboard.vue b/client/src/components/CopyToClipboard.vue index 1ceacf2d0eab..37dbaeee1edb 100644 --- a/client/src/components/CopyToClipboard.vue +++ b/client/src/components/CopyToClipboard.vue @@ -1,5 +1,5 @@ diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/constants.js b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/constants.js new file mode 100644 index 000000000000..5938974cd275 --- /dev/null +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/constants.js @@ -0,0 +1,29 @@ +export const fieldsTitles = { + name: "Name", + file_ext: "Data type", + genome_build: "Genome build", + size: "Size", + update_time: "Date last updated (UTC)", + date_uploaded: "Date uploaded (UTC)", + uploaded_by: "Uploaded by", + metadata_data_lines: "Data Lines", + metadata_comment_lines: "Comment Lines", + metadata_columns: "Number of Columns", + metadata_column_types: "Column Types", + message: "Message", + misc_blurb: "Misc. blurb", + misc_info: "Misc. info", + tags: "Tags", + uuid: "UUID", +}; +export const auto = { + id: "auto", + text: "Auto-detect", + description: + "This system will try to detect the file type automatically." + + " If your file is not detected properly as one of the known formats," + + " it most likely means that it has some format problems (e.g., different" + + " number of columns on different rows). You can still coerce the system" + + " to set your data to the format you think it should be." + + " You can also upload compressed files, which will automatically be decompressed.", +}; diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/fields.js b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/fields.js deleted file mode 100644 index 1891e0b5899f..000000000000 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/fields.js +++ /dev/null @@ -1,12 +0,0 @@ -export const fieldsTitles = { - name: "Name", - data_type: "Data type", - genome_build: "Genome build", - size: "Size", - update_time: "Date last updated (UTC)", - date_uploaded: "Date uploaded (UTC)", - uploaded_by: "Uploaded by", - misc_blurb: "Misc. blurb", - misc_info: "Misc. info", - uuid: "UUID", -}; diff --git a/client/src/components/Libraries/LibraryFolder/services.js b/client/src/components/Libraries/LibraryFolder/services.js index 5637b6b3af68..81f9b744e8cc 100644 --- a/client/src/components/Libraries/LibraryFolder/services.js +++ b/client/src/components/Libraries/LibraryFolder/services.js @@ -1,6 +1,7 @@ import axios from "axios"; import { rethrowSimple } from "utils/simple-error"; import { getAppRoot } from "onload/loadConfig"; +import { auto } from "components/Libraries/LibraryFolder/LibraryFolderDataset/constants"; export class Services { constructor(options = {}) { @@ -110,12 +111,12 @@ export class Services { rethrowSimple(e); } } - async detectDatatype(datasetID, data, onSucess, onError) { + async updateDataset(datasetID, data, onSucess, onError) { const url = `${this.root}api/libraries/datasets/${datasetID}`; try { await axios .patch(url, data) - .then(() => onSucess()) + .then((response) => onSucess(response.data)) .catch((error) => { onError(error.err_msg); }); @@ -123,4 +124,13 @@ export class Services { rethrowSimple(e); } } + async getDatatypes() { + const url = `${this.root}api/datatypes?extension_only=True`; + try { + const response = await axios.get(url); + return response.data; + } catch (e) { + rethrowSimple(e); + } + } } From 512b6de1dd6f4807762f577a950cc1c2026ec9b0 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 22 Nov 2021 14:30:44 +0100 Subject: [PATCH 238/401] Allow autodetect-type only if user can modify --- .../LibraryFolderDataset/LibraryDataset.vue | 21 ++++++++++++------- .../Libraries/LibraryFolder/services.js | 3 +-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue index 7633ba0bc189..ee273cd9b937 100644 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue @@ -3,23 +3,28 @@
- - - - + + Auto-detect datatype diff --git a/client/src/components/Libraries/LibraryFolder/services.js b/client/src/components/Libraries/LibraryFolder/services.js index 81f9b744e8cc..109e844feb5c 100644 --- a/client/src/components/Libraries/LibraryFolder/services.js +++ b/client/src/components/Libraries/LibraryFolder/services.js @@ -1,7 +1,6 @@ import axios from "axios"; -import { rethrowSimple } from "utils/simple-error"; import { getAppRoot } from "onload/loadConfig"; -import { auto } from "components/Libraries/LibraryFolder/LibraryFolderDataset/constants"; +import { rethrowSimple } from "utils/simple-error"; export class Services { constructor(options = {}) { From c76c21b215e10fd071cbae46ecbd876de43a49f3 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 23 Nov 2021 12:21:03 +0100 Subject: [PATCH 239/401] Add the rest of editable fields to dataset details table --- .../LibraryFolderDataset/LibraryDataset.vue | 138 +++++++++++------- .../LibraryFolderDataset/constants.js | 8 +- .../Libraries/LibraryFolder/services.js | 17 +++ 3 files changed, 106 insertions(+), 57 deletions(-) diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue index ee273cd9b937..2a799a978a89 100644 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue @@ -2,7 +2,7 @@
- + Download @@ -33,7 +33,7 @@ v-if="dataset.can_user_manage" class="mr-1 mb-2" :to="{ - name: `LibraryFolderDatasetPermissions`, + name: 'LibraryFolderDatasetPermissions', params: { folder_id: folder_id, dataset_id: dataset_id }, }" > @@ -49,40 +49,79 @@ title="Copy link to this dataset " />
- + + - + +
+ + + Cancel + + + + Save + +
+
- - - - Cancel - - - - Save -
diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/constants.js b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/constants.js index 5938974cd275..ab27ed6de4a2 100644 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/constants.js +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/constants.js @@ -1,8 +1,8 @@ -export const fieldsTitles = { +export const fieldTitles = { name: "Name", file_ext: "Data type", genome_build: "Genome build", - size: "Size", + file_size: "Size", update_time: "Date last updated (UTC)", date_uploaded: "Date uploaded (UTC)", uploaded_by: "Uploaded by", @@ -11,8 +11,8 @@ export const fieldsTitles = { metadata_columns: "Number of Columns", metadata_column_types: "Column Types", message: "Message", - misc_blurb: "Misc. blurb", - misc_info: "Misc. info", + misc_blurb: "Miscellaneous blurb", + misc_info: "Miscellaneous information", tags: "Tags", uuid: "UUID", }; diff --git a/client/src/components/Libraries/LibraryFolder/services.js b/client/src/components/Libraries/LibraryFolder/services.js index 109e844feb5c..ca83371f6193 100644 --- a/client/src/components/Libraries/LibraryFolder/services.js +++ b/client/src/components/Libraries/LibraryFolder/services.js @@ -132,4 +132,21 @@ export class Services { rethrowSimple(e); } } + async getGenomes() { + const url = `${this.root}api/genomes`; + try { + const response = await axios.get(url); + const genomes = response.data; + const listGenomes = []; + for (var key in genomes) { + listGenomes.push({ + id: genomes[key][1], + text: genomes[key][0], + }); + } + return listGenomes; + } catch (e) { + rethrowSimple(e); + } + } } From 17e505ce39767a7ee7d19875740d6785dd49e23e Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 23 Nov 2021 14:49:35 +0100 Subject: [PATCH 240/401] Add GenomeSelector renderer and use GenomeProvider To edit genome_build property in dataset details. --- .../LibraryFolderDataset/GenomeSelector.vue | 52 +++++++++++++++++++ .../LibraryFolderDataset/LibraryDataset.vue | 34 ++++++------ .../Libraries/LibraryFolder/services.js | 17 ------ 3 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/GenomeSelector.vue diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/GenomeSelector.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/GenomeSelector.vue new file mode 100644 index 000000000000..95794c2162b9 --- /dev/null +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/GenomeSelector.vue @@ -0,0 +1,52 @@ + + + diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue index 2a799a978a89..208615900c15 100644 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue @@ -77,16 +77,15 @@ :searchable="true" :allow-empty="false" /> - + + + genome.id == this.dataset.genome_build); }, methods: { populateDatasetDetailsTable(data) { @@ -203,9 +201,6 @@ export default { ); }, updateDataset() { - if (this.selectedGenome.id !== this.dataset.genome_build) { - this.modifiedDataset.genome_build = this.selectedGenome.id; - } //TODO send only diff? this.services.updateDataset( this.dataset_id, @@ -223,6 +218,9 @@ export default { selected: { dataset_ids: [this.dataset_id] }, }); }, + onSelectedGenome(genome) { + this.modifiedDataset.genome_build = genome.id; + }, }, }; diff --git a/client/src/components/Libraries/LibraryFolder/services.js b/client/src/components/Libraries/LibraryFolder/services.js index ca83371f6193..109e844feb5c 100644 --- a/client/src/components/Libraries/LibraryFolder/services.js +++ b/client/src/components/Libraries/LibraryFolder/services.js @@ -132,21 +132,4 @@ export class Services { rethrowSimple(e); } } - async getGenomes() { - const url = `${this.root}api/genomes`; - try { - const response = await axios.get(url); - const genomes = response.data; - const listGenomes = []; - for (var key in genomes) { - listGenomes.push({ - id: genomes[key][1], - text: genomes[key][0], - }); - } - return listGenomes; - } catch (e) { - rethrowSimple(e); - } - } } From ad947569fe812e51091d9b95db00c9da4842728e Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 23 Nov 2021 14:57:27 +0100 Subject: [PATCH 241/401] Fix prettier --- .../Libraries/LibraryFolder/LibraryFolder.vue | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue index bd557e30f273..6980f474ec6e 100644 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue @@ -90,9 +90,14 @@ >{{ row.item.name }} - {{ - row.item.name - }} + {{ row.item.name }}
From 8d6b12a2dddf12fd70a97f8951647329f59202f1 Mon Sep 17 00:00:00 2001 From: oleg Date: Tue, 23 Nov 2021 18:31:32 +0100 Subject: [PATCH 242/401] fix types --- .../LibraryFolderDataset/LibraryDataset.vue | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue index 208615900c15..0fa17ab6cf52 100644 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/LibraryDataset.vue @@ -4,7 +4,8 @@ - Download + + Download @@ -70,12 +71,13 @@ v-model="modifiedDataset.name" /> From e467cca39b7591cf8fc26726b56c8094d599ba5f Mon Sep 17 00:00:00 2001 From: oleg Date: Tue, 23 Nov 2021 18:51:31 +0100 Subject: [PATCH 243/401] make multiselect style global --- client/src/components/Sharing/Sharing.vue | 19 ------------------- client/src/style/scss/base.scss | 1 + 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/client/src/components/Sharing/Sharing.vue b/client/src/components/Sharing/Sharing.vue index d6645e08e0c4..e91e094d0954 100644 --- a/client/src/components/Sharing/Sharing.vue +++ b/client/src/components/Sharing/Sharing.vue @@ -92,7 +92,6 @@ -
+
This dataset is unrestricted so everybody with the link can access it. diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/cannotManageDataset.json b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/cannotManageDataset.json new file mode 100644 index 000000000000..d41180476373 --- /dev/null +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/cannotManageDataset.json @@ -0,0 +1,34 @@ +{ + "id": "cannot_manage_dataset_id", + "ldda_id": "cannot_manage_dataset_id", + "parent_library_id": "test_library_id", + "folder_id": "test_folder_id", + "model_class": "LibraryDataset", + "state": "ok", + "name": "dataset.xml", + "file_name": "/some/path/galaxy/database/objects/9/7/3/dataset_973c9701-36d6-4b5e-8af1-a085f902a583.dat", + "created_from_basename": null, + "uploaded_by": "test@galaxy.user", + "message": null, + "date_uploaded": "2021-11-22 04:43 PM", + "update_time": "2021-11-25 01:21 PM", + "file_size": "19.6 KB", + "file_ext": "xml", + "data_type": "galaxy.datatypes.xml.GenericXml", + "genome_build": "?", + "misc_info": "test xml", + "misc_blurb": "XML data", + "peek": "Test peek contents", + "uuid": "973c9701-36d6-4b5e-8af1-a085f902a583", + "metadata_dbkey": "?", + "metadata_data_lines": 380, + "full_path": [ + ["test_folder_id", "Folder"], + ["cannot_manage_dataset_id", "dataset.xml"] + ], + "tags": "", + "deleted": false, + "is_unrestricted": true, + "can_user_modify": true, + "can_user_manage": false +} diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/cannotModifyDataset.json b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/cannotModifyDataset.json new file mode 100644 index 000000000000..a694960b2eb3 --- /dev/null +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/cannotModifyDataset.json @@ -0,0 +1,34 @@ +{ + "id": "cannot_modify_dataset_id", + "ldda_id": "cannot_modify_dataset_id", + "parent_library_id": "test_library_id", + "folder_id": "test_folder_id", + "model_class": "LibraryDataset", + "state": "ok", + "name": "dataset.xml", + "file_name": "/some/path/galaxy/database/objects/9/7/3/dataset_973c9701-36d6-4b5e-8af1-a085f902a583.dat", + "created_from_basename": null, + "uploaded_by": "test@galaxy.user", + "message": null, + "date_uploaded": "2021-11-22 04:43 PM", + "update_time": "2021-11-25 01:21 PM", + "file_size": "19.6 KB", + "file_ext": "xml", + "data_type": "galaxy.datatypes.xml.GenericXml", + "genome_build": "?", + "misc_info": "test xml", + "misc_blurb": "XML data", + "peek": "Test peek contents", + "uuid": "973c9701-36d6-4b5e-8af1-a085f902a583", + "metadata_dbkey": "?", + "metadata_data_lines": 380, + "full_path": [ + ["test_folder_id", "Folder"], + ["cannot_modify_dataset_id", "dataset.xml"] + ], + "tags": "", + "deleted": false, + "is_unrestricted": true, + "can_user_modify": false, + "can_user_manage": true +} diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/restrictedDataset.json b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/restrictedDataset.json new file mode 100644 index 000000000000..02c10253a91f --- /dev/null +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/restrictedDataset.json @@ -0,0 +1,34 @@ +{ + "id": "restricted_dataset_id", + "ldda_id": "restricted_dataset_id", + "parent_library_id": "test_library_id", + "folder_id": "test_folder_id", + "model_class": "LibraryDataset", + "state": "ok", + "name": "dataset.xml", + "file_name": "/some/path/galaxy/database/objects/9/7/3/dataset_973c9701-36d6-4b5e-8af1-a085f902a583.dat", + "created_from_basename": null, + "uploaded_by": "test@galaxy.user", + "message": null, + "date_uploaded": "2021-11-22 04:43 PM", + "update_time": "2021-11-25 01:21 PM", + "file_size": "19.6 KB", + "file_ext": "xml", + "data_type": "galaxy.datatypes.xml.GenericXml", + "genome_build": "?", + "misc_info": "test xml", + "misc_blurb": "XML data", + "peek": "Test peek contents", + "uuid": "973c9701-36d6-4b5e-8af1-a085f902a583", + "metadata_dbkey": "?", + "metadata_data_lines": 380, + "full_path": [ + ["test_folder_id", "Folder"], + ["restricted_dataset_id", "dataset.xml"] + ], + "tags": "", + "deleted": false, + "is_unrestricted": false, + "can_user_modify": true, + "can_user_manage": true +} diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/unrestrictedDataset.json b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/unrestrictedDataset.json new file mode 100644 index 000000000000..42a727b78ea3 --- /dev/null +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolderDataset/testData/unrestrictedDataset.json @@ -0,0 +1,34 @@ +{ + "id": "unrestricted_dataset_id", + "ldda_id": "unrestricted_dataset_id", + "parent_library_id": "test_library_id", + "folder_id": "test_folder_id", + "model_class": "LibraryDataset", + "state": "ok", + "name": "dataset.xml", + "file_name": "/some/path/galaxy/database/objects/9/7/3/dataset_973c9701-36d6-4b5e-8af1-a085f902a583.dat", + "created_from_basename": null, + "uploaded_by": "test@galaxy.user", + "message": null, + "date_uploaded": "2021-11-22 04:43 PM", + "update_time": "2021-11-25 01:21 PM", + "file_size": "19.6 KB", + "file_ext": "xml", + "data_type": "galaxy.datatypes.xml.GenericXml", + "genome_build": "?", + "misc_info": "test xml", + "misc_blurb": "XML data", + "peek": "Test peek contents", + "uuid": "973c9701-36d6-4b5e-8af1-a085f902a583", + "metadata_dbkey": "?", + "metadata_data_lines": 380, + "full_path": [ + ["test_folder_id", "Folder"], + ["unrestricted_dataset_id", "dataset.xml"] + ], + "tags": "", + "deleted": false, + "is_unrestricted": true, + "can_user_modify": true, + "can_user_manage": true +} From c0134f97d683122da3406d992b474d58f2459be0 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 13 Dec 2021 12:58:25 +0100 Subject: [PATCH 255/401] Fix broken library datasets links - Some links to library elements were not replaced with the new library router paths. - Fix undeleted dataset link in toast message. --- .../Libraries/LibraryFolder/LibraryFolder.vue | 12 ++++++------ .../LibraryFolderDatasetPermissions.vue | 2 +- config/plugins/webhooks/demo/search/script.js | 2 +- .../galaxy/controllers/_create_history_template.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue index 6980f474ec6e..d7f210906178 100644 --- a/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue +++ b/client/src/components/Libraries/LibraryFolder/LibraryFolder.vue @@ -217,7 +217,7 @@ Manage
@@ -80,6 +78,7 @@ import { DatasetCollection } from "../../model/DatasetCollection"; import StatusIcon from "../../StatusIcon"; import JobStateProgress from "./JobStateProgress"; +import DscDescription from "./DscDescription"; import DscMenu from "./DscMenu"; import { Nametag } from "components/Nametags"; import IconButton from "components/IconButton"; @@ -91,6 +90,7 @@ export default { DscMenu, Nametag, IconButton, + DscDescription, }, props: { dsc: { type: DatasetCollection, required: true }, diff --git a/client/src/components/History/CurrentCollection/Details.vue b/client/src/components/History/CurrentCollection/Details.vue index 7286171641c7..e1cfdea2ffce 100644 --- a/client/src/components/History/CurrentCollection/Details.vue +++ b/client/src/components/History/CurrentCollection/Details.vue @@ -8,8 +8,7 @@

- a {{ dsc.collectionType | localize }} - {{ dsc.collectionCount | localize }} +

@@ -45,12 +44,14 @@ import { DatasetCollection } from "../model"; import { Nametag } from "components/Nametags"; import EditorMenu from "../EditorMenu"; import { StatelessTags } from "components/Tags"; +import DscDescription from "components/History/ContentItem/DatasetCollection/DscDescription"; export default { components: { Nametag, EditorMenu, StatelessTags, + DscDescription, }, props: { dsc: { type: DatasetCollection, required: true }, From 839bb01650b05806bed2981e5a442a775d96a76f Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 16 Dec 2021 18:38:41 +0100 Subject: [PATCH 335/401] Remove unused collectionCountDescription getter --- client/src/components/History/model/DatasetCollection.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/client/src/components/History/model/DatasetCollection.js b/client/src/components/History/model/DatasetCollection.js index a0995ab092fa..8a17d7084a2a 100644 --- a/client/src/components/History/model/DatasetCollection.js +++ b/client/src/components/History/model/DatasetCollection.js @@ -27,12 +27,6 @@ export class DatasetCollection extends Content { return undefined; } - // text for UI - get collectionCountDescription() { - const ct = this.totalElements; - return ct == 1 ? "with 1 item" : `with ${ct} items`; - } - // text for UI get collectionType() { if (this.collection_type) { From a7a1579cfb0c641e642de1f3f9c9027c1395a1bd Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 16 Dec 2021 19:18:57 +0100 Subject: [PATCH 336/401] Run prettier --- .../History/ContentItem/DatasetCollection/DscDescription.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscDescription.vue b/client/src/components/History/ContentItem/DatasetCollection/DscDescription.vue index e39f9add447a..1c4ee5ebae92 100644 --- a/client/src/components/History/ContentItem/DatasetCollection/DscDescription.vue +++ b/client/src/components/History/ContentItem/DatasetCollection/DscDescription.vue @@ -23,4 +23,4 @@ export default { }, }, }; - \ No newline at end of file + From 10f8d6cdefb74e1513020478e237678cd41308cb Mon Sep 17 00:00:00 2001 From: qiagu Date: Thu, 16 Dec 2021 11:36:07 -0800 Subject: [PATCH 337/401] Make html sublcass --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 +- lib/galaxy/datatypes/binary.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 1423a66e43c7..5e1d4a9032fe 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -181,7 +181,7 @@ - + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index de5ee9a956bc..e0aaad94c8a4 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -37,6 +37,7 @@ MetadataParameter, ) from galaxy.datatypes.sniff import build_sniff_from_prefix +from galaxy.datatypes.text import Html from galaxy.util import compression_utils, nice_size, sqlite from galaxy.util.checkers import is_bz2, is_gzip from . import data, dataproviders @@ -1611,12 +1612,12 @@ def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None return f"
{repr_}
{rval}
", headers -class LudwigModel(CompressedZipArchive): +class LudwigModel(Html): """ Composite datatype that encloses multiple files for a Ludwig trained model. """ composite_type = 'auto_primary_file' - file_ext = "ludwig_model.zip" + file_ext = "ludwig_model" def __init__(self, **kwd): super().__init__(**kwd) @@ -1628,7 +1629,7 @@ def __init__(self, **kwd): def generate_primary_file(self, dataset=None): rval = ['Ludwig Model Composite Dataset.

'] - rval.append('

This composite dataset is composed of the following files:

    ') + rval.append('
    This model dataset is composed of the following items:

      ') for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): fn = composite_name opt_text = '' From 65f7259b29d3886e526d9be670c60d9da9fbe038 Mon Sep 17 00:00:00 2001 From: Darren Cullerne Date: Fri, 17 Dec 2021 09:23:03 +1100 Subject: [PATCH 338/401] Adding /doc/source/admin/reports_options.rst from make config-rebuild and additional change to file_path value in /lib/galaxy/webapps/reports/config.py --- doc/source/admin/reports_options.rst | 2 +- lib/galaxy/webapps/reports/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/admin/reports_options.rst b/doc/source/admin/reports_options.rst index e60fb6459882..7f99b3474365 100644 --- a/doc/source/admin/reports_options.rst +++ b/doc/source/admin/reports_options.rst @@ -28,7 +28,7 @@ :Description: Where dataset files are stored. -:Default: ``database/files`` +:Default: ``database/objects`` :Type: str diff --git a/lib/galaxy/webapps/reports/config.py b/lib/galaxy/webapps/reports/config.py index e8e4213900cb..5f653fbe35db 100644 --- a/lib/galaxy/webapps/reports/config.py +++ b/lib/galaxy/webapps/reports/config.py @@ -28,7 +28,7 @@ def __init__(self, **kwargs): self.database_connection = kwargs.get("database_connection", False) self.database_engine_options = get_database_engine_options(kwargs) # Where dataset files are stored - self.file_path = resolve_path(kwargs.get("file_path", "database/files"), self.root) + self.file_path = resolve_path(kwargs.get("file_path", "database/objects"), self.root) self.new_file_path = resolve_path(kwargs.get("new_file_path", "database/tmp"), self.root) self.id_secret = kwargs.get("id_secret", "USING THE DEFAULT IS NOT SECURE!") self.use_remote_user = string_as_bool(kwargs.get("use_remote_user", "False")) From 5391d54a72129370c492a74a67ffd8554b27aee9 Mon Sep 17 00:00:00 2001 From: "Uehara.Keizo" Date: Fri, 17 Dec 2021 04:45:05 +0000 Subject: [PATCH 339/401] Make access_key,secret_key for aws optional Access_key and secret_key are not required because boto3 looks at various configuration locations. --- lib/galaxy/objectstore/cloud.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index f53efbfcefac..4c426e2c356e 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -186,11 +186,7 @@ def parse_xml(clazz, config_xml): missing_config = [] if provider == "aws": akey = auth_element.get("access_key") - if akey is None: - missing_config.append("access_key") skey = auth_element.get("secret_key") - if skey is None: - missing_config.append("secret_key") config["auth"] = { "access_key": akey, From 1f684bc22ed36be2880db8cba86a9f83c1c56464 Mon Sep 17 00:00:00 2001 From: "Uehara.Keizo" Date: Fri, 17 Dec 2021 04:46:05 +0000 Subject: [PATCH 340/401] Apply fixes in s3.py to cloud.py Checking rel_path first character has been removed in s3.py 37dfd4292a69e4f76dcec785361649956eb1c8ca apply same fixes to cloud.py --- lib/galaxy/objectstore/cloud.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 4c426e2c356e..9b19a4fcc5b7 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -400,8 +400,6 @@ def _key_exists(self, rel_path): except Exception: log.exception("Trouble checking existence of S3 key '%s'", rel_path) return False - if rel_path[0] == '/': - raise return exists def _in_cache(self, rel_path): From aa73b52a201d8ca8bf03e3b0237459a4cec27887 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 17 Dec 2021 11:53:48 +0100 Subject: [PATCH 341/401] Use yield_per option to limit amount of ORM objects loaded into memory I've applied it only to non-legacy code where we don't need the entire list up-front. This might help with > I have an issue with the job handlers from a few days. Randomly one of them gets trapped in a death loop. It starts to monitor jobs, the memory ramps up until the systemd limit of 12GB and then is killed by sytemd report by @gmauro on usegalaxy.eu --- lib/galaxy/jobs/handler.py | 4 ++-- lib/galaxy/model/__init__.py | 1 + lib/galaxy/webapps/galaxy/api/jobs.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 90d32966b68f..d1492150f8f3 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -237,11 +237,11 @@ def __check_jobs_at_startup(self): .outerjoin(model.User) \ .filter(model.Job.state.in_(in_list) & (model.Job.handler == self.app.config.server_name) - & or_((model.Job.user_id == null()), (model.User.active == true()))).all() + & or_((model.Job.user_id == null()), (model.User.active == true()))).yield_per(model.YIELD_PER_ROWS) else: jobs_at_startup = self.sa_session.query(model.Job).enable_eagerloads(False) \ .filter(model.Job.state.in_(in_list) - & (model.Job.handler == self.app.config.server_name)).all() + & (model.Job.handler == self.app.config.server_name)).yield_per(model.YIELD_PER_ROWS) for job in jobs_at_startup: if not self.app.toolbox.has_tool(job.tool_id, job.tool_version, exact=True): diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 072cb37f05ef..e2d1544cb932 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -135,6 +135,7 @@ JOB_METRIC_SCALE = 7 # Tags that get automatically propagated from inputs to outputs when running jobs. AUTO_PROPAGATED_TAGS = ["name"] +YIELD_PER_ROWS = 100 if TYPE_CHECKING: diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index e856a16450bc..a63f2efcdae2 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -204,7 +204,7 @@ def build_and_apply_filters(query, objects, filter_func): query = query.limit(limit) out = [] - for job in query.all(): + for job in query.yield_per(model.YIELD_PER_ROWS): job_dict = job.to_dict(view, system_details=is_admin) j = self.encode_all_ids(trans, job_dict, True) if view == 'admin_job_list': From 9e52fb6ac7a45a06839e4efd878f1020e96fc82f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 17 Dec 2021 12:08:07 +0100 Subject: [PATCH 342/401] Disable eagerload of job pja relationship --- lib/galaxy/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e2d1544cb932..ad671eaf3897 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -916,7 +916,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): back_populates='job', lazy=True) output_dataset_collections = relationship('JobToImplicitOutputDatasetCollectionAssociation', back_populates='job', lazy=True) - post_job_actions = relationship('PostJobActionAssociation', back_populates='job', lazy=False) + post_job_actions = relationship('PostJobActionAssociation', back_populates='job', lazy=True) input_library_datasets = relationship('JobToInputLibraryDatasetAssociation', back_populates='job') output_library_datasets = relationship('JobToOutputLibraryDatasetAssociation', From 06eac3e280f6bc3613c39c8776ab687613040940 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Fri, 17 Dec 2021 13:32:53 +0100 Subject: [PATCH 343/401] add implicit parquet>csv converter --- .../config/sample/datatypes_conf.xml.sample | 4 +++- .../converters/parquet_to_csv_converter.py | 22 +++++++++++++++++++ .../converters/parquet_to_csv_converter.xml | 21 ++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/datatypes/converters/parquet_to_csv_converter.py create mode 100644 lib/galaxy/datatypes/converters/parquet_to_csv_converter.xml diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 977efde94add..e5f8bb8053fb 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -82,7 +82,9 @@ - + + + diff --git a/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py b/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py new file mode 100644 index 000000000000..d3d6ef960651 --- /dev/null +++ b/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +""" +Input: parquet +Output: csv +""" +import os +import sys +from pyarrow import csv +from pyarrow import parquet + +def __main__(): + infile = sys.argv[1] + outfile = sys.argv[2] + + if not os.path.isfile(infile): + sys.stderr.write(f"Input file {infile!r} not found\n") + sys.exit(1) + + csv.write_csv(parquet.read_table(infile), outfile) + +if __name__ == "__main__": + __main__() diff --git a/lib/galaxy/datatypes/converters/parquet_to_csv_converter.xml b/lib/galaxy/datatypes/converters/parquet_to_csv_converter.xml new file mode 100644 index 000000000000..4c25eed750e6 --- /dev/null +++ b/lib/galaxy/datatypes/converters/parquet_to_csv_converter.xml @@ -0,0 +1,21 @@ + + + + pyarrow + + python '$__tool_directory__/parquet_to_csv_converter.py' '$input' '$output' + + + + + + + + + + + + + + + From 4176e796dac62951d233c821fa87dfb73b7a0e2c Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 9 Dec 2021 20:06:23 +0000 Subject: [PATCH 344/401] Fix test_singularity_search unit test in a way that it doesn't break when a new version of the container is released. --- test/unit/tool_util/mulled/test_mulled_search.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/unit/tool_util/mulled/test_mulled_search.py b/test/unit/tool_util/mulled/test_mulled_search.py index 8caaed59694d..9731ee37b57b 100644 --- a/test/unit/tool_util/mulled/test_mulled_search.py +++ b/test/unit/tool_util/mulled/test_mulled_search.py @@ -50,7 +50,10 @@ def test_get_package_hash(): @external_dependency_management def test_singularity_search(): sing1 = singularity_search('mulled-v2-0560a8046fc82aa4338588eca29ff18edab2c5aa') + sing1_versions = {result['version'] for result in sing1} + assert { + 'c17ce694dd57ab0ac1a2b86bb214e65fedef760e-0', + 'f471ba33d45697daad10614c5bd25a67693f67f1-0', + 'fc33176431a4b9ef3213640937e641d731db04f1-0'}.issubset(sing1_versions) sing2 = singularity_search('mulled-v2-19fa9431f5863b2be81ff13791f1b00160ed0852') - assert sing1[0]['version'] in ['c17ce694dd57ab0ac1a2b86bb214e65fedef760e-0', 'fc33176431a4b9ef3213640937e641d731db04f1-0'] - assert sing1[1]['version'] in ['c17ce694dd57ab0ac1a2b86bb214e65fedef760e-0', 'fc33176431a4b9ef3213640937e641d731db04f1-0'] assert sing2 == [] From b930e125ece90d56378f4d72236417eeaf888057 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 17 Dec 2021 14:31:48 +0100 Subject: [PATCH 345/401] Remove folder icon in expaded collection header --- client/src/components/History/CurrentCollection/Details.vue | 1 - 1 file changed, 1 deletion(-) diff --git a/client/src/components/History/CurrentCollection/Details.vue b/client/src/components/History/CurrentCollection/Details.vue index e1cfdea2ffce..d13afdc23a4c 100644 --- a/client/src/components/History/CurrentCollection/Details.vue +++ b/client/src/components/History/CurrentCollection/Details.vue @@ -7,7 +7,6 @@ {{ dscName || "(Collection Name)" }}

      -

      From eccab887bbf3080b71e7c9f0568be4a4ae7710e5 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 17 Dec 2021 16:06:45 +0100 Subject: [PATCH 346/401] Remove folder icon in collapsed collection item --- .../History/ContentItem/DatasetCollection/DscUI.vue | 8 -------- 1 file changed, 8 deletions(-) diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue index 8fbddae6bbb7..547b3b8e35ca 100644 --- a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue +++ b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue @@ -41,14 +41,6 @@ icon="trash-restore" @click.stop="$emit('undelete')" /> - -
      {{ dsc.hid }} From 41e088951dbdaa48893b6c3c682b6595b9883398 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Fri, 17 Dec 2021 16:16:21 +0100 Subject: [PATCH 347/401] Apply suggestions from @mvdbeek code review --- .../datatypes/converters/parquet_to_csv_converter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py b/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py index d3d6ef960651..5d70e6e38256 100644 --- a/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py +++ b/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py @@ -5,8 +5,11 @@ """ import os import sys -from pyarrow import csv -from pyarrow import parquet +try: + from pyarrow import csv + from pyarrow import parquet +except ImportError: + csv = parquet = None def __main__(): infile = sys.argv[1] @@ -16,6 +19,8 @@ def __main__(): sys.stderr.write(f"Input file {infile!r} not found\n") sys.exit(1) + if csv is None or parquet is None: + raise Exception("Cannot run conversion, pyarrow is not installed.") csv.write_csv(parquet.read_table(infile), outfile) if __name__ == "__main__": From 3c7bec2f12e35bfe795c7a1d0a2d11a817ca4b8a Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Fri, 17 Dec 2021 16:45:56 +0100 Subject: [PATCH 348/401] Adjust collection description text (slighly) --- .../History/ContentItem/DatasetCollection/DscDescription.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscDescription.vue b/client/src/components/History/ContentItem/DatasetCollection/DscDescription.vue index 1c4ee5ebae92..e61a881662ce 100644 --- a/client/src/components/History/ContentItem/DatasetCollection/DscDescription.vue +++ b/client/src/components/History/ContentItem/DatasetCollection/DscDescription.vue @@ -1,7 +1,7 @@ From f9de20d9b6d8951bd35aaec3e0a480efd59fe2e1 Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Fri, 17 Dec 2021 09:32:39 -0800 Subject: [PATCH 349/401] Fix tag attribute parsing on input element --- client/src/components/Form/Elements/parameters.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/components/Form/Elements/parameters.js b/client/src/components/Form/Elements/parameters.js index 5a5ce8eb30ea..99413dbf35f4 100644 --- a/client/src/components/Form/Elements/parameters.js +++ b/client/src/components/Form/Elements/parameters.js @@ -62,6 +62,7 @@ export default Backbone.View.extend({ type: input_def.type, flavor: input_def.flavor, data: input_def.options, + tag: input_def.tag, onchange: input_def.onchange, }); }, From 914818098f674054a49630c6d7c8854e91562dd9 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 17 Dec 2021 19:06:10 +0100 Subject: [PATCH 350/401] add doc for test/param/metadata currently only linking to a stub at the planemo docs --- doc/schema_template.md | 1 + lib/galaxy/tool_util/xsd/galaxy.xsd | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/schema_template.md b/doc/schema_template.md index a78b21180bf7..8cf7505f02f7 100644 --- a/doc/schema_template.md +++ b/doc/schema_template.md @@ -81,6 +81,7 @@ $tag:tool|outputs|collection|discover_datasets://complexType[@name='OutputCollec $tag:tool|tests://complexType[@name='Tests'] $tag:tool|tests|test://complexType[@name='Test'] $tag:tool|tests|test|param://complexType[@name='TestParam'] +$tag:tool|tests|test|param|metadata://complexType[@name='TestParamMetadata'] $tag:tool|tests|test|param|collection://complexType[@name='TestCollection'] $tag:tool|tests|test|repeat://complexType[@name='TestRepeat'] $tag:tool|tests|test|section://complexType[@name='TestSection'] diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index d38f265b2402..8ef122388f34 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -1732,14 +1732,21 @@ The following demonstrates verifying XML content with XPath-like expressions. + + + - Documentation for name + Name of the metadata element of the data parameter - Documentation for value + Value to set From 72874b74a529ad5b8b58cbcb7ddb1c4ac84a5ec5 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 17 Dec 2021 13:59:59 +0000 Subject: [PATCH 351/401] Add tox environment and GitHub workflow for reports startup test Also: - Fix tox envlist still using py36 - Partially fix running reports with `APP_WEBSERVER=dev` --- .ci/first_startup.sh | 24 +++++++++-- .github/workflows/reports_startup.yaml | 56 ++++++++++++++++++++++++++ scripts/common_startup_functions.sh | 2 +- tox.ini | 4 +- 4 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/reports_startup.yaml diff --git a/.ci/first_startup.sh b/.ci/first_startup.sh index 16d8952fc48d..7c757b1a03b6 100644 --- a/.ci/first_startup.sh +++ b/.ci/first_startup.sh @@ -1,16 +1,32 @@ #!/bin/sh +case "$1" in + galaxy|"") + SCRIPT=./run.sh + PORT=8080 + LOGFILE=galaxy.log + ;; + reports) + SCRIPT=./run_reports.sh + PORT=9001 + LOGFILE=reports_webapp.log + ;; + *) + echo "ERROR: Unrecognized app" + exit 1 + ;; +esac TRIES=120 -URL=http://localhost:8080 +URL=http://localhost:$PORT EXIT_CODE=1 i=0 echo "Testing for correct startup:" -bash run.sh --daemon && \ +$SCRIPT --daemon && \ while [ "$i" -le $TRIES ]; do curl --max-time 1 "$URL" && EXIT_CODE=0 && break sleep 1 i=$((i + 1)) done -bash run.sh --skip-wheels --stop-daemon +$SCRIPT --skip-wheels --stop-daemon echo "exit code:$EXIT_CODE, showing startup log:" -cat galaxy.log +cat "$LOGFILE" exit $EXIT_CODE diff --git a/.github/workflows/reports_startup.yaml b/.github/workflows/reports_startup.yaml new file mode 100644 index 000000000000..59a42974dd3f --- /dev/null +++ b/.github/workflows/reports_startup.yaml @@ -0,0 +1,56 @@ +name: Reports startup +on: + push: + paths-ignore: + - 'doc/**' + pull_request: + paths-ignore: + - 'doc/**' +env: + YARN_INSTALL_OPTS: --frozen-lockfile +concurrency: + group: reports-startup-${{ github.ref }} + cancel-in-progress: true +jobs: + + test: + name: Reports startup test + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.7', '3.10'] + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v2 + with: + path: 'galaxy root' + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Get full Python version + id: full-python-version + shell: bash + run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") + - name: Cache pip dir + uses: actions/cache@v1 + id: pip-cache + with: + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Cache tox env + uses: actions/cache@v2 + with: + path: .tox + key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-reports-startup + - uses: mvdbeek/gha-yarn-cache@master + with: + yarn-lock-file: 'galaxy root/client/yarn.lock' + - name: Install tox + run: pip install tox + - name: run tests + run: tox -e reports_startup + working-directory: 'galaxy root' diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index 49e8e46bb843..c9b08885b599 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -43,7 +43,7 @@ parse_common_args() { shift ;; --daemon|start) - circusd_args="$circusd_args --daemon --log-output $GALAXY_LOG" + circusd_args="$circusd_args --daemon --log-output $LOG_FILE" paster_args="$paster_args --daemon" gunicorn_args="$gunicorn_args --daemon" GALAXY_DAEMON_LOG="$GALAXY_LOG" diff --git a/tox.ini b/tox.ini index 3a403876a57f..0122b93703f0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] # envlist is the list of environments that are tested when `tox` is run without any option # hyphens in an environment name are used to delimit factors -envlist = py36-first_startup, py36-lint, py36-lint_docstring_include_list, py36-mypy, py36-unit, test_galaxy_packages, validate_test_tools +envlist = py37-first_startup, py37-lint, py37-lint_docstring_include_list, py37-mypy, py37-reports_startup, py37-unit, test_galaxy_packages, validate_test_tools skipsdist = True [testenv] @@ -10,7 +10,7 @@ commands = lint: bash .ci/flake8_wrapper.sh lint_docstring: bash .ci/flake8_wrapper_docstrings.sh --exclude lint_docstring_include_list: bash .ci/flake8_wrapper_docstrings.sh --include - + reports_startup: bash .ci/first_startup.sh reports unit: bash run_tests.sh -u # start with test here but obviously someday all of it... mypy: mypy test lib From 42b5971dafa86da913d93234d4fd666f5ee2ed56 Mon Sep 17 00:00:00 2001 From: "Uehara.Keizo" Date: Sun, 19 Dec 2021 09:23:30 +0000 Subject: [PATCH 352/401] test: access_key and secket_key is optional in cloud objectstore(aws) --- test/unit/objectstore/test_objectstore.py | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/test/unit/objectstore/test_objectstore.py b/test/unit/objectstore/test_objectstore.py index 953a432ff6e2..25a3ad20bf08 100644 --- a/test/unit/objectstore/test_objectstore.py +++ b/test/unit/objectstore/test_objectstore.py @@ -727,6 +727,65 @@ def test_config_parse_cloud(): assert len(extra_dirs) == 2 +CLOUD_AWS_NO_AUTH_TEST_CONFIG = """ + + + + + + +""" + + +def test_config_parse_cloud_noauth_for_aws(): + for config_str in [CLOUD_AWS_NO_AUTH_TEST_CONFIG]: + with TestConfig(config_str, clazz=UnitializedCloudObjectStore) as (directory, object_store): + + assert object_store.bucket_name == "unique_bucket_name_all_lowercase" + assert object_store.use_rr is False + + assert object_store.host is None + assert object_store.port == 6000 + assert object_store.multipart is True + assert object_store.is_secure is True + assert object_store.conn_path == "/" + + assert object_store.cache_size == 1000.0 + assert object_store.staging_path == "database/object_store_cache" + assert object_store.extra_dirs["job_work"] == "database/job_working_directory_cloud" + assert object_store.extra_dirs["temp"] == "database/tmp_cloud" + + as_dict = object_store.to_dict() + _assert_has_keys(as_dict, ["provider", "auth", "bucket", "connection", "cache", "extra_dirs", "type"]) + + _assert_key_has_value(as_dict, "type", "cloud") + + auth_dict = as_dict["auth"] + bucket_dict = as_dict["bucket"] + connection_dict = as_dict["connection"] + cache_dict = as_dict["cache"] + + provider = as_dict["provider"] + assert provider == "aws" + print(auth_dict["access_key"]) + _assert_key_has_value(auth_dict, "access_key", None) + _assert_key_has_value(auth_dict, "secret_key", None) + + _assert_key_has_value(bucket_dict, "name", "unique_bucket_name_all_lowercase") + _assert_key_has_value(bucket_dict, "use_reduced_redundancy", False) + + _assert_key_has_value(connection_dict, "host", None) + _assert_key_has_value(connection_dict, "port", 6000) + _assert_key_has_value(connection_dict, "multipart", True) + _assert_key_has_value(connection_dict, "is_secure", True) + + _assert_key_has_value(cache_dict, "size", 1000.0) + _assert_key_has_value(cache_dict, "path", "database/object_store_cache") + + extra_dirs = as_dict["extra_dirs"] + assert len(extra_dirs) == 2 + + AZURE_BLOB_TEST_CONFIG = """ From 2ed66ea25abefe3b15ee0c909a600e06f048fe5b Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 17 Dec 2021 00:24:37 +0530 Subject: [PATCH 353/401] Refactor extra pvc volume handling in k8s --- .../sample/job_conf.xml.sample_advanced | 6 +- lib/galaxy/jobs/runners/kubernetes.py | 53 +++--- lib/galaxy/jobs/runners/util/pykube_util.py | 155 ++++++++++-------- 3 files changed, 113 insertions(+), 101 deletions(-) diff --git a/lib/galaxy/config/sample/job_conf.xml.sample_advanced b/lib/galaxy/config/sample/job_conf.xml.sample_advanced index 081d2b6b426c..8cc36cc0536b 100644 --- a/lib/galaxy/config/sample/job_conf.xml.sample_advanced +++ b/lib/galaxy/config/sample/job_conf.xml.sample_advanced @@ -198,9 +198,11 @@ --> galaxy_pvc:/mount_point - Typically this is used for the PVC used for data inputs and outputs for each job --> - + galaxy_pvc:/mount_point + galaxy_pvc1:/mount_point1,galaxy_pvc2:/mount_point2 +
      + + +