Merge branch 'master' of github.com:quiltdata/quilt into selective-pa…

…ckage-download

.github/pull_request_template.md

@@ -9,6 +9,7 @@
 - [ ] Automated tests (e.g. Preflight)
 - [ ] Confirm that this change meets security best practices and does not violate the security model
 - [ ] Documentation
+    - [ ] run `optipng` on any new PNGs
     - [ ] [Python: Run `build.py`](../tree/master/gendocs/build.py) for new docstrings
     - [ ] JavaScript: basic explanation and screenshot of new features
     - [ ] Markdown somewhere in docs/**/*.md that explains the feature to end users (said .md files should be linked from SUMMARY.md so they appear on https://docs.quiltdata.com)

.github/workflows/py-ci.yml

@@ -133,15 +133,15 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.7'
+          python-version: '3.12'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools
+          python -m pip install build==1.2.2.post1 twine==5.1.1
       - name: verify git tag vs. version
         env:
           CIRCLE_TAG: ${{ github.ref_name }}
         run: python setup.py verify
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip setuptools
-          python -m pip install build==0.8.0 twine==4.0.0
       - name: build
         run: python -m build
       - name: upload to PyPI

api/python/quilt3/VERSION

@@ -1 +1 @@
-6.0.0
+6.1.0

api/python/quilt3/__init__.py

@@ -22,6 +22,6 @@
 from .bucket import Bucket
 from .imports import start_data_package_loader
 from .packages import Package
-from .session import logged_in, login, logout
+from .session import get_boto3_session, logged_in, login, logout
 
 start_data_package_loader()

api/python/quilt3/data_transfer.py

@@ -21,7 +21,6 @@
 from threading import Lock
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
 
-import boto3
 import jsonlines
 from boto3.s3.transfer import TransferConfig
 from botocore import UNSIGNED
@@ -43,7 +42,7 @@
 from tqdm import tqdm
 
 from . import util
-from .session import create_botocore_session
+from .session import get_boto3_session
 from .util import DISABLE_TQDM, PhysicalKey, QuiltException
 
 MAX_COPY_FILE_LIST_RETRIES = 3
@@ -152,9 +151,7 @@ def find_correct_client(self, api_type, bucket, param_dict):
                     raise S3NoValidClientError(f"S3 AccessDenied for {api_type} on bucket: {bucket}")
 
     def get_boto_session(self):
-        botocore_session = create_botocore_session()
-        boto_session = boto3.Session(botocore_session=botocore_session)
-        return boto_session
+        return get_boto3_session()
 
     def _build_client(self, is_unsigned):
         session = self.get_boto_session()

api/python/quilt3/session.py

@@ -9,8 +9,10 @@
 import subprocess
 import sys
 import time
+import typing as T
 from importlib import metadata
 
+import boto3
 import botocore.session
 import requests
 from botocore.credentials import (
@@ -288,14 +290,34 @@ def load(self):
         return creds
 
 
-def create_botocore_session():
+def create_botocore_session(*, credentials: T.Optional[dict] = None) -> botocore.session.Session:
     botocore_session = botocore.session.get_session()
 
     # If we have saved credentials, use them. Otherwise, create a normal Boto session.
-    credentials = _load_credentials()
+    if credentials is None:
+        credentials = _load_credentials()
     if credentials:
         provider = QuiltProvider(credentials)
         resolver = CredentialResolver([provider])
         botocore_session.register_component('credential_provider', resolver)
 
     return botocore_session
+
+
+def get_boto3_session(*, fallback: bool = True) -> boto3.Session:
+    """
+    Return a Boto3 session with Quilt stack credentials and AWS region.
+    In case of no Quilt credentials found, return a "normal" Boto3 session if `fallback` is `True`,
+    otherwise raise a `QuiltException`.
+
+    > Note: you need to call `quilt3.config("https://your-catalog-homepage/")` to have region set on the session,
+    if you previously called it in quilt3 < 6.1.0.
+    """
+    if not (credentials := _load_credentials()):
+        if fallback:
+            return boto3.Session()
+        raise QuiltException("No Quilt credentials found.")
+    return boto3.Session(
+        botocore_session=create_botocore_session(credentials=credentials),
+        region_name=get_from_config("region"),
+    )

api/python/quilt3/util.py

@@ -81,6 +81,9 @@ def get_bool_from_env(var_name: str):
 
 default_registry_version: 1
 
+# AWS region
+region:
+
 """.format(BASE_PATH.as_uri() + '/packages')
 
 

api/python/tests/test_api.py

@@ -17,7 +17,8 @@ def test_config(self):
             'telemetry_disabled': False,
             's3Proxy': None,
             'apiGatewayEndpoint': None,
-            'binaryApiGatewayEndpoint': None
+            'binaryApiGatewayEndpoint': None,
+            "region": "us-west-2",
         }
         self.requests_mock.add(responses.GET, 'https://foo.bar/config.json', json=content, status=200)
 

api/python/tests/test_session.py

@@ -5,13 +5,20 @@
 import datetime
 from unittest.mock import patch
 
+import boto3
+import pytest
 import responses
 
 import quilt3
+import quilt3.util
 
 from .utils import QuiltTestCase
 
 
+def format_date(date):
+    return date.replace(tzinfo=datetime.timezone.utc, microsecond=0).isoformat()
+
+
 class TestSession(QuiltTestCase):
     @patch('quilt3.session.open_url')
     @patch('quilt3.session.input', return_value='123456')
@@ -67,9 +74,6 @@ def test_login_with_token(self, mock_save_credentials, mock_save_auth):
     @patch('quilt3.session._save_credentials')
     @patch('quilt3.session._load_credentials')
     def test_create_botocore_session(self, mock_load_credentials, mock_save_credentials):
-        def format_date(date):
-            return date.replace(tzinfo=datetime.timezone.utc, microsecond=0).isoformat()
-
         # Test good credentials.
         future_date = datetime.datetime.utcnow() + datetime.timedelta(hours=1)
 
@@ -121,6 +125,62 @@ def format_date(date):
 
         mock_save_credentials.assert_called()
 
+    @patch("quilt3.util.load_config")
+    @patch("quilt3.session._load_credentials")
+    def test_get_boto3_session(self, mock_load_credentials, mock_load_config):
+        for kw in (
+            {"fallback": False},
+            {"fallback": True},
+            {},
+        ):
+            mock_load_credentials.reset_mock()
+            mock_load_config.reset_mock()
+            with self.subTest(kwargs=kw):
+                region = "us-west-2"
+                config = quilt3.util.load_config()
+                mock_load_config.return_value = {
+                    **config,
+                    "region": region,
+                }
+
+                future_date = datetime.datetime.now() + datetime.timedelta(hours=1)
+                mock_load_credentials.return_value = dict(
+                    access_key="access-key",
+                    secret_key="secret-key",
+                    token="session-token",
+                    expiry_time=format_date(future_date),
+                )
+
+                session = quilt3.get_boto3_session()
+                mock_load_credentials.assert_called_once_with()
+                mock_load_config.assert_called_with()
+
+                assert isinstance(session, boto3.Session)
+                credentials = session.get_credentials()
+
+                assert credentials.access_key == "access-key"
+                assert credentials.secret_key == "secret-key"
+                assert credentials.token == "session-token"
+
+                assert session.region_name == region
+
+    @patch("quilt3.session.create_botocore_session")
+    @patch("quilt3.session._load_credentials", return_value={})
+    def test_get_boto3_session_no_credentials_fallback_true(self, mock_load_credentials, mock_create_botocore_session):
+        session = quilt3.get_boto3_session()
+        mock_load_credentials.assert_called_once_with()
+        mock_create_botocore_session.assert_not_called()
+
+        assert isinstance(session, boto3.Session)
+
+    @patch("quilt3.session._load_credentials", return_value={})
+    def test_get_boto3_session_no_credentials_fallback_false(self, mock_load_credentials):
+        with pytest.raises(quilt3.util.QuiltException) as exc_info:
+            quilt3.get_boto3_session(fallback=False)
+
+        mock_load_credentials.assert_called_once_with()
+        assert "No Quilt credentials found" in str(exc_info.value)
+
     def test_logged_in(self):
         registry_url = quilt3.session.get_registry_url()
         other_registry_url = registry_url + 'other'

docs/CHANGELOG.md

@@ -13,11 +13,12 @@ Entries inside each section should be ordered by type:
 ## CLI
 !-->
 
-# unreleased - YYYY-MM-DD
+# 6.1.0 - 2024-10-14
 
 ## Python API
 
 * [Added] `quilt3.admin.tabulator` sub-module for managing Tabulator configuration ([#4136](https://github.com/quiltdata/quilt/pull/4136))
+* [Added] `quilt3.get_boto3_session()` function for creation of Boto3 sessions with Quilt credentials ([#4169](https://github.com/quiltdata/quilt/pull/4169))
 
 # 6.0.0 - 2024-08-19
 

docs/CONTRIBUTING.md

@@ -1,6 +1,5 @@
 <!--pytest-codeblocks:skipfile-->
-<!-- markdownlint-disable -->
-
+<!-- markdownlint-disable-next-line first-line-h1 -->
 Quilt is an open source project, and we welcome contributions from the community.
 
 Contributors must adhere to the [Code of Conduct](https://github.com/quiltdata/quilt/blob/master/docs/CODE_OF_CONDUCT.md).
@@ -16,15 +15,15 @@ Found a bug? File it in our [GitHub issues](https://github.com/quiltdata/quilt/i
 To work on `quilt` you will first need to clone the repository.
 
 ```bash
-$ git clone https://github.com/quiltdata/quilt
+git clone https://github.com/quiltdata/quilt
 ```
 
 You can then set up your own branch version of the code, and work
 on your changes for a pull request from there.
 
 ```bash
-$ cd quilt
-$ git checkout -B new-branch-name
+cd quilt
+git checkout -B new-branch-name
 ```
 
 ## Local package development
@@ -34,8 +33,8 @@ $ git checkout -B new-branch-name
 Use `pip` to install `quilt` locally (including development dependencies):
 
 ```bash
-$ cd api/python
-$ pip install -e '.[extra]'
+cd api/python
+pip install -e '.[extra]'
 ```
 
 This will create an [editable
@@ -52,8 +51,8 @@ Use `pytest` to test your changes during normal development. To run
 `pytest` on the entire codebase:
 
 ```bash
-$ cd api/python/tests
-$ pytest
+cd api/python/tests
+pytest
 ```
 
 ## Local catalog development
@@ -68,8 +67,8 @@ Elasticsearch Service) which cannot be run locally.
 Use `npm` to install the catalog (`quilt-navigator`) dependencies locally:
 
 ```bash
-$ cd catalog
-$ npm install
+cd catalog
+npm install
 ```
 
 There is one known issue with installation. At time of writing, the
@@ -83,8 +82,8 @@ To fix this, point `npm` to a Python 2 path on your machine. For
 example on macOS:
 
 ```bash
-$ npm config set python /usr/bin/python
-$ npm install
+npm config set python /usr/bin/python
+npm install
 ```
 
 Next, you need to create a `config.json` and `federation.json` file
@@ -137,13 +136,13 @@ For `config.json` use the following template:
 To build a static code bundle, as would be necessary in order to serve the catalog:
 
 ```bash
-$ npm run build
+npm run build
 ```
 
 To run the catalog in developer mode:
 
 ```bash
-$ npm start
+npm start
 ```
 
 This uses `webpack` under the hood to compile code changes on the
@@ -168,8 +167,8 @@ npm run test
 file and in [`CHANGELOG`](https://github.com/quiltdata/quilt/blob/master/docs/CHANGELOG.md).
 1. Create PR with these changes.
 1. Once PR is merged, create a tag from commit with merge: `git tag $VERSION $COMMIT_HASH`.
-1. Once you push the tag to GitHub with `git push origin $VERSION` a new CI build that makes
-PyPI release is triggered.
+1. Once you push the tag to GitHub with `git push origin $VERSION` a new CI build
+that makes PyPI release is triggered.
 
 ## Updating documentation
 

docs/SUMMARY.md

@@ -32,6 +32,7 @@
   * [Restrict Access by Bucket Prefix](advanced-features/s3-prefix-permissions.md)
   * [S3 Events via EventBridge](EventBridge.md)
   * [SSO Permissions Mapping](advanced-features/sso-permissions.md)
+  * [Tabulator](advanced-features/tabulator.md)
 * **Best Practices**
   * [GxP for Security & Compliance](advanced-features/good-practice.md)
   * [Organizing S3 Buckets](advanced-features/s3-bucket-organization.md)