Merge branch 'model' into v2_main

.github/CONTRIBUTING.md

@@ -10,7 +10,6 @@ Before submitting a pull request, please make sure the enhancement or bugfix you
 
 This will ensure no work is duplicated, and that a general approach has been agreed.
 
-
 ## Local development setup
 
 ```shell
@@ -21,15 +20,19 @@ pip install -e .[dev]
 
 ## Build
 
-```shell
-download edoc.proto from internal version control
-python _generate\__main__.py
-```
+> [!IMPORTANT]
+> If you're writing support for a new version of ThoughtSpot, this step requires you to download `scriptability`'s EDoc protocol buffer specification from ThoughtSpot's internal version control.
+>
+> ```shell
+> python _generate\__main__.py
+> ```
+
+If you're not building support for a new version of ThoughtSpot, simply **[Fork this repository](https://github.com/thoughtspot/thoughtspot_tml/fork)** and when ready, **[Open a Pull Request](https://github.com/thoughtspot/thoughtspot_tml/compare)** to contribute your changes.
 
 ## Linting
 
 ```shell
-black . --config pyproject.toml
+ruff check src/ --config pyproject.toml
 ```
 
 ## Running Tests

.github/workflows/lint.yml

@@ -7,11 +7,8 @@ on:
       - v2_main
 
 jobs:
-  lint:
+  ruff:
     runs-on: ubuntu-latest
-
     steps:
-      - uses: actions/checkout@v2
-      - uses: psf/black@stable
-        with:
-          options: "--check --verbose"
+      - uses: actions/checkout@v3
+      - uses: chartboost/ruff-action@v1

.github/workflows/test.yml

@@ -24,6 +24,7 @@ jobs:
           - "3.9"
           - "3.10"
           - "3.11"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v3

.pre-commit-config.yaml

@@ -9,15 +9,14 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: v0.1.6
+    rev: v0.1.7
     hooks:
+      # Run the linter.
       - id: ruff
+        args: [--fix]
 
-  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 23.11.0
-    hooks:
-      - id: black
+      # Run the formatter.
+      - id: ruff-format
 
   - repo: https://github.com/dhruvmanila/remove-print-statements
     rev: v0.5.2

pyproject.toml

@@ -7,7 +7,7 @@ name = "thoughtspot_tml"
 dynamic = ["version"]
 description = "Library for manipulating ThoughtSpot Modeling Language (TML) files"
 readme = "README.md"
-requires-python = ">=3.7 , < 3.12"
+requires-python = ">=3.7 , < 3.13"
 license = {file = "LICENSE"}
 authors = [
     {name = "Bryant Howell", email = "bryant.howell@thoughtspot.com"},
@@ -28,6 +28,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 
 dependencies = [
@@ -55,7 +56,6 @@ dev = [
     # code quality
     "pre-commit",
     "ruff",
-    "black",
     "mypy",
 
     # tests
@@ -75,17 +75,13 @@ exclude = ["_generate*"]
 [tool.setuptools.dynamic]
 version = {attr = "thoughtspot_tml._version.__version__"}
 
-[tool.black]
-line-length = 120
-exclude = '''
-(
-      __pycache__   # ignore compiled bytecode
-    | \.venv*       # ignore virtual environments
-    | \.nox         # ignore virtual environments
-)
-'''
-
 [tool.mypy]
+warn_unused_configs = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+check_untyped_defs = true
+strict_equality = true
+strict_concatenate = true
 exclude = '''
 (?x)(
     ^_scriptability.py$  # ignore auto-generated files
@@ -94,33 +90,64 @@ exclude = '''
 
 [tool.ruff]
 line-length = 120
+src = ["src/thoughtspot_tml"]
+exclude = [
+    "__pycache__",   # ignore compiled bytecode
+    ".venv*",        # ignore virtual environments
+    ".nox",          # ignore virtual environments
+
+    # project specific ignores
+    "__init__.py",        # ignore __init__.py
+    "_version.py",        # ignore __version__
+    "*compat.py",         # ignore compatibles
+    "_scriptability.py",  # ignore auto-generated files
+]
+
+[tool.ruff.lint]
+ignore-init-module-imports = true
 select = [
     "A",     # flake8-builtins: https://pypi.org/project/flake8-builtins/
     "ARG",   # flake8-unused-arguments: https://pypi.org/project/flake8-unused-arguments/
     "B",     # flake8-bugbear: https://pypi.org/project/flake8-bugbear/
     "C4",    # flake8-comprehensions: https://pypi.org/project/flake8-comprehensions/
-    "C90",   # mccabe: https://pypi.org/project/mccabe/
     "COM",   # flake8-commas: https://pypi.org/project/flake8-commas/
+    "DTZ",   # flake8-datetimez: https://pypi.org/project/flake8-datetimez/
     "E",     # pycodestyle: https://pypi.org/project/pycodestyle/
     "F",     # pyflakes: https://pypi.org/project/pyflakes/
+    "FA",    # flake8-future-annotations: https://pypi.org/project/flake8-future-annotations/
+    "I",     # isort: https://pypi.org/project/isort/
     "Q",     # flake8-quotes: https://pypi.org/project/flake8-quotes/
     "RUF",   # ruff-specific: https://beta.ruff.rs/docs/rules/#ruff-specific-rules-ruf
+    "T20",   # flake8-print: https://pypi.org/project/flake8-print/
     "TCH",   # flake8-type-checking: https://pypi.org/project/flake8-type-checking/
 ]
-src = ["src/thoughtspot_tml"]
-exclude = [
-    "__pycache__",   # ignore compiled bytecode
-    ".venv*",        # ignore virtual environments
-    ".nox",          # ignore virtual environments
 
-    # project specific ignores
-    "*compat.py",         # ignore compatibles
-    "_scriptability.py",  # ignore auto-generated files
+[tool.ruff.lint.flake8-import-conventions.aliases]
+# Declare the default aliases.
+datetime = "dt"
+sqlalchemy = "sa"
+
+[tool.ruff.lint.flake8-type-checking]
+runtime-evaluated-base-classes = [
+    "cs_tools.datastructures._GlobalModel",
+    "cs_tools.datastructures._GlobalSettings",
+    "cs_tools.datastructures.ValidatedSQLModel",
+    "typer.params.Option",
+    "typer.params.Argument",
+    "pydantic.BaseModel",
+    "pydantic_settings.BaseSettings",
+    "sqlalchemy.orm.DeclarativeBase",
+    "sqlmodel.SQLModel",
 ]
-ignore-init-module-imports = true
 
-[tool.ruff.per-file-ignores]
-"types.py" = ["TCH001"]
+[flake8]
+type-checking-pydantic-enabled = true
+
+[tool.ruff.lint.isort]
+combine-as-imports = true
+force-wrap-aliases = true
+from-first = true
+required-imports = ["from __future__ import annotations"]
 
 [tool.ward]
 path = "tests/"

src/thoughtspot_tml/_tml.py

@@ -1,19 +1,19 @@
 from __future__ import annotations
 
 from collections.abc import Collection
-from typing import TYPE_CHECKING
 from dataclasses import asdict, dataclass, fields, is_dataclass
-import warnings
-import pathlib
-import typing
+from typing import TYPE_CHECKING
 import json
+import pathlib
 import re
+import typing
+import warnings
 
 import yaml
 
-from thoughtspot_tml.exceptions import TMLDecodeError, TMLExtensionWarning
-from thoughtspot_tml._compat import get_origin, get_args
 from thoughtspot_tml import _scriptability, _yaml
+from thoughtspot_tml._compat import get_args, get_origin
+from thoughtspot_tml.exceptions import TMLDecodeError, TMLExtensionWarning
 
 if TYPE_CHECKING:
     from typing import Any, Dict
@@ -22,7 +22,7 @@
 RE_CAMEL_CASE = re.compile(r"[A-Z]?[a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|\d+")
 
 
-def recursive_complex_attrs_to_dataclasses(instance: Any) -> None:  # noqa: C901
+def recursive_complex_attrs_to_dataclasses(instance: Any) -> None:
     """
     Convert all fields of type `dataclass` into an instance of the
     specified dataclass if the current value is a dict.
@@ -137,12 +137,12 @@ def __post_init__(self):
     @classmethod
     def _loads(cls, tml_document: str) -> Dict[str, Any]:
         # @boonhapus note: do not override this!!
-        #   ONLY exists to enable a TML interface over Connections and fix SCAL-134095
+        #   These exist to handle backwards compatible changes between TML versions.
         return _yaml.load(tml_document)
 
     def _to_dict(self) -> Dict[str, Any]:
         # @boonhapus note: do not override this!!
-        #   ONLY exists to enable a TML interface over Connections and fix SCAL-134095
+        #   These exist to handle backwards compatible changes between TML versions.
         return asdict(self)
 
     @classmethod

src/thoughtspot_tml/_version.py

@@ -1 +1 @@
-__version__ = "2.0.13"
+__version__ = "2.0.14"

src/thoughtspot_tml/_yaml.py

@@ -1,10 +1,11 @@
+from __future__ import annotations
+
 from math import inf as INFINITY
 from typing import Any, Dict
 import re
 
 import yaml
 
-
 # TML column ids typically take the form..
 #
 #   LOGICAL_TABLE_NAME_#::LOGICAL_COLUMN_NAME
@@ -52,8 +53,6 @@ def _double_quote_when_special_char(dumper: yaml.Dumper, data: str) -> yaml.Scal
 
     if (special and not is_tml_id) or reserved or empty_str:
         style = '"'
-    # elif len(data.splitlines()) > 1:
-    #     style = "|"
     else:
         style = ""
 

src/thoughtspot_tml/exceptions.py

@@ -1,17 +1,16 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, Optional, Type
 import dataclasses
 
-from typing import Any, Dict, Optional, Type
-
 if TYPE_CHECKING:
     from collections.abc import Iterable
     from pathlib import Path
 
-    from thoughtspot_tml.types import TMLObject, GUID
     from yaml import error
 
+    from thoughtspot_tml.types import GUID, TMLObject
+
 
 class TMLError(Exception):
     """

src/thoughtspot_tml/spotapp.py

@@ -1,24 +1,23 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Dict, List, Optional
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict, List, Optional
+import json
 import pathlib
 import zipfile
-import json
 
+from thoughtspot_tml import _yaml
 from thoughtspot_tml._compat import ZipPath
+from thoughtspot_tml.tml import Answer, Liveboard, SQLView, Table, View, Worksheet
 from thoughtspot_tml.utils import determine_tml_type
-from thoughtspot_tml.tml import Table, View, SQLView, Worksheet, Answer, Liveboard
-from thoughtspot_tml import _yaml
 
 if TYPE_CHECKING:
-    from thoughtspot_tml.types import EDocExportResponses, TMLObject, TMLDocInfo, SpotAppInfo
+    from thoughtspot_tml.types import EDocExportResponses, SpotAppInfo, TMLDocInfo, TMLObject
 
 
 @dataclass
 class Manifest:
-    object: List[TMLDocInfo]  # noqa: A003
+    object: List[TMLDocInfo]
 
 
 @dataclass