From 32c403b32854c8089b88eddaf796adea98d1b3e6 Mon Sep 17 00:00:00 2001 From: WarpedPixel Date: Thu, 20 Jun 2024 16:34:27 -0700 Subject: [PATCH 1/4] Can't read Pydantic Settings from stdin Fixes #296 Adds a method _read_env_file to DotEnvSettingsSource that can be easily overridden. This fix keeps a global function so it can continue to be tested, but adds an underscore to the name. The plain read_env_file is marked as deprecated. --- pydantic_settings/sources.py | 43 ++++++++++++++++++++++++++++-------- tests/test_settings.py | 8 +++---- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index c7c019a4..414b0826 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -753,6 +753,18 @@ def __init__( def _load_env_vars(self) -> Mapping[str, str | None]: return self._read_env_files() + def _read_env_file( + self, + file_path: Path, + ) -> Mapping[str, str | None]: + return _read_env_file( + file_path, + encoding=self.env_file_encoding, + case_sensitive=self.case_sensitive, + ignore_empty=self.env_ignore_empty, + parse_none_str=self.env_parse_none_str, + ) + def _read_env_files(self) -> Mapping[str, str | None]: env_files = self.env_file if env_files is None: @@ -765,15 +777,7 @@ def _read_env_files(self) -> Mapping[str, str | None]: for env_file in env_files: env_path = Path(env_file).expanduser() if env_path.is_file(): - dotenv_vars.update( - read_env_file( - env_path, - encoding=self.env_file_encoding, - case_sensitive=self.case_sensitive, - ignore_empty=self.env_ignore_empty, - parse_none_str=self.env_parse_none_str, - ) - ) + dotenv_vars.update(self._read_env_file(env_path)) return dotenv_vars @@ -1707,6 +1711,27 @@ def read_env_file( case_sensitive: bool = False, ignore_empty: bool = False, parse_none_str: str | None = None, +) -> Mapping[str, str | None]: + warnings.warn( + 'read_env_file will be removed in the next version, use DotEnvSettingsSource._read_env_file instead', + DeprecationWarning, + ) + return _read_env_file( + file_path, + encoding=encoding, + case_sensitive=case_sensitive, + ignore_empty=ignore_empty, + parse_none_str=parse_none_str, + ) + + +def _read_env_file( + file_path: Path, + *, + encoding: str | None = None, + case_sensitive: bool = False, + ignore_empty: bool = False, + parse_none_str: str | None = None, ) -> Mapping[str, str | None]: file_vars: dict[str, str | None] = dotenv_values(file_path, encoding=encoding or 'utf8') return parse_env_vars(file_vars, case_sensitive, ignore_empty, parse_none_str) diff --git a/tests/test_settings.py b/tests/test_settings.py index 7690e3c6..e86c8ba9 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -49,7 +49,7 @@ TomlConfigSettingsSource, YamlConfigSettingsSource, ) -from pydantic_settings.sources import CliPositionalArg, CliSettingsSource, CliSubCommand, SettingsError, read_env_file +from pydantic_settings.sources import CliPositionalArg, CliSettingsSource, CliSubCommand, SettingsError, _read_env_file try: import dotenv @@ -1039,15 +1039,15 @@ def test_read_env_file_case_sensitive(tmp_path): p = tmp_path / '.env' p.write_text('a="test"\nB=123') - assert read_env_file(p) == {'a': 'test', 'b': '123'} - assert read_env_file(p, case_sensitive=True) == {'a': 'test', 'B': '123'} + assert _read_env_file(p) == {'a': 'test', 'b': '123'} + assert _read_env_file(p, case_sensitive=True) == {'a': 'test', 'B': '123'} def test_read_env_file_syntax_wrong(tmp_path): p = tmp_path / '.env' p.write_text('NOT_AN_ASSIGNMENT') - assert read_env_file(p, case_sensitive=True) == {'NOT_AN_ASSIGNMENT': None} + assert _read_env_file(p, case_sensitive=True) == {'NOT_AN_ASSIGNMENT': None} def test_env_file_example(tmp_path): From c6b1bf25a5c5d2f67e54251c2be2e7628b77c43e Mon Sep 17 00:00:00 2001 From: WarpedPixel Date: Tue, 25 Jun 2024 10:32:47 -0700 Subject: [PATCH 2/4] Can't read Pydantic Settings from stdin Fixes #296 Added test and moved bulk of dotenv code into DotEnvSettingsSource so future deprecation will be cleaner --- pydantic_settings/sources.py | 30 +++++++++++++++--------------- tests/test_settings.py | 25 ++++++++++++++++++++----- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 414b0826..23bbbf13 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -753,11 +753,23 @@ def __init__( def _load_env_vars(self) -> Mapping[str, str | None]: return self._read_env_files() + @staticmethod + def _static_read_env_file( + file_path: Path, + *, + encoding: str | None = None, + case_sensitive: bool = False, + ignore_empty: bool = False, + parse_none_str: str | None = None, + ) -> Mapping[str, str | None]: + file_vars: dict[str, str | None] = dotenv_values(file_path, encoding=encoding or 'utf8') + return parse_env_vars(file_vars, case_sensitive, ignore_empty, parse_none_str) + def _read_env_file( self, file_path: Path, ) -> Mapping[str, str | None]: - return _read_env_file( + return self._static_read_env_file( file_path, encoding=self.env_file_encoding, case_sensitive=self.case_sensitive, @@ -1713,10 +1725,10 @@ def read_env_file( parse_none_str: str | None = None, ) -> Mapping[str, str | None]: warnings.warn( - 'read_env_file will be removed in the next version, use DotEnvSettingsSource._read_env_file instead', + 'read_env_file will be removed in the next version, use DotEnvSettingsSource._static_read_env_file if you must', DeprecationWarning, ) - return _read_env_file( + return DotEnvSettingsSource._static_read_env_file( file_path, encoding=encoding, case_sensitive=case_sensitive, @@ -1725,18 +1737,6 @@ def read_env_file( ) -def _read_env_file( - file_path: Path, - *, - encoding: str | None = None, - case_sensitive: bool = False, - ignore_empty: bool = False, - parse_none_str: str | None = None, -) -> Mapping[str, str | None]: - file_vars: dict[str, str | None] = dotenv_values(file_path, encoding=encoding or 'utf8') - return parse_env_vars(file_vars, case_sensitive, ignore_empty, parse_none_str) - - def _annotation_is_complex(annotation: type[Any] | None, metadata: list[Any]) -> bool: if any(isinstance(md, Json) for md in metadata): # type: ignore[misc] return False diff --git a/tests/test_settings.py b/tests/test_settings.py index e86c8ba9..29435877 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -9,7 +9,7 @@ from datetime import datetime, timezone from enum import IntEnum from pathlib import Path -from typing import Any, Callable, Dict, Generic, Hashable, List, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Any, Callable, Dict, Generic, Hashable, List, Optional, Set, Tuple, Type, TypeVar, Union, override import pytest import typing_extensions @@ -49,7 +49,7 @@ TomlConfigSettingsSource, YamlConfigSettingsSource, ) -from pydantic_settings.sources import CliPositionalArg, CliSettingsSource, CliSubCommand, SettingsError, _read_env_file +from pydantic_settings.sources import CliPositionalArg, CliSettingsSource, CliSubCommand, SettingsError try: import dotenv @@ -1039,15 +1039,15 @@ def test_read_env_file_case_sensitive(tmp_path): p = tmp_path / '.env' p.write_text('a="test"\nB=123') - assert _read_env_file(p) == {'a': 'test', 'b': '123'} - assert _read_env_file(p, case_sensitive=True) == {'a': 'test', 'B': '123'} + assert DotEnvSettingsSource._static_read_env_file(p) == {'a': 'test', 'b': '123'} + assert DotEnvSettingsSource._static_read_env_file(p, case_sensitive=True) == {'a': 'test', 'B': '123'} def test_read_env_file_syntax_wrong(tmp_path): p = tmp_path / '.env' p.write_text('NOT_AN_ASSIGNMENT') - assert _read_env_file(p, case_sensitive=True) == {'NOT_AN_ASSIGNMENT': None} + assert DotEnvSettingsSource._static_read_env_file(p, case_sensitive=True) == {'NOT_AN_ASSIGNMENT': None} def test_env_file_example(tmp_path): @@ -1188,6 +1188,21 @@ def test_read_dotenv_vars_when_env_file_is_none(): ) +def test_dotenvsource_override(env): + class StdinDotEnvSettingsSource(DotEnvSettingsSource): + @override + def _read_env_file(self, file_path: Path) -> Dict[str, str]: + assert str(file_path) == '-' + return {'foo': 'stdin_foo', 'bar': 'stdin_bar'} + + @override + def _read_env_files(self) -> Dict[str, str]: + return self._read_env_file(Path('-')) + + source = StdinDotEnvSettingsSource(BaseSettings()) + assert source._read_env_files() == {'foo': 'stdin_foo', 'bar': 'stdin_bar'} + + @pytest.mark.skipif(yaml, reason='PyYAML is installed') def test_yaml_not_installed(tmp_path): p = tmp_path / '.env' From a8c404cec111f311b2697551283bed8cbe3c23d8 Mon Sep 17 00:00:00 2001 From: WarpedPixel Date: Tue, 25 Jun 2024 10:42:51 -0700 Subject: [PATCH 3/4] Fixed for older versions of Python --- tests/test_settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 29435877..79d1f4b0 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -9,7 +9,7 @@ from datetime import datetime, timezone from enum import IntEnum from pathlib import Path -from typing import Any, Callable, Dict, Generic, Hashable, List, Optional, Set, Tuple, Type, TypeVar, Union, override +from typing import Any, Callable, Dict, Generic, Hashable, List, Optional, Set, Tuple, Type, TypeVar, Union import pytest import typing_extensions @@ -34,7 +34,7 @@ from pydantic._internal._repr import Representation from pydantic.fields import FieldInfo from pytest_mock import MockerFixture -from typing_extensions import Annotated, Literal +from typing_extensions import Annotated, Literal, override from pydantic_settings import ( BaseSettings, From 33f5646de4438480b0027f1a2439b1f626fb69d8 Mon Sep 17 00:00:00 2001 From: WarpedPixel Date: Mon, 1 Jul 2024 11:32:54 -0700 Subject: [PATCH 4/4] Can't read Pydantic Settings from stdin Fixes #296 Addressed CR feedback --- tests/test_settings.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_settings.py b/tests/test_settings.py index 79d1f4b0..e812ffed 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1203,6 +1203,22 @@ def _read_env_files(self) -> Dict[str, str]: assert source._read_env_files() == {'foo': 'stdin_foo', 'bar': 'stdin_bar'} +# test that calling read_env_file issues a DeprecationWarning +# TODO: remove this test once read_env_file is removed +def test_read_env_file_deprecation(tmp_path): + from pydantic_settings.sources import read_env_file + + base_env = tmp_path / '.env' + base_env.write_text(test_default_env_file) + + with pytest.deprecated_call(): + assert read_env_file(base_env) == { + 'debug_mode': 'true', + 'host': 'localhost', + 'port': '8000', + } + + @pytest.mark.skipif(yaml, reason='PyYAML is installed') def test_yaml_not_installed(tmp_path): p = tmp_path / '.env'