This commit is contained in:
macbook_max
2026-10-15 20:36:59 +08:00
commit 6bf949949c
7923 changed files with 1369582 additions and 0 deletions
@@ -0,0 +1,48 @@
"""Package containing individual source implementations."""
from .aws import AWSSecretsManagerSettingsSource, AWSSystemsManagerSettingsSource
from .azure import AzureKeyVaultSettingsSource
from .cli import (
CliDualFlag,
CliExplicitFlag,
CliImplicitFlag,
CliMutuallyExclusiveGroup,
CliPositionalArg,
CliSettingsSource,
CliSubCommand,
CliSuppress,
CliToggleFlag,
CliVariadicArg,
)
from .dotenv import DotEnvSettingsSource
from .env import EnvSettingsSource
from .gcp import GoogleSecretManagerSettingsSource
from .json import JsonConfigSettingsSource
from .pyproject import PyprojectTomlConfigSettingsSource
from .secrets import SecretsSettingsSource
from .toml import TomlConfigSettingsSource
from .yaml import YamlConfigSettingsSource
__all__ = [
'AWSSecretsManagerSettingsSource',
'AWSSystemsManagerSettingsSource',
'AzureKeyVaultSettingsSource',
'CliDualFlag',
'CliExplicitFlag',
'CliImplicitFlag',
'CliMutuallyExclusiveGroup',
'CliPositionalArg',
'CliSettingsSource',
'CliSubCommand',
'CliSuppress',
'CliToggleFlag',
'CliVariadicArg',
'DotEnvSettingsSource',
'EnvSettingsSource',
'GoogleSecretManagerSettingsSource',
'JsonConfigSettingsSource',
'PyprojectTomlConfigSettingsSource',
'SecretsSettingsSource',
'TomlConfigSettingsSource',
'YamlConfigSettingsSource',
]
@@ -0,0 +1,158 @@
from __future__ import annotations as _annotations # important for BaseSettings import to work
import json
from collections.abc import Mapping
from typing import TYPE_CHECKING
from ...exceptions import SettingsError
from ..utils import InitState, parse_env_vars
from .env import EnvSettingsSource
if TYPE_CHECKING:
from types_boto3_secretsmanager.client import SecretsManagerClient
from types_boto3_ssm.client import SSMClient
from pydantic_settings.main import BaseSettings
boto3_client = None
def import_aws_secrets_manager() -> None:
global boto3_client
try:
from boto3 import client as boto3_client
except ImportError as e: # pragma: no cover
raise ImportError(
'AWS Secrets Manager dependencies are not installed, run `pip install pydantic-settings[aws-secrets-manager]`'
) from e
def import_aws_systems_manager() -> None:
global boto3_client
try:
from boto3 import client as boto3_client
except ImportError as e: # pragma: no cover
raise ImportError(
'AWS Systems Manager dependencies are not installed, run `pip install pydantic-settings[aws-systems-manager]`'
) from e
class AWSSecretsManagerSettingsSource(EnvSettingsSource):
_secret_id: str
_secretsmanager_client: SecretsManagerClient
def __init__(
self,
settings_cls: type[BaseSettings],
secret_id: str,
region_name: str | None = None,
endpoint_url: str | None = None,
case_sensitive: bool | None = True,
env_prefix: str | None = None,
env_nested_delimiter: str | None = '--',
env_parse_none_str: str | None = None,
env_parse_enums: bool | None = None,
version_id: str | None = None,
_init_state: InitState | None = None,
) -> None:
import_aws_secrets_manager()
self._secretsmanager_client = boto3_client('secretsmanager', region_name=region_name, endpoint_url=endpoint_url) # type: ignore
self._secret_id = secret_id
self._version_id = version_id
super().__init__(
settings_cls,
case_sensitive=case_sensitive,
env_prefix=env_prefix,
env_nested_delimiter=env_nested_delimiter,
env_ignore_empty=False,
env_parse_none_str=env_parse_none_str,
env_parse_enums=env_parse_enums,
_init_state=_init_state,
)
def _load_env_vars(self) -> Mapping[str, str | None]:
request = {'SecretId': self._secret_id}
if self._version_id:
request['VersionId'] = self._version_id
response = self._secretsmanager_client.get_secret_value(**request)
return parse_env_vars(
json.loads(response['SecretString']),
self.case_sensitive,
self.env_ignore_empty,
self.env_parse_none_str,
)
def __repr__(self) -> str:
return (
f'{self.__class__.__name__}(secret_id={self._secret_id!r}, '
f'env_nested_delimiter={self.env_nested_delimiter!r})'
)
class AWSSystemsManagerSettingsSource(EnvSettingsSource):
_ssm_path: str
_ssm_client: SSMClient
def __init__(
self,
settings_cls: type[BaseSettings],
ssm_path: str = '/',
region_name: str | None = None,
endpoint_url: str | None = None,
case_sensitive: bool | None = True,
env_prefix: str | None = None,
env_nested_delimiter: str | None = '/',
env_parse_none_str: str | None = None,
env_parse_enums: bool | None = None,
_init_state: InitState | None = None,
) -> None:
if not ssm_path.startswith('/'):
raise SettingsError(f'AWS Systems Manager path must start with "/": {ssm_path!r}')
import_aws_systems_manager()
self._ssm_client = boto3_client('ssm', region_name=region_name, endpoint_url=endpoint_url) # type: ignore
self._ssm_path = ssm_path
super().__init__(
settings_cls,
case_sensitive=case_sensitive,
env_prefix=env_prefix,
env_nested_delimiter=env_nested_delimiter,
env_ignore_empty=False,
env_parse_none_str=env_parse_none_str,
env_parse_enums=env_parse_enums,
_init_state=_init_state,
)
def _load_env_vars(self) -> Mapping[str, str | None]:
prefix = self._ssm_path.rstrip('/') + '/'
paginator = self._ssm_client.get_paginator('get_parameters_by_path')
page_iterator = paginator.paginate(Path=self._ssm_path, Recursive=True, WithDecryption=True)
parameters: dict[str, str | None] = {}
for page in page_iterator:
for parameter in page['Parameters']:
key = parameter['Name'].removeprefix(prefix)
parameters[key] = parameter.get('Value')
return parse_env_vars(
parameters,
self.case_sensitive,
self.env_ignore_empty,
self.env_parse_none_str,
)
def __repr__(self) -> str:
return f'{self.__class__.__name__}(ssm_path={self._ssm_path!r}, env_nested_delimiter={self.env_nested_delimiter!r})'
__all__ = [
'AWSSecretsManagerSettingsSource',
'AWSSystemsManagerSettingsSource',
]
@@ -0,0 +1,246 @@
"""Azure Key Vault settings source."""
from __future__ import annotations as _annotations
from collections.abc import Iterable, Iterator, Mapping
from typing import TYPE_CHECKING, Any, get_args, get_origin
from pydantic._internal._utils import is_model_class
from pydantic.alias_generators import to_snake
from pydantic.dataclasses import is_pydantic_dataclass
from pydantic.fields import FieldInfo
from typing_inspection.introspection import is_union_origin
from ...utils import _lenient_issubclass
from ..utils import (
InitState,
_get_model_config,
_get_model_fields,
_parse_env_none_str,
_resolve_type_alias,
_strip_annotated,
)
from .env import EnvSettingsSource
if TYPE_CHECKING:
from azure.core.credentials import TokenCredential
from azure.core.exceptions import ResourceNotFoundError
from azure.keyvault.secrets import SecretClient
from pydantic_settings.main import BaseSettings
else:
TokenCredential = None
ResourceNotFoundError = None
SecretClient = None
def import_azure_key_vault() -> None:
global TokenCredential
global SecretClient
global ResourceNotFoundError
try:
from azure.core.credentials import TokenCredential
from azure.core.exceptions import ResourceNotFoundError
from azure.keyvault.secrets import SecretClient
except ImportError as e: # pragma: no cover
raise ImportError(
'Azure Key Vault dependencies are not installed, run `pip install pydantic-settings[azure-key-vault]`'
) from e
def _collect_nested_types(annotation: Any, models: list[Any], dict_value_types: list[Any]) -> None:
"""Collect the models and dict value types that nested keys of `annotation` can belong to."""
annotation = _strip_annotated(_resolve_type_alias(annotation))
origin = get_origin(annotation)
if is_union_origin(origin):
for arg in get_args(annotation):
_collect_nested_types(arg, models, dict_value_types)
elif _lenient_issubclass(origin, dict):
# A bare `typing.Dict` has no value type, so its keys are left as-is.
if args := get_args(annotation):
dict_value_types.append(args[-1])
elif is_model_class(annotation) or is_pydantic_dataclass(annotation):
models.append(annotation)
class AzureKeyVaultMapping(Mapping[str, str | None]):
_loaded_secrets: dict[str, str | None]
_secret_client: SecretClient
_secret_names: list[str]
def __init__(
self,
secret_client: SecretClient,
case_sensitive: bool,
snake_case_conversion: bool,
env_prefix: str | None,
env_parse_none_str: str | None = None,
) -> None:
self._loaded_secrets = {}
self._secret_client = secret_client
self._case_sensitive = case_sensitive
self._snake_case_conversion = snake_case_conversion
self._env_prefix = env_prefix or ''
self._env_parse_none_str = env_parse_none_str
self._secret_map: dict[str, str] = self._load_remote()
def _load_remote(self) -> dict[str, str]:
secret_names: Iterator[str] = (
secret.name for secret in self._secret_client.list_properties_of_secrets() if secret.name and secret.enabled
)
if self._snake_case_conversion:
name_map: dict[str, str] = {}
for name in secret_names:
if name.startswith(self._env_prefix):
name_map[f'{self._env_prefix}{to_snake(name[len(self._env_prefix) :])}'] = name
else:
name_map[to_snake(name)] = name
return name_map
if self._case_sensitive:
return {name: name for name in secret_names}
return {name.lower(): name for name in secret_names}
def __getitem__(self, key: str) -> str | None:
new_key = key
if self._snake_case_conversion:
if key.startswith(self._env_prefix):
new_key = f'{self._env_prefix}{to_snake(key[len(self._env_prefix) :])}'
else:
new_key = to_snake(key)
elif not self._case_sensitive:
new_key = key.lower()
if new_key not in self._loaded_secrets:
if new_key in self._secret_map:
self._loaded_secrets[new_key] = self._secret_client.get_secret(self._secret_map[new_key]).value
else:
raise KeyError(key)
return _parse_env_none_str(self._loaded_secrets[new_key], self._env_parse_none_str)
def __len__(self) -> int:
return len(self._secret_map)
def __iter__(self) -> Iterator[str]:
return iter(self._secret_map.keys())
class AzureKeyVaultSettingsSource(EnvSettingsSource):
_url: str
_credential: TokenCredential
def __init__(
self,
settings_cls: type[BaseSettings],
url: str,
credential: TokenCredential,
dash_to_underscore: bool = False,
case_sensitive: bool | None = None,
snake_case_conversion: bool = False,
env_prefix: str | None = None,
env_parse_none_str: str | None = None,
env_parse_enums: bool | None = None,
_init_state: InitState | None = None,
) -> None:
import_azure_key_vault()
self._url = url
self._credential = credential
self._dash_to_underscore = dash_to_underscore
self._snake_case_conversion = snake_case_conversion
super().__init__(
settings_cls,
case_sensitive=True if snake_case_conversion else case_sensitive,
env_prefix=env_prefix,
env_nested_delimiter='__' if snake_case_conversion else '--',
env_ignore_empty=False,
env_parse_none_str=env_parse_none_str,
env_parse_enums=env_parse_enums,
_init_state=_init_state,
)
def _load_env_vars(self) -> Mapping[str, str | None]:
secret_client = SecretClient(vault_url=self._url, credential=self._credential)
return AzureKeyVaultMapping(
secret_client=secret_client,
case_sensitive=self.case_sensitive,
snake_case_conversion=self._snake_case_conversion,
env_prefix=self.env_prefix,
env_parse_none_str=self.env_parse_none_str,
)
def _extract_field_info(
self, field: FieldInfo, field_name: str, model_config: Mapping[str, Any] | None = None
) -> list[tuple[str, str, bool]]:
field_info = super()._extract_field_info(field, field_name, model_config)
if self._dash_to_underscore and not self._snake_case_conversion:
return [
(field_key, env_name.replace('_', '-'), is_complex) for field_key, env_name, is_complex in field_info
]
return field_info
def explode_env_vars(self, field_name: str, field: FieldInfo, env_vars: Mapping[str, str | None]) -> dict[str, Any]:
values = super().explode_env_vars(field_name, field, env_vars)
if self._dash_to_underscore and not self._snake_case_conversion:
self._translate_nested_keys(values, [field.annotation])
return values
def _translate_nested_keys(self, values: dict[str, Any], annotations: Iterable[Any]) -> None:
"""Rename the kebab-case keys of `values` to the keys of the nested model fields they belong to.
The parent prefix has already been removed from the keys in `values`.
"""
models: list[Any] = []
dict_value_types: list[Any] = []
for annotation in annotations:
_collect_nested_types(annotation, models, dict_value_types)
targets = self._nested_field_targets(models)
field_keys = {field_key for field_targets in targets.values() for field_key, _ in field_targets}
# Translate each nested dict once, against every field it may belong to.
nested: dict[int, tuple[dict[str, Any], list[Any]]] = {}
for env_name, field_targets in targets.items():
if env_name not in values:
continue
value = values[env_name]
for field_key, annotation in field_targets:
values.setdefault(field_key, value)
if isinstance(value, dict):
nested.setdefault(id(value), (value, []))[1].append(annotation)
# Keep the dashed key when it is itself a field key, e.g. an alias of another union member.
if env_name not in field_keys:
del values[env_name]
for value, field_annotations in nested.values():
self._translate_nested_keys(value, field_annotations)
if dict_value_types:
for value in values.values():
if isinstance(value, dict):
self._translate_nested_keys(value, dict_value_types)
def _nested_field_targets(self, models: list[Any]) -> dict[str, list[tuple[str, Any]]]:
"""Map the dashed secret name segment of each field of `models` to its field keys and annotations.
A dashed name may belong to several fields, e.g. the same field in two union members.
"""
targets: dict[str, list[tuple[str, Any]]] = {}
for model in models:
config = _get_model_config(model)
for field_name, field in _get_model_fields(model).items():
for field_key, _, _ in self._extract_field_info(field, field_name, config):
env_name = self._apply_case_sensitive(field_key).replace('_', '-')
targets.setdefault(env_name, []).append((field_key, field.annotation))
return targets
def __repr__(self) -> str:
return f'{self.__class__.__name__}(url={self._url!r}, env_nested_delimiter={self.env_nested_delimiter!r})'
__all__ = ['AzureKeyVaultMapping', 'AzureKeyVaultSettingsSource']
@@ -0,0 +1,226 @@
"""Dotenv file settings source."""
from __future__ import annotations as _annotations
import logging
import os
import warnings
from collections.abc import Mapping
from pathlib import Path
from typing import TYPE_CHECKING, Any
from dotenv import dotenv_values
from pydantic._internal._typing_extra import ( # type: ignore[attr-defined]
get_origin,
)
from typing_inspection.introspection import is_union_origin
from ...utils import _settings_debug_enabled, logger
from ..types import ENV_FILE_SENTINEL, DotenvFiltering, DotenvType, EnvPrefixTarget
from ..utils import (
InitState,
_annotation_is_complex,
_get_field_annotation_and_metadata,
_resolve_config_file,
_union_is_complex,
parse_env_vars,
)
from .env import EnvSettingsSource
if TYPE_CHECKING:
from pydantic_settings.main import BaseSettings
class DotEnvSettingsSource(EnvSettingsSource):
"""
Source class for loading settings values from env files.
"""
def __init__(
self,
settings_cls: type[BaseSettings],
env_file: DotenvType | None = ENV_FILE_SENTINEL,
env_file_encoding: str | None = None,
dotenv_filtering: DotenvFiltering | None = None,
case_sensitive: bool | None = None,
env_prefix: str | None = None,
env_prefix_target: EnvPrefixTarget | None = None,
env_nested_delimiter: str | None = None,
env_nested_max_split: int | None = None,
env_ignore_empty: bool | None = None,
env_parse_none_str: str | None = None,
env_parse_enums: bool | None = None,
env_file_depth: int | None = None,
_init_state: InitState | None = None,
) -> None:
self.env_file = env_file if env_file != ENV_FILE_SENTINEL else settings_cls.model_config.get('env_file')
self.env_file_encoding = (
env_file_encoding if env_file_encoding is not None else settings_cls.model_config.get('env_file_encoding')
)
self.env_file_depth = (
env_file_depth if env_file_depth is not None else settings_cls.model_config.get('env_file_depth', 0)
)
self.dotenv_filtering = (
dotenv_filtering if dotenv_filtering is not None else settings_cls.model_config.get('dotenv_filtering')
)
super().__init__(
settings_cls,
case_sensitive,
env_prefix,
env_prefix_target,
env_nested_delimiter,
env_nested_max_split,
env_ignore_empty,
env_parse_none_str,
env_parse_enums,
_init_state,
)
def _load_env_vars(self) -> Mapping[str, str | None]:
return self._read_env_files()
@staticmethod
def _static_read_env_file(
file_path: Path,
*,
encoding: str | None = None,
case_sensitive: bool = False,
ignore_empty: bool = False,
parse_none_str: str | None = None,
) -> Mapping[str, str | None]:
encoding = encoding if encoding is not None else 'utf-8'
file_vars: dict[str, str | None] = dotenv_values(file_path, encoding=encoding)
return parse_env_vars(file_vars, case_sensitive, ignore_empty, parse_none_str)
def _read_env_file(
self,
file_path: Path,
) -> Mapping[str, str | None]:
return self._static_read_env_file(
file_path,
encoding=self.env_file_encoding,
case_sensitive=self.case_sensitive,
ignore_empty=self.env_ignore_empty,
parse_none_str=self.env_parse_none_str,
)
def _read_env_files(self) -> Mapping[str, str | None]:
env_files = self.env_file
if env_files is None:
return {}
if isinstance(env_files, (str, os.PathLike)):
env_files = [env_files]
debug = _settings_debug_enabled() and logger.isEnabledFor(logging.DEBUG)
dotenv_vars: dict[str, str | None] = {}
for env_file in env_files:
env_path = _resolve_config_file(env_file, self.env_file_depth, allow_fifo=True)
if env_path is not None:
if debug:
logger.debug('Loading env file: %s', env_path.resolve())
dotenv_vars.update(self._read_env_file(env_path))
elif debug:
logger.debug('Env file not found, skipping: %s', Path(env_file).expanduser().resolve())
return dotenv_vars
def __call__(self) -> dict[str, Any]: # noqa: C901
data: dict[str, Any] = super().__call__()
if self.dotenv_filtering == 'only_existing':
# This case behaves like the EnvSettingsSource, only return existing fields
return data
if self.dotenv_filtering == 'match_prefix':
# In this case add all env vars that match the prefix, stripping the prefix.
prefix = self._apply_case_sensitive(self.env_prefix)
for env_name, env_value in self.env_vars.items():
if env_name.startswith(prefix):
# Slice with the case-normalized prefix: `str.lower()` can change
# length (e.g. `'İ'` lowers to two code points), so the raw
# `env_prefix` length would under-strip and leave a partial prefix.
normalized_env_name = env_name[len(prefix) :]
if (
self.env_nested_delimiter
and self.env_nested_delimiter in normalized_env_name
and normalized_env_name.partition(self.env_nested_delimiter)[0] in data
):
continue
if normalized_env_name not in data:
data[normalized_env_name] = env_value
return data
is_extra_allowed = self.config.get('extra') != 'forbid'
# `env_vars` keys are case-normalised, so the prefix must be too before it
# is matched against them.
env_prefix = self._apply_case_sensitive(self.env_prefix)
# As `extra` config is allowed in dotenv settings source, We have to
# update data with extra env variables from dotenv file.
for env_name, env_value in self.env_vars.items():
if not env_value or env_name in data or (self.env_prefix and env_name in self.settings_cls.model_fields):
continue
env_used = False
for field_name, field in self.settings_cls.model_fields.items():
annotation, metadata = _get_field_annotation_and_metadata(field)
for _, field_env_name, _ in self._extract_field_info(field, field_name):
if env_name == field_env_name or (
(
_annotation_is_complex(annotation, metadata, self._init_state)
or (
is_union_origin(get_origin(annotation))
and _union_is_complex(annotation, metadata, self._init_state)
)
)
# A var only belongs to a complex field when it sits at the nested
# delimiter boundary (`db<delim>...`, matching what explode_env_vars
# consumes). A bare name-prefix match (e.g. `dbx_token` vs field `db`)
# must not claim the var, otherwise it is silently dropped instead of
# being kept as an extra.
and self.env_nested_delimiter
and env_name.startswith(f'{field_env_name}{self.env_nested_delimiter}')
):
env_used = True
break
if env_used:
break
if not env_used:
if is_extra_allowed and env_name.startswith(env_prefix):
# env_prefix should be respected and removed from the env_name
normalized_env_name = env_name[len(env_prefix) :]
data[normalized_env_name] = env_value
else:
data[env_name] = env_value
return data
def __repr__(self) -> str:
return (
f'{self.__class__.__name__}(env_file={self.env_file!r}, env_file_encoding={self.env_file_encoding!r}, '
f'env_file_depth={self.env_file_depth!r}, env_nested_delimiter={self.env_nested_delimiter!r}, '
f'env_prefix_len={self.env_prefix_len!r})'
)
def read_env_file(
file_path: Path,
*,
encoding: str | None = None,
case_sensitive: bool = False,
ignore_empty: bool = False,
parse_none_str: str | None = None,
) -> Mapping[str, str | None]:
warnings.warn(
'read_env_file will be removed in the next version, use DotEnvSettingsSource._static_read_env_file if you must',
DeprecationWarning,
stacklevel=2,
)
return DotEnvSettingsSource._static_read_env_file(
file_path,
encoding=encoding,
case_sensitive=case_sensitive,
ignore_empty=ignore_empty,
parse_none_str=parse_none_str,
)
__all__ = ['DotEnvSettingsSource', 'read_env_file']
@@ -0,0 +1,393 @@
from __future__ import annotations as _annotations
import json
import os
from collections.abc import Mapping
from typing import (
TYPE_CHECKING,
Any,
cast,
get_args,
get_origin,
)
from pydantic import AliasChoices, AliasPath, Json, TypeAdapter, ValidationError
from pydantic._internal._utils import deep_update, is_model_class
from pydantic.dataclasses import is_pydantic_dataclass
from pydantic.fields import FieldInfo
from typing_inspection.introspection import is_union_origin
from ...utils import _lenient_issubclass
from ..base import PydanticBaseEnvSettingsSource
from ..types import EnvNoneType, EnvPrefixTarget
from ..utils import (
InitState,
_annotation_contains_types,
_annotation_enum_name_to_val,
_annotation_is_complex,
_get_field_annotation_and_metadata,
_get_model_config,
_get_model_fields,
_literal_has_numeric_enum,
_union_has_strict_types,
_union_is_complex,
parse_env_vars,
)
if TYPE_CHECKING:
from pydantic_settings.main import BaseSettings
def _environ_is_case_insensitive() -> bool:
"""Whether ``os.environ`` is the OS-backed, case-insensitive mapping (i.e. on Windows).
When ``os.environ`` has been replaced (e.g. patched to a plain ``dict`` in tests), the
requested case sensitivity is honored as-is.
"""
return os.name == 'nt' and isinstance(os.environ, os._Environ)
class EnvSettingsSource(PydanticBaseEnvSettingsSource):
"""
Source class for loading settings values from environment variables.
"""
def __init__(
self,
settings_cls: type[BaseSettings],
case_sensitive: bool | None = None,
env_prefix: str | None = None,
env_prefix_target: EnvPrefixTarget | None = None,
env_nested_delimiter: str | None = None,
env_nested_max_split: int | None = None,
env_ignore_empty: bool | None = None,
env_parse_none_str: str | None = None,
env_parse_enums: bool | None = None,
_init_state: InitState | None = None,
) -> None:
super().__init__(
settings_cls,
case_sensitive,
env_prefix,
env_prefix_target,
env_ignore_empty,
env_parse_none_str,
env_parse_enums,
_init_state,
)
self.env_nested_delimiter = (
env_nested_delimiter if env_nested_delimiter is not None else self.config.get('env_nested_delimiter')
)
self.env_nested_max_split = (
env_nested_max_split if env_nested_max_split is not None else self.config.get('env_nested_max_split')
)
self.maxsplit = (self.env_nested_max_split or 0) - 1
self.env_prefix_len = len(self.env_prefix)
self.env_vars = self._load_env_vars()
def _load_env_vars(self) -> Mapping[str, str | None]:
# On Windows, ``os.environ`` is case-insensitive: variable names are normalized to
# upper-case and looked up regardless of case. Matching environment variables
# case-sensitively is therefore impossible, so we fall back to case-insensitive
# matching to avoid spurious "field required" errors (see #295). ``.env`` files do
# preserve case and are handled by ``DotEnvSettingsSource``, which overrides this method.
if self.case_sensitive and _environ_is_case_insensitive():
self.case_sensitive = False
return parse_env_vars(os.environ, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str)
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
"""
Gets the value for field from environment variables and a flag to determine whether value is complex.
Args:
field: The field.
field_name: The field name.
Returns:
A tuple that contains the value (`None` if not found), key, and
a flag to determine whether value is complex.
"""
env_val: str | None = None
# `field_key` and `value_is_complex` are deliberately read after the loop, so B007's
# "unused loop variable" report doesn't apply here.
for field_key, env_name, value_is_complex in self._extract_field_info(field, field_name): # noqa: B007
env_val = self.env_vars.get(env_name)
if env_val is not None:
break
return env_val, field_key, value_is_complex
def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
"""
Prepare value for the field.
* Extract value for nested field.
* Deserialize value to python object for complex field.
Args:
field: The field.
field_name: The field name.
Returns:
A tuple contains prepared value for the field.
Raises:
ValuesError: When There is an error in deserializing value for complex field.
"""
is_complex, allow_parse_failure = self._field_is_complex(field)
if self.env_parse_enums:
enum_val = _annotation_enum_name_to_val(field.annotation, value)
value = value if enum_val is None else enum_val
if is_complex or value_is_complex:
if isinstance(value, EnvNoneType):
return value
if value is None:
# field is complex but no value found so far, try explode_env_vars
env_val_built = self.explode_env_vars(field_name, field, self.env_vars)
if env_val_built:
return env_val_built
else:
# field is complex and there's a value, decode that as JSON, then add explode_env_vars
try:
value = self.decode_complex_value(field_name, field, value)
except ValueError:
if not allow_parse_failure:
raise
if isinstance(value, dict):
return deep_update(value, self.explode_env_vars(field_name, field, self.env_vars))
return value
elif value is not None:
# simplest case, field is not complex, we only need to add the value if it was found
return self._coerce_env_val_strict(field, value)
# No value resolved for this field; `None` tells the caller to skip it.
return None
def _matches_alias_path_head(self, field: FieldInfo | Any | None, key: str) -> bool:
"""
Whether ``key`` matches the first element of a multi-part ``AliasPath`` on ``field``.
When a nested field is aliased via e.g. ``AliasPath('path', 0)``, the env value found
under ``path`` is a container that must be JSON-decoded before pydantic can navigate
into it (see #670). This mirrors the complexity that ``_extract_field_info`` already
reports for top-level ``AliasPath`` fields.
Args:
field: The field (or raw type) matched for ``key``.
key: The (case-normalized) env name segment matched for the field.
Returns:
Whether ``key`` is the head of a multi-element ``AliasPath``.
"""
if not isinstance(field, FieldInfo):
return False
v_alias = field.validation_alias
paths: list[tuple[str | int, ...]] = []
if isinstance(v_alias, AliasPath):
paths.append(tuple(v_alias.path))
elif isinstance(v_alias, AliasChoices):
paths.extend(tuple(choice.path) for choice in v_alias.choices if isinstance(choice, AliasPath))
for path in paths:
head = path[0]
if len(path) > 1 and isinstance(head, str) and self._apply_case_sensitive(head) == key:
return True
return False
def _field_is_complex(self, field: FieldInfo) -> tuple[bool, bool]:
"""
Find out if a field is complex, and if so whether JSON errors should be ignored
"""
annotation, metadata = _get_field_annotation_and_metadata(field)
if self.field_is_complex(field):
allow_parse_failure = False
elif is_union_origin(get_origin(annotation)) and _union_is_complex(annotation, metadata, self._init_state):
allow_parse_failure = True
else:
return False, False
return True, allow_parse_failure
# Default value of `case_sensitive` is `None`, because we don't want to break existing behavior.
# We have to change the method to a non-static method and use
# `self.case_sensitive` instead in V3.
def next_field(
self, field: FieldInfo | Any | None, key: str, case_sensitive: bool | None = None
) -> FieldInfo | None:
"""
Find the field in a sub model by key(env name)
By having the following models:
```py
class SubSubModel(BaseSettings):
dvals: Dict
class SubModel(BaseSettings):
vals: list[str]
sub_sub_model: SubSubModel
class Cfg(BaseSettings):
sub_model: SubModel
```
Then:
next_field(sub_model, 'vals') Returns the `vals` field of `SubModel` class
next_field(sub_model, 'sub_sub_model') Returns `sub_sub_model` field of `SubModel` class
Args:
field: The field.
key: The key (env name).
case_sensitive: Whether to search for key case sensitively.
Returns:
Field if it finds the next field otherwise `None`.
"""
if not field:
return None
annotation = field.annotation if isinstance(field, FieldInfo) else field
for type_ in get_args(annotation):
type_has_key = self.next_field(type_, key, case_sensitive)
if type_has_key:
return type_has_key
if _lenient_issubclass(get_origin(annotation), dict):
# get value type if it's a dict
args = get_args(annotation)
return args[-1] if args else None
if is_model_class(annotation) or is_pydantic_dataclass(annotation): # type: ignore[arg-type]
fields = _get_model_fields(annotation)
model_config = _get_model_config(annotation)
# `case_sensitive is None` is here to be compatible with the old behavior.
# Has to be removed in V3.
for field_name, f in fields.items():
for _, env_name, _ in self._extract_field_info(f, field_name, model_config):
if case_sensitive is None or case_sensitive:
if field_name == key or env_name == key:
return f
elif field_name.lower() == key.lower() or env_name.lower() == key.lower():
return f
return None
def explode_env_vars(self, field_name: str, field: FieldInfo, env_vars: Mapping[str, str | None]) -> dict[str, Any]: # noqa: C901
"""
Process env_vars and extract the values of keys containing env_nested_delimiter into nested dictionaries.
This is applied to a single field, hence filtering by env_var prefix.
Args:
field_name: The field name.
field: The field.
env_vars: Environment variables.
Returns:
A dictionary contains extracted values from nested env values.
"""
if not self.env_nested_delimiter:
return {}
ann = field.annotation
is_dict = ann is dict or _lenient_issubclass(get_origin(ann), dict)
prefixes = [
f'{env_name}{self.env_nested_delimiter}' for _, env_name, _ in self._extract_field_info(field, field_name)
]
result: dict[str, Any] = {}
for env_name, env_val in env_vars.items():
try:
prefix = next(prefix for prefix in prefixes if env_name.startswith(prefix))
except StopIteration:
continue
# we remove the prefix before splitting in case the prefix has characters in common with the delimiter
env_name_without_prefix = env_name[len(prefix) :]
*keys, last_key = env_name_without_prefix.split(self.env_nested_delimiter, self.maxsplit)
env_var = result
target_field: FieldInfo | None = field
for key in keys:
target_field = self.next_field(target_field, key, self.case_sensitive)
if isinstance(env_var, dict):
env_var = env_var.setdefault(key, {})
# get proper field with last_key
target_field = self.next_field(target_field, last_key, self.case_sensitive)
# check if env_val maps to a complex field and if so, parse the env_val
if (target_field or is_dict) and env_val:
if isinstance(target_field, FieldInfo):
is_complex, allow_json_failure = self._field_is_complex(target_field)
if not is_complex and self._matches_alias_path_head(target_field, last_key):
# The env value sits under an AliasPath head (e.g. AliasPath('path', 0));
# decode it so pydantic can navigate into the container (see #670).
is_complex, allow_json_failure = True, True
if self.env_parse_enums:
enum_val = _annotation_enum_name_to_val(target_field.annotation, env_val)
env_val = env_val if enum_val is None else enum_val
elif target_field:
# target_field is a raw type (e.g. from dict value type annotation)
is_complex = _annotation_is_complex(target_field, [], self._init_state)
allow_json_failure = True
else:
# nested field type is dict
is_complex, allow_json_failure = True, True
if is_complex:
try:
field_info = target_field if isinstance(target_field, FieldInfo) else None
env_val = self.decode_complex_value(last_key, field_info, env_val) # type: ignore
except ValueError:
if not allow_json_failure:
raise
if isinstance(env_var, dict) and (
last_key not in env_var or not isinstance(env_val, EnvNoneType) or env_var[last_key] == {}
):
env_var[last_key] = self._coerce_env_val_strict(target_field, env_val)
return result
def _coerce_env_val_strict(self, field: FieldInfo | None, value: Any) -> Any:
"""
Coerce environment string values based on field annotation if model config is `strict=True`
or if the field annotation contains strict-annotated types (e.g. Optional[StrictBool]).
Args:
field: The field.
value: The value to coerce.
Returns:
The coerced value if successful, otherwise the original value.
"""
try:
should_coerce = self.config.get('strict')
if isinstance(field, FieldInfo):
annotation, metadata = _get_field_annotation_and_metadata(field)
if not should_coerce:
should_coerce = (
is_union_origin(get_origin(annotation)) and _union_has_strict_types(annotation)
) or _literal_has_numeric_enum(annotation)
if should_coerce and isinstance(value, str) and isinstance(field, FieldInfo):
if value == self.env_parse_none_str:
return value
has_json = any(isinstance(md, Json) for md in metadata) # type: ignore[misc]
if not has_json and not _annotation_contains_types(annotation, (Json,), is_instance=True):
try:
return TypeAdapter(cast('Any', field.annotation)).validate_python(value)
except ValidationError:
# Try JSON decoding as fallback (e.g. 'true' -> True for StrictBool)
decoded = json.loads(value)
if not isinstance(decoded, str):
return TypeAdapter(cast('Any', field.annotation)).validate_python(decoded)
raise
except (ValidationError, json.JSONDecodeError):
# Allow validation error to be raised at time of instantiation
pass
return value
def __repr__(self) -> str:
return (
f'{self.__class__.__name__}(env_nested_delimiter={self.env_nested_delimiter!r}, '
f'env_prefix_len={self.env_prefix_len!r})'
)
__all__ = ['EnvSettingsSource']
@@ -0,0 +1,365 @@
from __future__ import annotations as _annotations
import warnings
from collections.abc import Iterator, Mapping
from functools import cached_property
from typing import TYPE_CHECKING, Any
from pydantic.fields import FieldInfo
from ...exceptions import SettingsError
from ..types import SecretVersion
from ..utils import InitState, _parse_env_none_str, _validate_by_name_enabled
from .env import EnvSettingsSource
if TYPE_CHECKING:
from google.auth import default as google_auth_default
from google.auth.credentials import Credentials
from google.cloud.secretmanager import SecretManagerServiceClient
from pydantic_settings.main import BaseSettings
else:
Credentials = None
SecretManagerServiceClient = None
google_auth_default = None
def import_gcp_secret_manager() -> None:
global Credentials
global SecretManagerServiceClient
global google_auth_default
try:
from google.auth import default as google_auth_default
from google.auth.credentials import Credentials
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=FutureWarning)
from google.cloud.secretmanager import SecretManagerServiceClient
except ImportError as e: # pragma: no cover
raise ImportError(
'GCP Secret Manager dependencies are not installed, run `pip install pydantic-settings[gcp-secret-manager]`'
) from e
def _is_not_found_error(exc: Exception) -> bool:
try:
from google.api_core.exceptions import NotFound
return isinstance(exc, NotFound)
except ImportError: # pragma: no cover
return False
class GoogleSecretManagerMapping(Mapping[str, str | None]):
_loaded_secrets: dict[str, str | None]
_secret_client: SecretManagerServiceClient
def __init__(
self,
secret_client: SecretManagerServiceClient,
project_id: str,
case_sensitive: bool,
env_parse_none_str: str | None = None,
) -> None:
self._loaded_secrets = {}
self._secret_client = secret_client
self._project_id = project_id
self._case_sensitive = case_sensitive
self._env_parse_none_str = env_parse_none_str
@property
def _gcp_project_path(self) -> str:
return self._secret_client.common_project_path(self._project_id)
def _select_case_insensitive_secret(self, lower_name: str, candidates: list[str]) -> str:
if len(candidates) == 1:
return candidates[0]
# Sort to ensure deterministic selection (prefer lowercase / ASCII last)
candidates.sort()
winner = candidates[-1]
warnings.warn(
f"Secret collision: Found multiple secrets {candidates} normalizing to '{lower_name}'. "
f"Using '{winner}' for case-insensitive lookup.",
UserWarning,
stacklevel=2,
)
return winner
@cached_property
def _secret_name_map(self) -> dict[str, str]:
mapping: dict[str, str] = {}
# Group secrets by normalized name to detect collisions
normalized_groups: dict[str, list[str]] = {}
secrets = self._secret_client.list_secrets(parent=self._gcp_project_path)
for secret in secrets:
name = self._secret_client.parse_secret_path(secret.name).get('secret', '')
mapping[name] = name
if not self._case_sensitive:
lower_name = name.lower()
if lower_name not in normalized_groups:
normalized_groups[lower_name] = []
normalized_groups[lower_name].append(name)
if not self._case_sensitive:
for lower_name, candidates in normalized_groups.items():
mapping[lower_name] = self._select_case_insensitive_secret(lower_name, candidates)
return mapping
@property
def _secret_names(self) -> list[str]:
return list(self._secret_name_map.keys())
def _secret_version_path(self, key: str, version: str = 'latest') -> str:
return self._secret_client.secret_version_path(self._project_id, key, version)
def _get_secret_value(self, gcp_secret_name: str, version: str = 'latest') -> str | None:
try:
value = self._secret_client.access_secret_version(
name=self._secret_version_path(gcp_secret_name, version)
).payload.data.decode('UTF-8')
except Exception:
return None
return _parse_env_none_str(value, self._env_parse_none_str)
def _get_secret_value_or_raise(self, gcp_secret_name: str) -> str | None:
try:
value = self._secret_client.access_secret_version(
name=self._secret_version_path(gcp_secret_name)
).payload.data.decode('UTF-8')
except Exception as e:
if _is_not_found_error(e):
raise KeyError(gcp_secret_name) from e
return None
return _parse_env_none_str(value, self._env_parse_none_str)
def __getitem__(self, key: str) -> str | None:
if key in self._loaded_secrets:
return self._loaded_secrets[key]
if self._case_sensitive:
self._loaded_secrets[key] = self._get_secret_value_or_raise(key)
return self._loaded_secrets[key]
gcp_secret_name = self._secret_name_map.get(key)
if gcp_secret_name is None:
gcp_secret_name = self._secret_name_map.get(key.lower())
if gcp_secret_name:
self._loaded_secrets[key] = self._get_secret_value(gcp_secret_name)
else:
raise KeyError(key)
return self._loaded_secrets[key]
def __len__(self) -> int:
return len(self._secret_names)
def __iter__(self) -> Iterator[str]:
return iter(self._secret_names)
class GoogleSecretManagerSettingsSource(EnvSettingsSource):
_credentials: Credentials | None
_secret_client: SecretManagerServiceClient | None
_project_id: str | None
_explicit_project_id: str | None
def __init__(
self,
settings_cls: type[BaseSettings],
credentials: Credentials | None = None,
project_id: str | None = None,
env_prefix: str | None = None,
env_parse_none_str: str | None = None,
env_parse_enums: bool | None = None,
secret_client: SecretManagerServiceClient | None = None,
case_sensitive: bool | None = True,
project_id_field: str = 'project_id',
_init_state: InitState | None = None,
) -> None:
"""Settings source that reads secrets from Google Cloud Secret Manager.
Args:
project_id: The GCP project to read secrets from. If not provided, it is
resolved lazily (see below).
project_id_field: The key, populated by a previous (higher priority)
settings source, to use as the ``project_id`` when one is not passed
explicitly. This must match the key used in ``current_state`` by the
previous source (typically the field name or its preferred alias).
For example, with ``some_field: str = Field(alias='GCP_PROJECT')``,
previous env sources will expose the value under ``'GCP_PROJECT'``.
Defaults to ``'project_id'``.
The ``project_id`` is resolved lazily in :meth:`__call__` rather than at
construction time so that it can be sourced from settings resolved by previous
sources (only available via ``current_state`` once the source is called).
Resolution order is:
1. the explicit ``project_id`` argument
2. the ``project_id_field`` value from previous settings sources
3. ``google.auth.default()``
"""
# Import Google Packages if they haven't already been imported
if SecretManagerServiceClient is None or Credentials is None or google_auth_default is None:
import_gcp_secret_manager()
# Resolution is deferred to __call__ (see _resolve_gcp_project / _load_env_vars).
# _explicit_project_id is the immutable user input; _project_id is the resolved
# value and is written exclusively by _resolve_gcp_project.
self._explicit_project_id = project_id
self._project_id = None
self._credentials = credentials
self._secret_client = secret_client
self._project_id_field = project_id_field
self._env_vars_loaded = False
super().__init__(
settings_cls,
case_sensitive=case_sensitive,
env_prefix=env_prefix,
env_ignore_empty=False,
env_parse_none_str=env_parse_none_str,
env_parse_enums=env_parse_enums,
_init_state=_init_state,
)
# __init__ is past; subsequent _load_env_vars() calls may now resolve the project.
self._env_vars_loaded = True
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
"""Override get_field_value to get the secret value from GCP Secret Manager.
Look for a SecretVersion metadata field to specify a particular SecretVersion.
Args:
field: The field to get the value for
field_name: The declared name of the field
Returns:
A tuple of (value, key, value_is_complex), where `key` is the identifier used
to populate the model (either the field name or an alias, depending on
configuration).
"""
secret_version = next((m.version for m in field.metadata if isinstance(m, SecretVersion)), None)
# If a secret version is specified, try to get that specific version of the secret from
# GCP Secret Manager via the GoogleSecretManagerMapping. This allows different versions
# of the same secret name to be retrieved independently and cached in the GoogleSecretManagerMapping
if secret_version and isinstance(self.env_vars, GoogleSecretManagerMapping):
for field_key, env_name, value_is_complex in self._extract_field_info(field, field_name):
if self.case_sensitive:
gcp_secret_name: str | None = env_name
else:
gcp_secret_name = self.env_vars._secret_name_map.get(env_name)
if gcp_secret_name is None: # pragma: no cover
# Unreachable in practice: `_extract_field_info` already lower-cases
# `env_name` when case-insensitive. Kept as a defensive fallback.
gcp_secret_name = self.env_vars._secret_name_map.get(env_name.lower())
if gcp_secret_name:
env_val = self.env_vars._get_secret_value(gcp_secret_name, secret_version)
if env_val is not None:
# If validation by name is enabled, return field_name to allow multiple fields
# with the same alias but different versions to be distinguished
if _validate_by_name_enabled(self.config):
return env_val, field_name, value_is_complex
return env_val, field_key, value_is_complex
# If a secret version is specified but not found, we should not fall back to "latest" (default behavior)
# as that would be incorrect. We return None to indicate the value was not found.
return None, field_name, False
val, key, is_complex = super().get_field_value(field, field_name)
# If validation by name is enabled, we need to return the field_name as the key
# without this being enabled, you cannot load two secrets with the same name but different versions
if _validate_by_name_enabled(self.config) and val is not None:
return val, field_name, is_complex
return val, key, is_complex
def _resolve_gcp_project(self) -> None:
"""Resolve the credentials, project_id and Secret Manager client.
``project_id`` is resolved, in order of precedence, from: the explicit
``project_id`` argument, the ``project_id_field`` value from previous settings
sources (``current_state``), and finally ``google.auth.default()``.
"""
if self._project_id is not None:
return
project_id = self._explicit_project_id
credentials = self._credentials
# Fall back to a value resolved by a previous (higher priority) settings source.
if project_id is None:
state_project_id = self.current_state.get(self._project_id_field)
if isinstance(state_project_id, str):
project_id = state_project_id
# Fall back to google.auth.default for whatever is still missing.
# Credentials are only needed if we have to build a client ourselves — if the
# caller supplied a pre-built secret_client we can skip the auth call entirely
# when project_id is also known, avoiding an unnecessary RPC / file read.
need_credentials = self._secret_client is None and credentials is None
need_project = project_id is None
if need_credentials or need_project:
_creds, _project_id = google_auth_default()
if need_credentials:
credentials = _creds
if need_project and isinstance(_project_id, str):
project_id = _project_id
if project_id is None:
raise AttributeError(
'project_id is required to be specified either as an argument, via a previous '
'settings source, or from google.auth.default. See '
'https://google-auth.readthedocs.io/en/master/reference/google.auth.html#google.auth.default'
)
self._credentials = credentials
self._project_id = project_id
if self._secret_client is None:
self._secret_client = SecretManagerServiceClient(credentials=self._credentials)
def _load_env_vars(self) -> Mapping[str, str | None]:
# During __init__ the previous sources have not run yet, so defer building the
# mapping until __call__, where current_state (and thus project_id) is available.
if not self._env_vars_loaded:
return {}
self._resolve_gcp_project()
# Unreachable in practice: `_resolve_gcp_project` either raises or sets both
# attributes. Kept as a defensive fallback.
if self._project_id is None or self._secret_client is None: # pragma: no cover
raise SettingsError(
'GoogleSecretManagerSettingsSource: could not determine GCP project_id or initialize the Secret Manager client. '
'Pass project_id explicitly or ensure it is available via application default credentials or a previous settings source.'
)
return GoogleSecretManagerMapping(
self._secret_client,
project_id=self._project_id,
case_sensitive=self.case_sensitive,
env_parse_none_str=self.env_parse_none_str,
)
def __call__(self) -> dict[str, Any]:
# current_state is populated by previous sources right before __call__; (re)build
# the mapping now so project_id can be sourced from them.
self._env_vars_loaded = True
self.env_vars = self._load_env_vars()
return super().__call__()
def __repr__(self) -> str:
# Prefer the resolved project_id, falling back to the explicit constructor
# arg so the repr is informative before the source has been called.
project_id = self._project_id if self._project_id is not None else self._explicit_project_id
return (
f'{self.__class__.__name__}(project_id={project_id!r}, env_nested_delimiter={self.env_nested_delimiter!r})'
)
__all__ = ['GoogleSecretManagerMapping', 'GoogleSecretManagerSettingsSource']
@@ -0,0 +1,61 @@
"""JSON file settings source."""
from __future__ import annotations as _annotations
import json
from typing import (
TYPE_CHECKING,
Any,
)
from ..base import ConfigFileSourceMixin, InitSettingsSource
from ..types import DEFAULT_PATH, ConfigFileSourceType
from ..utils import InitState
if TYPE_CHECKING:
from pathlib import Path
from pydantic_settings.main import BaseSettings
from ..types import Traversable
class JsonConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin):
"""
A source class that loads variables from a JSON file
"""
def __init__(
self,
settings_cls: type[BaseSettings],
json_file: ConfigFileSourceType | None = DEFAULT_PATH,
json_file_encoding: str | None = None,
deep_merge: bool = False,
_init_state: InitState | None = None,
):
self.json_file_path = json_file if json_file != DEFAULT_PATH else settings_cls.model_config.get('json_file')
self.json_file_encoding = (
json_file_encoding
if json_file_encoding is not None
else settings_cls.model_config.get('json_file_encoding')
)
self.json_data = self._read_files(self.json_file_path, deep_merge=deep_merge)
super().__init__(settings_cls, self.json_data, _init_state=_init_state)
def _read_file(self, file_path: Path | Traversable) -> dict[str, Any]:
# Default to UTF-8 rather than the locale encoding, matching `DotEnvSettingsSource`
# and `SecretsSettingsSource`: RFC 8259 mandates UTF-8 for interchanged JSON, and
# the locale default corrupts (or fails to decode) it on a non-UTF-8 Windows code page.
encoding = self.json_file_encoding if self.json_file_encoding is not None else 'utf-8'
with file_path.open(encoding=encoding) as json_file:
content = json_file.read()
# An empty (or whitespace-only) file is not valid JSON; treat it as an empty
# mapping so it falls back to defaults, mirroring `YamlConfigSettingsSource`'s
# `yaml.safe_load(...) or {}` (and `tomllib`, which returns `{}` for an empty file).
return json.loads(content) if content.strip() else {}
def __repr__(self) -> str:
return f'{self.__class__.__name__}(json_file={self.json_file_path})'
__all__ = ['JsonConfigSettingsSource']
@@ -0,0 +1,220 @@
import os
import warnings
from collections.abc import Iterator, Mapping
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, Optional
from ...exceptions import SettingsError
from ...utils import path_type_label
from ..base import PydanticBaseSettingsSource
from ..utils import InitState, parse_env_vars
from .env import EnvSettingsSource
from .secrets import SecretsSettingsSource
if TYPE_CHECKING:
from ...main import BaseSettings
from ...sources import PathType
SECRETS_DIR_MAX_SIZE = 16 * 2**20 # 16 MiB seems to be a reasonable default
class NestedSecretsSettingsSource(EnvSettingsSource):
def __init__(
self,
file_secret_settings: PydanticBaseSettingsSource | SecretsSettingsSource,
secrets_dir: Optional['PathType'] = None,
secrets_dir_missing: Literal['ok', 'warn', 'error'] | None = None,
secrets_dir_max_size: int | None = None,
secrets_case_sensitive: bool | None = None,
secrets_prefix: str | None = None,
secrets_nested_delimiter: str | None = None,
secrets_nested_subdir: bool | None = None,
# args for compatibility with SecretsSettingsSource, don't use directly
case_sensitive: bool | None = None,
env_prefix: str | None = None,
_init_state: InitState | None = None,
) -> None:
# We allow the first argument to be settings_cls like original
# SecretsSettingsSource. However, it is recommended to pass
# SecretsSettingsSource instance instead (as it is shown in usage examples),
# otherwise `_secrets_dir` arg passed to Settings() constructor will be ignored.
settings_cls: type[BaseSettings] = getattr(
file_secret_settings,
'settings_cls',
file_secret_settings, # type: ignore[arg-type]
)
# config options
conf = settings_cls.model_config
self.secrets_dir: PathType | None = first_not_none(
getattr(file_secret_settings, 'secrets_dir', None),
secrets_dir,
conf.get('secrets_dir'),
)
self.secrets_dir_missing: Literal['ok', 'warn', 'error'] = first_not_none(
secrets_dir_missing,
conf.get('secrets_dir_missing'),
'warn',
)
if self.secrets_dir_missing not in ('ok', 'warn', 'error'):
raise SettingsError(f'invalid secrets_dir_missing value: {self.secrets_dir_missing}')
self.secrets_dir_max_size: int = first_not_none(
secrets_dir_max_size,
conf.get('secrets_dir_max_size'),
SECRETS_DIR_MAX_SIZE,
)
self.case_sensitive: bool = first_not_none(
secrets_case_sensitive,
conf.get('secrets_case_sensitive'),
case_sensitive,
conf.get('case_sensitive'),
False,
)
self.secrets_prefix: str = first_not_none(
secrets_prefix,
conf.get('secrets_prefix'),
env_prefix,
conf.get('env_prefix'),
'',
)
# nested options
self.secrets_nested_delimiter: str | None = first_not_none(
secrets_nested_delimiter,
conf.get('secrets_nested_delimiter'),
conf.get('env_nested_delimiter'),
)
self.secrets_nested_subdir: bool = first_not_none(
secrets_nested_subdir,
conf.get('secrets_nested_subdir'),
False,
)
if self.secrets_nested_subdir:
if secrets_nested_delimiter or conf.get('secrets_nested_delimiter'):
raise SettingsError('Options secrets_nested_delimiter and secrets_nested_subdir are mutually exclusive')
self.secrets_nested_delimiter = os.sep
# ensure valid secrets_path
if self.secrets_dir is None:
paths = []
elif isinstance(self.secrets_dir, (Path, str)):
paths = [self.secrets_dir]
else:
paths = list(self.secrets_dir)
self.secrets_paths: list[Path] = [Path(p).expanduser().resolve() for p in paths]
for path in self.secrets_paths:
self.validate_secrets_path(path)
# construct parent, inheriting the init state of the wrapped source if not provided
super().__init__(
settings_cls,
case_sensitive=self.case_sensitive,
env_prefix=self.secrets_prefix,
env_nested_delimiter=self.secrets_nested_delimiter,
env_ignore_empty=False, # match SecretsSettingsSource behaviour
env_parse_enums=True, # we can pass everything here, it will still behave as "True"
env_parse_none_str=None, # match SecretsSettingsSource behaviour
_init_state=_init_state if _init_state is not None else getattr(file_secret_settings, '_init_state', None),
)
self.env_parse_none_str = None # update manually because of None
def _load_env_vars(self) -> Mapping[str, str | None]:
# Overridden so that the base implementation never runs: secrets come from files,
# not ``os.environ``, so the case-insensitivity of the Windows environment must not
# be applied here. NTFS preserves filename case, and the downgrade would silently
# ignore ``secrets_case_sensitive=True``. ``DotEnvSettingsSource`` overrides this
# method for the same reason.
self.env_parse_none_str = None # match SecretsSettingsSource behaviour
# Normalize each directory before merging so that the last directory wins. Merging
# raw filenames first would keep the earlier key's insertion position, so a later
# ``TOKEN`` could not displace an earlier ``token`` once names are case-folded.
secrets: dict[str, str | None] = {}
for path in self.secrets_paths:
secrets.update(
parse_env_vars(
self.load_secrets(path),
self.case_sensitive,
self.env_ignore_empty,
self.env_parse_none_str,
)
)
return secrets
def validate_secrets_path(self, path: Path) -> None:
if not path.exists():
if self.secrets_dir_missing == 'ok':
pass
elif self.secrets_dir_missing == 'warn':
warnings.warn(f'directory "{path}" does not exist', stacklevel=2)
elif self.secrets_dir_missing == 'error':
raise SettingsError(f'directory "{path}" does not exist')
else: # pragma: no cover
raise ValueError # unreachable, checked before
else:
if not path.is_dir():
raise SettingsError(f'secrets_dir must reference a directory, not a {path_type_label(path)}')
secrets_dir_size = sum(f.stat().st_size for f in self._iter_secret_files(path))
if secrets_dir_size > self.secrets_dir_max_size:
raise SettingsError(f'secrets_dir size is above {self.secrets_dir_max_size} bytes')
@staticmethod
def _iter_secret_files(path: Path) -> Iterator[Path]:
"""Yield the secret files contained in ``path``.
``path`` is expected to already be resolved. The directory tree is walked
explicitly so that symbolic links are handled safely:
* a file is only yielded if its real location stays within ``path``; entries
that resolve outside of it (e.g. through a symbolic link) are skipped, so
they neither contribute to the ``secrets_dir_max_size`` accounting nor get
loaded;
* each real directory is visited at most once, so cyclic or repeated
symlinks cannot make the walk loop and inflate the size accounting or the
number of loaded secrets.
Because the size check and the loader share this iterator, they always see
the same set of files.
"""
seen_dirs: set[Path] = set()
def walk(directory: Path) -> Iterator[Path]:
# Guard against symlink loops / a directory reachable through multiple
# links being traversed more than once.
resolved_dir = directory.resolve()
if resolved_dir in seen_dirs:
return
seen_dirs.add(resolved_dir)
try:
entries = sorted(directory.iterdir())
except OSError:
return
for entry in entries:
resolved = entry.resolve()
if resolved.is_dir():
# Only descend into directories that stay within secrets_dir.
# A symlinked directory pointing outside of ``path`` is not
# followed at all, so we never walk (potentially large) external
# trees and never read files from outside secrets_dir.
if resolved == path or path in resolved.parents:
yield from walk(entry)
elif resolved.is_file() and path in resolved.parents:
# Defense in depth: a file whose real location escapes
# secrets_dir (e.g. a symlink pointing outside of ``path``) is
# skipped from both the size accounting and the load.
yield entry
yield from walk(path)
@classmethod
def load_secrets(cls, path: Path) -> dict[str, str]:
# Explicit encoding, matching SecretsSettingsSource: the locale default would
# corrupt UTF-8 secrets on a non-UTF-8 Windows code page.
return {str(p.relative_to(path)): p.read_text(encoding='utf-8').strip() for p in cls._iter_secret_files(path)}
def __repr__(self) -> str:
return f'NestedSecretsSettingsSource(secrets_dir={self.secrets_dir!r})'
def first_not_none(*objs: Any) -> Any:
return next(filter(lambda o: o is not None, objs), None)
@@ -0,0 +1,64 @@
"""Pyproject TOML file settings source."""
from __future__ import annotations as _annotations
from pathlib import Path
from typing import (
TYPE_CHECKING,
)
from ..utils import InitState
from .toml import TomlConfigSettingsSource
if TYPE_CHECKING:
from pydantic_settings.main import BaseSettings
class PyprojectTomlConfigSettingsSource(TomlConfigSettingsSource):
"""
A source class that loads variables from a `pyproject.toml` file.
"""
def __init__(
self,
settings_cls: type[BaseSettings],
toml_file: Path | None = None,
_init_state: InitState | None = None,
) -> None:
self.toml_file_path = self._pick_pyproject_toml_file(
toml_file, settings_cls.model_config.get('pyproject_toml_depth', 0)
)
self.toml_table_header: tuple[str, ...] = settings_cls.model_config.get(
'pyproject_toml_table_header', ('tool', 'pydantic-settings')
)
self.toml_data = self._read_files(self.toml_file_path)
for key in self.toml_table_header:
self.toml_data = self.toml_data.get(key, {})
super(TomlConfigSettingsSource, self).__init__(settings_cls, self.toml_data, _init_state=_init_state)
@staticmethod
def _pick_pyproject_toml_file(provided: Path | None, depth: int) -> Path:
"""Pick a `pyproject.toml` file path to use.
Args:
provided: Explicit path provided when instantiating this class.
depth: Number of directories up the tree to check for a pyproject.toml.
"""
if provided:
return provided.resolve()
rv = Path.cwd() / 'pyproject.toml'
count = 0
if not rv.is_file():
child = rv.parent.parent / 'pyproject.toml'
while count < depth:
if child.is_file():
return child
if str(child.parent) == rv.root:
break # end discovery after checking system root once
child = child.parent.parent / 'pyproject.toml'
count += 1
return rv
__all__ = ['PyprojectTomlConfigSettingsSource']
@@ -0,0 +1,146 @@
"""Secrets file settings source."""
from __future__ import annotations as _annotations
import logging
import os
import warnings
from pathlib import Path
from typing import (
TYPE_CHECKING,
Any,
)
from pydantic.fields import FieldInfo
from pydantic_settings.utils import _settings_debug_enabled, logger, path_type_label
from ...exceptions import SettingsError
from ..base import PydanticBaseEnvSettingsSource
from ..types import EnvPrefixTarget, PathType
from ..utils import InitState
if TYPE_CHECKING:
from pydantic_settings.main import BaseSettings
class SecretsSettingsSource(PydanticBaseEnvSettingsSource):
"""
Source class for loading settings values from secret files.
"""
def __init__(
self,
settings_cls: type[BaseSettings],
secrets_dir: PathType | None = None,
case_sensitive: bool | None = None,
env_prefix: str | None = None,
env_prefix_target: EnvPrefixTarget | None = None,
env_ignore_empty: bool | None = None,
env_parse_none_str: str | None = None,
env_parse_enums: bool | None = None,
_init_state: InitState | None = None,
) -> None:
super().__init__(
settings_cls,
case_sensitive,
env_prefix,
env_prefix_target,
env_ignore_empty,
env_parse_none_str,
env_parse_enums,
_init_state,
)
self.secrets_dir = secrets_dir if secrets_dir is not None else self.config.get('secrets_dir')
def __call__(self) -> dict[str, Any]:
"""
Build fields from "secrets" files.
"""
secrets: dict[str, str | None] = {}
if self.secrets_dir is None:
return secrets
secrets_dirs = [self.secrets_dir] if isinstance(self.secrets_dir, (str, os.PathLike)) else self.secrets_dir
secrets_paths = [Path(p).expanduser() for p in secrets_dirs]
self.secrets_paths = []
for path in secrets_paths:
if not path.exists():
# No `stacklevel`: this source is reached at different depths — via
# `Settings()` (through `_settings_build_values`) or by instantiating
# the source directly — so no fixed value attributes the warning to
# user code in both cases.
warnings.warn(f'directory "{path}" does not exist') # noqa: B028
else:
self.secrets_paths.append(path)
if not len(self.secrets_paths):
return secrets
for path in self.secrets_paths:
if not path.is_dir():
raise SettingsError(f'secrets_dir must reference a directory, not a {path_type_label(path)}')
return super().__call__()
@classmethod
def find_case_path(cls, dir_path: Path, file_name: str, case_sensitive: bool) -> Path | None:
"""
Find a file within path's directory matching filename, optionally ignoring case.
Args:
dir_path: Directory path.
file_name: File name.
case_sensitive: Whether to search for file name case sensitively.
Returns:
Whether file path or `None` if file does not exist in directory.
"""
for f in dir_path.iterdir():
if f.name == file_name or (not case_sensitive and f.name.lower() == file_name.lower()):
return f
return None
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
"""
Gets the value for field from secret file and a flag to determine whether value is complex.
Args:
field: The field.
field_name: The field name.
Returns:
A tuple that contains the value (`None` if the file does not exist), key, and
a flag to determine whether value is complex.
"""
debug = _settings_debug_enabled() and logger.isEnabledFor(logging.DEBUG)
for field_key, env_name, value_is_complex in self._extract_field_info(field, field_name):
# paths reversed to match the last-wins behaviour of `env_file`
for secrets_path in reversed(self.secrets_paths):
path = self.find_case_path(secrets_path, env_name, self.case_sensitive)
if not path:
# path does not exist, we currently don't return a warning for this
if debug:
logger.debug('Secret file not found, skipping: %s', (secrets_path / env_name).resolve())
continue
if path.is_file():
if debug:
logger.debug('Loading secret file: %s', path.resolve())
# Explicit encoding: `read_text()` would otherwise decode with the
# locale encoding, so a UTF-8 secret is corrupted on a non-UTF-8
# Windows code page (silently, when every byte happens to map).
return path.read_text(encoding='utf-8').strip(), field_key, value_is_complex
warnings.warn(
f'attempted to load secret file "{path}" but found a {path_type_label(path)} instead.',
stacklevel=4,
)
return None, field_key, value_is_complex
def __repr__(self) -> str:
return f'{self.__class__.__name__}(secrets_dir={self.secrets_dir!r})'
@@ -0,0 +1,72 @@
"""TOML file settings source."""
from __future__ import annotations as _annotations
from pathlib import Path
from typing import (
TYPE_CHECKING,
Any,
)
from ..base import ConfigFileSourceMixin, InitSettingsSource
from ..types import DEFAULT_PATH, ConfigFileSourceType
from ..utils import InitState
if TYPE_CHECKING:
import tomllib
from pydantic_settings.main import BaseSettings
from ..types import Traversable
else:
tomllib = None
def import_toml() -> None:
global tomllib
if tomllib is not None:
return
import tomllib
class TomlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin):
"""
A source class that loads variables from a TOML file
"""
def __init__(
self,
settings_cls: type[BaseSettings],
toml_file: ConfigFileSourceType | None = DEFAULT_PATH,
toml_table_header: tuple[str, ...] | None = None,
deep_merge: bool = False,
_init_state: InitState | None = None,
):
self.toml_file_path = toml_file if toml_file != DEFAULT_PATH else settings_cls.model_config.get('toml_file')
self.toml_table_header = (
toml_table_header
if toml_table_header is not None
else settings_cls.model_config.get('toml_table_header', ())
)
self.toml_data = self._read_files(self.toml_file_path, deep_merge=deep_merge)
if self._any_file_exists(self.toml_file_path):
for key in self.toml_table_header:
if key not in self.toml_data:
raise KeyError(f'toml_table_header key "{key}" not found in {self.toml_file_path}')
self.toml_data = self.toml_data[key]
super().__init__(settings_cls, self.toml_data, _init_state=_init_state)
def _read_file(self, file_path: Path | Traversable) -> dict[str, Any]:
import_toml()
with file_path.open(mode='rb') as toml_file:
return tomllib.load(toml_file)
@classmethod
def _any_file_exists(cls, paths: ConfigFileSourceType | None) -> bool:
"""Check if any of the given file paths would be read by `_read_files`."""
return bool(cls._existing_files(paths))
def __repr__(self) -> str:
return f'{self.__class__.__name__}(toml_file={self.toml_file_path}, toml_table_header={self.toml_table_header})'
@@ -0,0 +1,147 @@
"""YAML file settings source."""
from __future__ import annotations as _annotations
from typing import (
TYPE_CHECKING,
Any,
)
from ..base import ConfigFileSourceMixin, InitSettingsSource
from ..types import DEFAULT_PATH, ConfigFileSourceType
from ..utils import InitState
if TYPE_CHECKING:
from pathlib import Path
import yaml
from pydantic_settings.main import BaseSettings
from ..types import Traversable
else:
yaml = None
def import_yaml() -> None:
global yaml
if yaml is not None:
return
try:
import yaml
except ImportError as e:
raise ImportError('PyYAML is not installed, run `pip install pydantic-settings[yaml]`') from e
class YamlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin):
"""
A source class that loads variables from a yaml file
"""
def __init__(
self,
settings_cls: type[BaseSettings],
yaml_file: ConfigFileSourceType | None = DEFAULT_PATH,
yaml_file_encoding: str | None = None,
yaml_config_section: str | None = None,
deep_merge: bool = False,
_init_state: InitState | None = None,
):
self.yaml_file_path = yaml_file if yaml_file != DEFAULT_PATH else settings_cls.model_config.get('yaml_file')
self.yaml_file_encoding = (
yaml_file_encoding
if yaml_file_encoding is not None
else settings_cls.model_config.get('yaml_file_encoding')
)
self.yaml_config_section = (
yaml_config_section
if yaml_config_section is not None
else settings_cls.model_config.get('yaml_config_section')
)
self.yaml_data = self._read_files(self.yaml_file_path, deep_merge=deep_merge)
if self.yaml_config_section is not None:
section_data = self._traverse_nested_section(
self.yaml_data, self.yaml_config_section, self.yaml_config_section
)
# An empty section (`section:` with no children) parses to `None`; treat it as an
# empty mapping so it falls back to defaults, mirroring `_read_file`'s `or {}` for
# an empty whole file. A non-mapping section (e.g. a scalar) is a user error, so
# raise a clear message instead of letting `InitSettingsSource` choke on `.keys()`.
if section_data is None:
section_data = {}
elif not isinstance(section_data, dict):
raise TypeError(
f'yaml_config_section "{self.yaml_config_section}" in {self.yaml_file_path} '
f'must be a mapping, got {type(section_data).__name__}.'
)
self.yaml_data = section_data
super().__init__(settings_cls, self.yaml_data, _init_state=_init_state)
def _read_file(self, file_path: Path | Traversable) -> dict[str, Any]:
import_yaml()
# Default to UTF-8 rather than the locale encoding, matching `DotEnvSettingsSource`
# and `SecretsSettingsSource`: the YAML spec requires UTF-8/UTF-16, and the locale
# default corrupts (or fails to decode) it on a non-UTF-8 Windows code page.
encoding = self.yaml_file_encoding if self.yaml_file_encoding is not None else 'utf-8'
with file_path.open(encoding=encoding) as yaml_file:
return yaml.safe_load(yaml_file) or {}
def _traverse_nested_section(
self, data: dict[str, Any], section_path: str, original_path: str | None = None
) -> dict[str, Any]:
"""
Traverse nested YAML sections using dot-notation path.
This method tries to match the longest possible key first before splitting on dots,
allowing access to YAML keys that contain literal dot characters.
For example, with section_path="a.b.c", it will try:
1. "a.b.c" as a literal key
2. "a.b" as a key, then traverse to "c"
3. "a" as a key, then traverse to "b.c"
4. "a" as a key, then "b" as a key, then "c" as a key
"""
# Track the original path for error messages
if original_path is None:
original_path = section_path
# Only reject truly empty paths
if not section_path:
raise ValueError('yaml_config_section cannot be empty')
# Try the full path as a literal key first (even with leading/trailing/consecutive dots)
try:
return data[section_path]
except KeyError:
pass # Not a literal key, try splitting
except TypeError as e:
raise TypeError(
f'yaml_config_section path "{original_path}" cannot be traversed in {self.yaml_file_path}. '
f'An intermediate value is not a dictionary.'
) from e
# If path contains no dots, we already tried it as a literal key above
if '.' not in section_path:
raise KeyError(f'yaml_config_section key "{original_path}" not found in {self.yaml_file_path}')
# Try progressively shorter prefixes (greedy left-to-right approach)
parts = section_path.split('.')
for i in range(len(parts) - 1, 0, -1):
prefix = '.'.join(parts[:i])
suffix = '.'.join(parts[i:])
if prefix in data:
# Found the prefix as a literal key, now recursively traverse the suffix
# The recursive call already raises a `TypeError` carrying `original_path`,
# so let it propagate rather than re-wrapping it in an identical message.
return self._traverse_nested_section(data[prefix], suffix, original_path)
# If we get here, no match was found
raise KeyError(f'yaml_config_section key "{original_path}" not found in {self.yaml_file_path}')
def __repr__(self) -> str:
return f'{self.__class__.__name__}(yaml_file={self.yaml_file_path})'
__all__ = ['YamlConfigSettingsSource']