"""HIMPORT client-side fieldset registry for redis-py. `HIMPORT` lets a client register an ordered list of hash field names once per connection (a *fieldset*) and then create many hashes by sending only values. Because a fieldset is server-side session state bound to a single physical connection, redis-py keeps a client-level registry of the fieldsets the application has declared and prepares them lazily, per connection, on first use by ``himport_set``. This module holds only that registry. It is pure in-memory state with no I/O and no ``asyncio`` primitives, so a single class is shared by both the sync and async clients. Example:: >>> from redis.himport import HImportRegistry >>> registry = HImportRegistry() >>> registry.prepare("account_data", ["name", "email", "age"]) >>> registry.get("account_data").fields ('name', 'email', 'age') """ import threading from collections.abc import Iterable, Iterator from dataclasses import dataclass from redis.exceptions import DataError from redis.typing import EncodableT, FieldT, KeyT # --------------------------------------------------------------------------- # Wire command tokens # --------------------------------------------------------------------------- # Centralised so the sync/async clients, the cluster clients and the # CoreCommands mixin never repeat the literal HIMPORT strings. ``HIMPORT_*`` are # the full command names passed to ``execute_command`` and registered as # response callbacks (the request packer splits the space). The # ``himport_*_command`` builders return the positional wire args used when a # connection packs commands directly (lazy PREPARE bundled with SET, deferred # DISCARD reconcile, etc.). _HIMPORT = "HIMPORT" _PREPARE = "PREPARE" _SET = "SET" _DISCARD = "DISCARD" _DISCARDALL = "DISCARDALL" HIMPORT_PREPARE = f"{_HIMPORT} {_PREPARE}" HIMPORT_SET = f"{_HIMPORT} {_SET}" HIMPORT_DISCARD = f"{_HIMPORT} {_DISCARD}" HIMPORT_DISCARDALL = f"{_HIMPORT} {_DISCARDALL}" def himport_prepare_command(fieldset_name: str, fields: Iterable[FieldT]) -> tuple: """Positional wire args for ``HIMPORT PREPARE fieldset_name field [field ...]``.""" return (_HIMPORT, _PREPARE, fieldset_name, *fields) def himport_set_command( key: KeyT, fieldset_name: str, values: Iterable[EncodableT] ) -> tuple: """Positional wire args for ``HIMPORT SET key fieldset_name value [value ...]``.""" return (_HIMPORT, _SET, key, fieldset_name, *values) def himport_discard_command(fieldset_name: str) -> tuple: """Positional wire args for ``HIMPORT DISCARD fieldset_name``.""" return (_HIMPORT, _DISCARD, fieldset_name) _HIMPORT_SET_LEN = len(HIMPORT_SET) _HIMPORT_SET_BYTES = HIMPORT_SET.encode() def is_himport_set_command(command_name) -> bool: """Return ``True`` if ``command_name`` names the ``HIMPORT SET`` command. Redis command names are case-insensitive on the wire, and a caller using the raw ``execute_command`` API may pass the name in any case and as either ``str`` or ``bytes`` (e.g. ``execute_command("himport set", ...)`` or ``b"HIMPORT SET"``). The connection-state-aware ``HIMPORT SET`` path keys off the command name, so it must recognise all of those spellings: an exact comparison against :data:`HIMPORT_SET` would miss them and send a bare SET that fails with ``no such fieldset`` for a fieldset registered in the client but not yet PREPAREd on the borrowed connection. The length check keeps the per-command cost on the hot path to a single comparison for the common case of a differently-sized command name (``upper()`` runs only on a size match). """ if isinstance(command_name, str): return ( len(command_name) == _HIMPORT_SET_LEN and command_name.upper() == HIMPORT_SET ) if isinstance(command_name, (bytes, bytearray, memoryview)): raw = bytes(command_name) return len(raw) == _HIMPORT_SET_LEN and raw.upper() == _HIMPORT_SET_BYTES return False _HIMPORT_BYTES = _HIMPORT.encode() _SET_BYTES = _SET.encode() def _token_is(value, upper_token: str, upper_token_bytes: bytes) -> bool: """Case-insensitive match of a single ``str``/``bytes`` command token. ``upper_token`` / ``upper_token_bytes`` must already be upper-cased. The length guard keeps the hot path to a single comparison for a differently sized token (``upper()`` runs only on a size match). """ n = len(upper_token) if isinstance(value, str): return len(value) == n and value.upper() == upper_token if isinstance(value, (bytes, bytearray, memoryview)): raw = bytes(value) return len(raw) == n and raw.upper() == upper_token_bytes return False def parse_himport_set_args(args): """Detect an ``HIMPORT SET`` command in ``args`` and return its operands. ``HIMPORT SET`` reaches the raw ``execute_command`` API in two wire-equivalent forms that the serializer both accept: * the joined form ``("HIMPORT SET", key, fieldset, *values)`` (``args[0]`` is the two-word command name the request packer splits on the space), and * the split form ``("HIMPORT", "SET", key, fieldset, *values)``. Both are case- and encoding-insensitive. The connection-state-aware executor and the pipeline pre-flight need the operands at the right offsets for either form, so this returns ``(key, fieldset_name, values_list)`` when ``args`` is an ``HIMPORT SET`` with enough operands, or ``None`` otherwise (a non-``HIMPORT SET`` command, or one with too few operands -- which falls through to the plain send so the server returns its own arity error). """ if not args: return None first = args[0] # Joined form: args[0] == "HIMPORT SET". if is_himport_set_command(first): if len(args) < 3: return None return args[1], args[2], list(args[3:]) # Split form: args[0] == "HIMPORT", args[1] == "SET". if ( len(args) >= 2 and _token_is(first, _HIMPORT, _HIMPORT_BYTES) and _token_is(args[1], _SET, _SET_BYTES) ): if len(args) < 4: return None return args[2], args[3], list(args[4:]) return None @dataclass(frozen=True) class HImportFieldset: """An immutable HIMPORT fieldset entry. Attributes: name: Fieldset name used by ``HIMPORT SET`` / ``HIMPORT DISCARD``. fields: Ordered field names, exactly as supplied by the caller. They are never reordered or deduplicated (HLD R.2): the server canonicalizes field order internally and rejects duplicate field names, so the client only preserves the caller's positional order. version: Monotonic stamp bumped each time the fieldset is (re)declared. Connections compare the version they last prepared against this value to detect a stale prepared state; the stamp is drawn from the registry's mutation clock (:attr:`HImportRegistry.revision`), which never repeats, so discarding and re-declaring the same name yields a fresh version rather than a colliding one. This is the prepare-side signal; the discard-side counterpart is the clock advancing on removal, since a removed fieldset leaves no entry to stamp. """ name: str fields: tuple[FieldT, ...] version: int class HImportRegistry: """Client-level registry of HIMPORT fieldsets. Pure in-memory state, shared by the sync and async clients. It is mutated only through the client's ``himport_prepare`` / ``himport_discard`` / ``himport_discard_all`` methods and exposed read-only through the client's ``himport_registry`` property. Mutations are serialized under a lock so the revision bump and dict change stay consistent when a sync ``Redis`` instance is shared across threads. Reads that iterate or snapshot the registry take the same lock, so a concurrent mutation cannot make them observe a torn view or raise ``dictionary changed size during iteration``; single-key/scalar reads (``get``, ``__contains__``, ``__len__``, :attr:`revision`) are atomic and stay lock-free. The async client runs single-threaded and never contends. The registry always starts empty; fieldsets are declared at runtime through the client's ``himport_prepare`` method. """ def __init__(self) -> None: self._fieldsets: dict[str, HImportFieldset] = {} # Serializes mutations (prepare/discard/discard_all) so the revision bump and # the dict change are applied atomically under concurrent access from a # thread-shared sync client, and guards the reads that iterate/snapshot the # dict so they never see a torn view or a mid-iteration resize. Held only # across synchronous, non-blocking bodies, never across I/O, so it is harmless # for the single-threaded async client. self._lock = threading.Lock() # Monotonic mutation clock. It advances on every registry change (prepare # and discard). prepare stamps the new entry with the advanced value as its # per-fieldset version; discard has no surviving entry to stamp, so the # advance itself is the signal that a removal happened. Because the clock # only ever increases, a re-declared name never reuses a stamp, while # comparing per-fieldset versions still avoids forcing unrelated fieldsets # to re-prepare. self._revision: int = 0 # -- internal helpers ------------------------------------------------- # ``_advance`` and ``_set`` assume ``_lock`` is already held (they run under the # public mutation methods); they never acquire the lock themselves. def _advance(self) -> int: self._revision += 1 return self._revision @staticmethod def _materialize_fields(fields: Iterable[FieldT]) -> tuple: """Validate and materialize the caller's field iterable into a tuple. Done *before* the mutation lock is taken: consuming an arbitrary iterable can be slow, or -- for a generator that inspects this same registry -- can re-enter a locked read (e.g. ``yield`` then ``registry.names()``). Running it under the non-reentrant ``_lock`` would stall every registry user or deadlock permanently. Field order is preserved; nothing is reordered or deduplicated. """ # A bare single field name (str/bytes/bytearray/memoryview) is itself # iterable element-by-element; that is almost certainly a caller mistake and # would silently register single-character/single-byte "fields" (e.g. # memoryview(b"id") -> field names 105, 100), so reject it as invalid local # API usage. int/float are not iterable, so tuple() below rejects them. if isinstance(fields, (str, bytes, bytearray, memoryview)): raise DataError( "HIMPORT fields must be a collection of field names, " "not a single string or binary value" ) field_tuple = tuple(fields) if not field_tuple: raise DataError("HIMPORT fieldset must have at least one field") return field_tuple def _set(self, name: str, field_tuple: tuple) -> HImportFieldset: # ``field_tuple`` is already validated/materialized by # :meth:`_materialize_fields`; only the (cheap, non-blocking) revision bump # and dict mutation run here, so ``_lock`` is never held across arbitrary # caller code. fieldset = HImportFieldset( name=name, fields=field_tuple, version=self._advance(), ) self._fieldsets[name] = fieldset return fieldset # -- mutation --------------------------------------------------------- def prepare(self, name: str, fields: Iterable[FieldT]) -> HImportFieldset: """Add or replace a fieldset, bumping its version, and return the entry. Re-declaring an existing name replaces its fields and bumps its version. Field order is preserved verbatim; nothing is reordered or deduplicated. """ # Materialize/validate the iterable *outside* the lock so a slow or # registry-re-entrant generator can never stall or deadlock other users; # the lock then covers only the atomic revision-bump + dict mutation. field_tuple = self._materialize_fields(fields) with self._lock: return self._set(name, field_tuple) def discard(self, name: str) -> bool: """Remove a fieldset from the registry. Returns ``True`` if a fieldset was removed, ``False`` if ``name`` was not registered. Advances :attr:`revision` when a fieldset is actually removed. """ with self._lock: if name not in self._fieldsets: return False del self._fieldsets[name] self._advance() return True def discard_all(self) -> int: """Remove all fieldsets and return the number removed. Advances :attr:`revision` when at least one fieldset is removed. """ with self._lock: count = len(self._fieldsets) if count: self._fieldsets.clear() self._advance() return count # -- read-only access ------------------------------------------------- # Reads that iterate or build a snapshot of the dict take ``_lock`` so a # concurrent mutation cannot resize it mid-iteration or expose a torn view; # single-key/scalar reads below (get/__contains__/__len__/revision) are atomic # and deliberately stay lock-free. @property def revision(self) -> int: """Monotonic mutation clock, advanced on every registry change. It is the discard-side counterpart to per-fieldset :attr:`HImportFieldset.version`. A connection records the revision it last reconciled against; when it differs, a discard (or prepare) has occurred since, so the connection recomputes which of its prepared fieldsets are no longer registered — see :meth:`names_to_discard` — and discards those when it is released back to the pool. """ return self._revision def names_to_discard(self, prepared_names: Iterable[str]) -> list[str]: """Return which of ``prepared_names`` are no longer registered. Given the fieldset names a connection has prepared on the server, this is the set that must be sent ``HIMPORT DISCARD`` (typically when the connection is released), because they have been removed from the registry. """ # Hold the lock across the comprehension so every membership test sees one # consistent snapshot; ``prepared_names`` is an external iterable of plain # strings and never calls back into the registry. with self._lock: return [name for name in prepared_names if name not in self._fieldsets] def get(self, name: str) -> HImportFieldset | None: """Return the fieldset registered under ``name``, or ``None``.""" return self._fieldsets.get(name) def names(self) -> list[str]: """Return the registered fieldset names.""" with self._lock: return list(self._fieldsets) def items(self) -> list[tuple[str, HImportFieldset]]: """Return a snapshot of ``(name, fieldset)`` pairs.""" with self._lock: return list(self._fieldsets.items()) def __contains__(self, name: object) -> bool: return name in self._fieldsets def __len__(self) -> int: return len(self._fieldsets) def __iter__(self) -> Iterator[str]: # Snapshot under the lock and iterate the copy, so the lock is never held # across caller consumption and a concurrent mutation cannot resize the # underlying dict mid-iteration. with self._lock: return iter(list(self._fieldsets)) def __repr__(self) -> str: with self._lock: entries = list(self._fieldsets.items()) body = ", ".join( f"{name}={list(fieldset.fields)}" for name, fieldset in entries ) return f"{self.__class__.__name__}({body})"