add
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
from redis.asyncio.client import Redis, StrictRedis
|
||||
from redis.asyncio.cluster import RedisCluster
|
||||
from redis.asyncio.connection import (
|
||||
BlockingConnectionPool,
|
||||
Connection,
|
||||
ConnectionPool,
|
||||
SSLConnection,
|
||||
UnixDomainSocketConnection,
|
||||
)
|
||||
from redis.asyncio.keyspace_notifications import (
|
||||
AsyncClusterKeyspaceNotifications,
|
||||
AsyncKeyspaceNotifications,
|
||||
AsyncKeyspaceNotificationsInterface,
|
||||
)
|
||||
from redis.asyncio.sentinel import (
|
||||
Sentinel,
|
||||
SentinelConnectionPool,
|
||||
SentinelManagedConnection,
|
||||
SentinelManagedSSLConnection,
|
||||
)
|
||||
from redis.asyncio.utils import from_url
|
||||
from redis.backoff import default_backoff
|
||||
from redis.exceptions import (
|
||||
AuthenticationError,
|
||||
AuthenticationWrongNumberOfArgsError,
|
||||
BusyLoadingError,
|
||||
ChildDeadlockedError,
|
||||
ConnectionError,
|
||||
DataError,
|
||||
InvalidResponse,
|
||||
OutOfMemoryError,
|
||||
PubSubError,
|
||||
ReadOnlyError,
|
||||
RedisError,
|
||||
ResponseError,
|
||||
TimeoutError,
|
||||
WatchError,
|
||||
)
|
||||
from redis.typing import Subscription
|
||||
|
||||
__all__ = [
|
||||
"AsyncClusterKeyspaceNotifications",
|
||||
"AsyncKeyspaceNotifications",
|
||||
"AsyncKeyspaceNotificationsInterface",
|
||||
"AuthenticationError",
|
||||
"AuthenticationWrongNumberOfArgsError",
|
||||
"BlockingConnectionPool",
|
||||
"BusyLoadingError",
|
||||
"ChildDeadlockedError",
|
||||
"Connection",
|
||||
"ConnectionError",
|
||||
"ConnectionPool",
|
||||
"DataError",
|
||||
"from_url",
|
||||
"default_backoff",
|
||||
"InvalidResponse",
|
||||
"PubSubError",
|
||||
"OutOfMemoryError",
|
||||
"ReadOnlyError",
|
||||
"Redis",
|
||||
"RedisCluster",
|
||||
"RedisError",
|
||||
"ResponseError",
|
||||
"Sentinel",
|
||||
"SentinelConnectionPool",
|
||||
"SentinelManagedConnection",
|
||||
"SentinelManagedSSLConnection",
|
||||
"SSLConnection",
|
||||
"StrictRedis",
|
||||
"Subscription",
|
||||
"TimeoutError",
|
||||
"UnixDomainSocketConnection",
|
||||
"WatchError",
|
||||
]
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,260 @@
|
||||
"""Shared HIMPORT wire-execution helpers for the asynchronous clients.
|
||||
|
||||
Async mirror of :mod:`redis._himport_exec`. The PREPARE / SET / DISCARD
|
||||
packed-write drain loops, per-connection version bookkeeping, and
|
||||
``NoSuchFieldsetError`` re-prepare-and-retry are identical for the standalone
|
||||
(:class:`redis.asyncio.Redis`) and cluster (async ``ClusterNode``) clients,
|
||||
differing only in (a) which object provides ``parse_response`` and (b) the
|
||||
cluster-only ASK-redirect handling. These coroutines take that object as
|
||||
``node`` and an ``asking`` flag (``False`` -- and a no-op -- for standalone), so
|
||||
both clients share one implementation. The per-class ``_himport_*`` methods are
|
||||
thin delegators to these coroutines.
|
||||
|
||||
The sync version lives in :mod:`redis._himport_exec`; the two are kept separate
|
||||
on purpose (the project maintains parallel sync/async stacks by hand).
|
||||
"""
|
||||
|
||||
from redis.exceptions import NoSuchFieldsetError, ResponseError
|
||||
from redis.himport import (
|
||||
HIMPORT_DISCARD,
|
||||
HIMPORT_PREPARE,
|
||||
HIMPORT_SET,
|
||||
HImportRegistry,
|
||||
himport_discard_command,
|
||||
himport_prepare_command,
|
||||
himport_set_command,
|
||||
parse_himport_set_args,
|
||||
)
|
||||
|
||||
|
||||
async def reconcile_discards(node, conn):
|
||||
"""DISCARD, on ``conn``, any prepared fieldset removed from the registry.
|
||||
|
||||
Runs at most once per registry mutation: the connection records the registry
|
||||
``revision`` it last reconciled against, so unchanged registries are a no-op.
|
||||
``node`` supplies ``parse_response`` (the standalone client itself, or the
|
||||
owning ``ClusterNode`` in cluster mode).
|
||||
"""
|
||||
registry = conn.himport_registry
|
||||
if registry is None or conn._himport_reconciled_revision == registry.revision:
|
||||
return
|
||||
# Snapshot the revision *before* computing ``stale`` so the value stamped at
|
||||
# the end is never newer than the registry state ``stale`` reflects. A
|
||||
# concurrent ``himport_discard`` (or another task while this awaits the
|
||||
# DISCARD replies) that lands after this point only leaves the connection
|
||||
# marked behind the live revision, so the next reconcile re-runs and catches
|
||||
# it. Re-reading ``registry.revision`` at the end instead would stamp a
|
||||
# discard this connection never sent.
|
||||
reconciled_to = registry.revision
|
||||
stale = registry.names_to_discard(list(conn._himport_prepared))
|
||||
if stale:
|
||||
await conn.send_packed_command(
|
||||
conn.pack_commands([himport_discard_command(n) for n in stale])
|
||||
)
|
||||
# One reply per packed DISCARD must be read regardless of a per-command
|
||||
# ResponseError, otherwise the unread replies desync the pooled socket.
|
||||
# Drain every reply, then surface the first error (ConnectionError is not
|
||||
# caught: it tears the socket down, so no desync is possible).
|
||||
first_error = None
|
||||
for n in stale:
|
||||
try:
|
||||
await node.parse_response(conn, HIMPORT_DISCARD)
|
||||
except ResponseError as e:
|
||||
first_error = first_error or e
|
||||
conn._himport_prepared.pop(n, None)
|
||||
if first_error is not None:
|
||||
raise first_error
|
||||
conn._himport_reconciled_revision = reconciled_to
|
||||
|
||||
|
||||
async def prepare_and_set(
|
||||
node, conn, key, fieldset_name, values, fieldset, asking=False
|
||||
):
|
||||
"""PREPARE ``fieldset`` bundled with the SET on ``conn`` (one packed write).
|
||||
|
||||
When ``asking`` is set (an ASK-redirected cluster SET) the batch becomes
|
||||
``[PREPARE, ASKING, SET]`` so the per-command ASKING allowance falls
|
||||
immediately before the SET -- the only slot-scoped command. PREPARE is a
|
||||
connection-session command the ASKING flag does not gate, so placing it
|
||||
before ASKING is safe. Every reply is drained even on a per-command error so
|
||||
the packed replies never desync the pooled socket.
|
||||
"""
|
||||
commands = [himport_prepare_command(fieldset_name, fieldset.fields)]
|
||||
if asking:
|
||||
commands.append(("ASKING",))
|
||||
commands.append(himport_set_command(key, fieldset_name, values))
|
||||
await conn.send_packed_command(conn.pack_commands(commands))
|
||||
prep_error = ask_error = set_error = None
|
||||
set_resp = None
|
||||
try:
|
||||
await node.parse_response(conn, HIMPORT_PREPARE)
|
||||
except ResponseError as e:
|
||||
prep_error = e
|
||||
if asking:
|
||||
try:
|
||||
await node.parse_response(conn, "ASKING")
|
||||
except ResponseError as e:
|
||||
ask_error = e
|
||||
try:
|
||||
set_resp = await node.parse_response(conn, HIMPORT_SET)
|
||||
except ResponseError as e:
|
||||
set_error = e
|
||||
|
||||
if prep_error:
|
||||
raise prep_error # PREPARE failure is the root cause
|
||||
else:
|
||||
conn._himport_prepared[fieldset_name] = fieldset.version
|
||||
|
||||
if ask_error:
|
||||
raise ask_error
|
||||
if set_error:
|
||||
raise set_error
|
||||
return set_resp
|
||||
|
||||
|
||||
async def execute_set(node, conn, key, fieldset_name, values, asking=False):
|
||||
"""Execute an ``HIMPORT SET`` on ``conn`` with the required session setup.
|
||||
|
||||
Reconciles deferred discards, lazily bundles PREPARE with the SET on first
|
||||
use of a fieldset, and recovers once from a mid-connection fieldset loss
|
||||
(``NoSuchFieldsetError``) by re-PREPARE-and-retry. When ``asking`` is set the
|
||||
ASKING allowance is folded into the SET's own packed write so it immediately
|
||||
precedes the (slot-scoped) SET; the session setup runs first, since those are
|
||||
connection-session commands the flag does not gate.
|
||||
"""
|
||||
await reconcile_discards(node, conn)
|
||||
|
||||
registry = conn.himport_registry
|
||||
fieldset = registry.get(fieldset_name) if registry is not None else None
|
||||
# Lazy PREPARE bundled with SET on first use of this fieldset.
|
||||
if (
|
||||
fieldset is not None
|
||||
and conn._himport_prepared.get(fieldset_name) != fieldset.version
|
||||
):
|
||||
return await prepare_and_set(
|
||||
node, conn, key, fieldset_name, values, fieldset, asking=asking
|
||||
)
|
||||
|
||||
# Believed already prepared (or an unregistered fieldset): bare SET, with
|
||||
# ASKING packed immediately before it when this is an ASK redirect.
|
||||
if asking:
|
||||
await conn.send_packed_command(
|
||||
conn.pack_commands(
|
||||
[("ASKING",), himport_set_command(key, fieldset_name, values)]
|
||||
)
|
||||
)
|
||||
try:
|
||||
await node.parse_response(conn, "ASKING")
|
||||
except ResponseError as ask_error:
|
||||
# ASKING and SET were one packed write, so the SET reply is still
|
||||
# queued. Drain it before surfacing the ASKING error, otherwise the
|
||||
# connection returns to the pool with an unread reply and desyncs the
|
||||
# next borrower.
|
||||
try:
|
||||
await node.parse_response(conn, HIMPORT_SET)
|
||||
except ResponseError:
|
||||
pass
|
||||
raise ask_error
|
||||
else:
|
||||
await conn.send_command(*himport_set_command(key, fieldset_name, values))
|
||||
try:
|
||||
return await node.parse_response(conn, HIMPORT_SET)
|
||||
except NoSuchFieldsetError:
|
||||
# Server dropped the fieldset mid-connection without dropping the socket
|
||||
# (e.g. RESET / maxmemory-clients eviction): re-PREPARE on this healthy
|
||||
# connection and retry the SET once rather than reconnecting. Only for
|
||||
# registry-backed fieldsets; manual/unregistered usage propagates.
|
||||
if fieldset is None:
|
||||
raise
|
||||
conn._himport_prepared.pop(fieldset_name, None)
|
||||
return await prepare_and_set(
|
||||
node, conn, key, fieldset_name, values, fieldset, asking=asking
|
||||
)
|
||||
|
||||
|
||||
async def prepare_pipeline(node, conn, command_arg_lists):
|
||||
"""Pre-flight ``conn`` for a pipeline batch containing ``HIMPORT SET``s.
|
||||
|
||||
The packed pipeline write bypasses the per-command lazy-PREPARE path, so the
|
||||
fieldsets referenced by the buffered SETs must be PREPAREd on ``conn`` first.
|
||||
Reconciles deferred discards, then PREPAREs every distinct registered fieldset
|
||||
the batch references that this connection has not already prepared, in one
|
||||
packed write. ``command_arg_lists`` is the batch's per-command positional-arg
|
||||
sequences (the caller extracts them from its own command representation).
|
||||
No-op when the batch has no registry-backed ``HIMPORT SET``.
|
||||
"""
|
||||
# Selection (registry check, deferred-discard reconcile, scan/dedup/version)
|
||||
# is shared with pipeline_prepares. This path differs only in that it sends the
|
||||
# PREPAREs as their own packed exchange -- rather than folding them into a
|
||||
# queued write -- then drains their replies and raises the first error.
|
||||
to_prepare = await pipeline_prepares(node, conn, command_arg_lists)
|
||||
if not to_prepare:
|
||||
return
|
||||
await conn.send_packed_command(
|
||||
conn.pack_commands(prepare_wire_commands(to_prepare))
|
||||
)
|
||||
# Every reply must be drained even on a per-command error, or the unread
|
||||
# replies desync the socket before the buffered batch is sent; then raise.
|
||||
first_error = await drain_pipeline_prepares(node, conn, to_prepare)
|
||||
if first_error is not None:
|
||||
raise first_error
|
||||
|
||||
|
||||
async def pipeline_prepares(node, conn, command_arg_lists):
|
||||
"""Return the fieldsets that must be PREPAREd on ``conn`` for this batch.
|
||||
|
||||
Like :func:`prepare_pipeline`, but does **not** send the PREPAREs: the caller
|
||||
folds them into the same packed write as the queued commands (see the pipeline
|
||||
executors), so the first pipeline use of a fieldset on a fresh or reconnected
|
||||
connection stays a single round trip instead of a separate PREPARE exchange
|
||||
followed by the batch. Deferred-discard reconciliation is still performed here,
|
||||
but it only touches the socket when discards are actually pending (rare); the
|
||||
common warm-up cost -- the first-use PREPARE -- is what gets folded. Returns an
|
||||
empty list when the batch references no not-yet-prepared registered fieldset,
|
||||
or when ``conn`` carries no real HIMPORT registry.
|
||||
"""
|
||||
registry = getattr(conn, "himport_registry", None)
|
||||
if not isinstance(registry, HImportRegistry):
|
||||
return []
|
||||
await reconcile_discards(node, conn)
|
||||
to_prepare = []
|
||||
seen = set()
|
||||
for args in command_arg_lists:
|
||||
parsed = parse_himport_set_args(args)
|
||||
if parsed is None:
|
||||
continue
|
||||
fieldset_name = parsed[1]
|
||||
if fieldset_name in seen:
|
||||
continue
|
||||
seen.add(fieldset_name)
|
||||
fieldset = registry.get(fieldset_name)
|
||||
if (
|
||||
fieldset is not None
|
||||
and conn._himport_prepared.get(fieldset_name) != fieldset.version
|
||||
):
|
||||
to_prepare.append(fieldset)
|
||||
return to_prepare
|
||||
|
||||
|
||||
def prepare_wire_commands(fieldsets):
|
||||
"""The leading ``HIMPORT PREPARE`` wire commands the caller folds into a batch."""
|
||||
return [himport_prepare_command(fs.name, fs.fields) for fs in fieldsets]
|
||||
|
||||
|
||||
async def drain_pipeline_prepares(node, conn, fieldsets):
|
||||
"""Drain the ``len(fieldsets)`` leading PREPARE replies of a folded pipeline
|
||||
write, marking each fieldset prepared on success.
|
||||
|
||||
Returns the first ``ResponseError`` (or ``None``). The caller must still drain
|
||||
the queued command replies and only then surface this error: every reply on
|
||||
the wire has to be read before raising, or the pooled socket desyncs.
|
||||
"""
|
||||
first_error = None
|
||||
for fs in fieldsets:
|
||||
try:
|
||||
await node.parse_response(conn, HIMPORT_PREPARE)
|
||||
except ResponseError as e:
|
||||
first_error = first_error or e
|
||||
continue
|
||||
conn._himport_prepared[fs.name] = fs.version
|
||||
return first_error
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,265 @@
|
||||
import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any, Mapping, Optional, Union
|
||||
|
||||
from redis.http.http_client import HttpClient, HttpResponse
|
||||
|
||||
DEFAULT_USER_AGENT = "HttpClient/1.0 (+https://example.invalid)"
|
||||
DEFAULT_TIMEOUT = 30.0
|
||||
RETRY_STATUS_CODES = {429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
class AsyncHTTPClient(ABC):
|
||||
@abstractmethod
|
||||
async def get(
|
||||
self,
|
||||
path: str,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
"""
|
||||
Invoke HTTP GET request."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def delete(
|
||||
self,
|
||||
path: str,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
"""
|
||||
Invoke HTTP DELETE request."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def post(
|
||||
self,
|
||||
path: str,
|
||||
json_body: Optional[Any] = None,
|
||||
data: Optional[Union[bytes, str]] = None,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
"""
|
||||
Invoke HTTP POST request."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def put(
|
||||
self,
|
||||
path: str,
|
||||
json_body: Optional[Any] = None,
|
||||
data: Optional[Union[bytes, str]] = None,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
"""
|
||||
Invoke HTTP PUT request."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def patch(
|
||||
self,
|
||||
path: str,
|
||||
json_body: Optional[Any] = None,
|
||||
data: Optional[Union[bytes, str]] = None,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
"""
|
||||
Invoke HTTP PATCH request."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
body: Optional[Union[bytes, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
) -> HttpResponse:
|
||||
"""
|
||||
Invoke HTTP request with given method."""
|
||||
pass
|
||||
|
||||
|
||||
class AsyncHTTPClientWrapper(AsyncHTTPClient):
|
||||
"""
|
||||
An async wrapper around sync HTTP client with thread pool execution.
|
||||
"""
|
||||
|
||||
def __init__(self, client: HttpClient, max_workers: int = 10) -> None:
|
||||
"""
|
||||
Initialize a new HTTP client instance.
|
||||
|
||||
Args:
|
||||
client: Sync HTTP client instance.
|
||||
max_workers: Maximum number of concurrent requests.
|
||||
|
||||
The client supports both regular HTTPS with server verification and mutual TLS
|
||||
authentication. For server verification, provide CA certificate information via
|
||||
ca_file, ca_path or ca_data. For mutual TLS, additionally provide a client
|
||||
certificate and key via client_cert_file and client_key_file.
|
||||
"""
|
||||
self.client = client
|
||||
self._executor = ThreadPoolExecutor(max_workers=max_workers)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
path: str,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
self._executor, self.client.get, path, params, headers, timeout, expect_json
|
||||
)
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
path: str,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
self._executor,
|
||||
self.client.delete,
|
||||
path,
|
||||
params,
|
||||
headers,
|
||||
timeout,
|
||||
expect_json,
|
||||
)
|
||||
|
||||
async def post(
|
||||
self,
|
||||
path: str,
|
||||
json_body: Optional[Any] = None,
|
||||
data: Optional[Union[bytes, str]] = None,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
self._executor,
|
||||
self.client.post,
|
||||
path,
|
||||
json_body,
|
||||
data,
|
||||
params,
|
||||
headers,
|
||||
timeout,
|
||||
expect_json,
|
||||
)
|
||||
|
||||
async def put(
|
||||
self,
|
||||
path: str,
|
||||
json_body: Optional[Any] = None,
|
||||
data: Optional[Union[bytes, str]] = None,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
self._executor,
|
||||
self.client.put,
|
||||
path,
|
||||
json_body,
|
||||
data,
|
||||
params,
|
||||
headers,
|
||||
timeout,
|
||||
expect_json,
|
||||
)
|
||||
|
||||
async def patch(
|
||||
self,
|
||||
path: str,
|
||||
json_body: Optional[Any] = None,
|
||||
data: Optional[Union[bytes, str]] = None,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
expect_json: bool = True,
|
||||
) -> Union[HttpResponse, Any]:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
self._executor,
|
||||
self.client.patch,
|
||||
path,
|
||||
json_body,
|
||||
data,
|
||||
params,
|
||||
headers,
|
||||
timeout,
|
||||
expect_json,
|
||||
)
|
||||
|
||||
async def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
params: Optional[
|
||||
Mapping[str, Union[None, str, int, float, bool, list, tuple]]
|
||||
] = None,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
body: Optional[Union[bytes, str]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
) -> HttpResponse:
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
self._executor,
|
||||
self.client.request,
|
||||
method,
|
||||
path,
|
||||
params,
|
||||
headers,
|
||||
body,
|
||||
timeout,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,345 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Awaitable, Literal, Optional, Union
|
||||
|
||||
from redis.exceptions import LockError, LockNotOwnedError
|
||||
from redis.typing import Number
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from redis.asyncio import Redis, RedisCluster
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Lock:
|
||||
"""
|
||||
A shared, distributed Lock. Using Redis for locking allows the Lock
|
||||
to be shared across processes and/or machines.
|
||||
|
||||
It's left to the user to resolve deadlock issues and make sure
|
||||
multiple clients play nicely together.
|
||||
"""
|
||||
|
||||
lua_release = None
|
||||
lua_extend = None
|
||||
lua_reacquire = None
|
||||
|
||||
# KEYS[1] - lock name
|
||||
# ARGV[1] - token
|
||||
# return 1 if the lock was released, otherwise 0
|
||||
LUA_RELEASE_SCRIPT = """
|
||||
local token = redis.call('get', KEYS[1])
|
||||
if not token or token ~= ARGV[1] then
|
||||
return 0
|
||||
end
|
||||
redis.call('del', KEYS[1])
|
||||
return 1
|
||||
"""
|
||||
|
||||
# KEYS[1] - lock name
|
||||
# ARGV[1] - token
|
||||
# ARGV[2] - additional milliseconds
|
||||
# ARGV[3] - "0" if the additional time should be added to the lock's
|
||||
# existing ttl or "1" if the existing ttl should be replaced
|
||||
# return 1 if the locks time was extended, otherwise 0
|
||||
LUA_EXTEND_SCRIPT = """
|
||||
local token = redis.call('get', KEYS[1])
|
||||
if not token or token ~= ARGV[1] then
|
||||
return 0
|
||||
end
|
||||
local expiration = redis.call('pttl', KEYS[1])
|
||||
if not expiration then
|
||||
expiration = 0
|
||||
end
|
||||
if expiration < 0 then
|
||||
return 0
|
||||
end
|
||||
|
||||
local newttl = ARGV[2]
|
||||
if ARGV[3] == "0" then
|
||||
newttl = ARGV[2] + expiration
|
||||
end
|
||||
redis.call('pexpire', KEYS[1], newttl)
|
||||
return 1
|
||||
"""
|
||||
|
||||
# KEYS[1] - lock name
|
||||
# ARGV[1] - token
|
||||
# ARGV[2] - milliseconds
|
||||
# return 1 if the locks time was reacquired, otherwise 0
|
||||
LUA_REACQUIRE_SCRIPT = """
|
||||
local token = redis.call('get', KEYS[1])
|
||||
if not token or token ~= ARGV[1] then
|
||||
return 0
|
||||
end
|
||||
redis.call('pexpire', KEYS[1], ARGV[2])
|
||||
return 1
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
redis: Union["Redis", "RedisCluster"],
|
||||
name: Union[str, bytes, memoryview],
|
||||
timeout: Optional[float] = None,
|
||||
sleep: float = 0.1,
|
||||
blocking: bool = True,
|
||||
blocking_timeout: Optional[Number] = None,
|
||||
thread_local: bool = True,
|
||||
raise_on_release_error: bool = True,
|
||||
):
|
||||
"""
|
||||
Create a new Lock instance named ``name`` using the Redis client
|
||||
supplied by ``redis``.
|
||||
|
||||
``timeout`` indicates a maximum life for the lock in seconds.
|
||||
By default, it will remain locked until release() is called.
|
||||
``timeout`` can be specified as a float or integer, both representing
|
||||
the number of seconds to wait.
|
||||
|
||||
``sleep`` indicates the amount of time to sleep in seconds per loop
|
||||
iteration when the lock is in blocking mode and another client is
|
||||
currently holding the lock.
|
||||
|
||||
``blocking`` indicates whether calling ``acquire`` should block until
|
||||
the lock has been acquired or to fail immediately, causing ``acquire``
|
||||
to return False and the lock not being acquired. Defaults to True.
|
||||
Note this value can be overridden by passing a ``blocking``
|
||||
argument to ``acquire``.
|
||||
|
||||
``blocking_timeout`` indicates the maximum amount of time in seconds to
|
||||
spend trying to acquire the lock. A value of ``None`` indicates
|
||||
continue trying forever. ``blocking_timeout`` can be specified as a
|
||||
float or integer, both representing the number of seconds to wait.
|
||||
|
||||
``thread_local`` indicates whether the lock token is placed in
|
||||
thread-local storage. By default, the token is placed in thread local
|
||||
storage so that a thread only sees its token, not a token set by
|
||||
another thread. Consider the following timeline:
|
||||
|
||||
time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds.
|
||||
thread-1 sets the token to "abc"
|
||||
time: 1, thread-2 blocks trying to acquire `my-lock` using the
|
||||
Lock instance.
|
||||
time: 5, thread-1 has not yet completed. redis expires the lock
|
||||
key.
|
||||
time: 5, thread-2 acquired `my-lock` now that it's available.
|
||||
thread-2 sets the token to "xyz"
|
||||
time: 6, thread-1 finishes its work and calls release(). if the
|
||||
token is *not* stored in thread local storage, then
|
||||
thread-1 would see the token value as "xyz" and would be
|
||||
able to successfully release the thread-2's lock.
|
||||
|
||||
``raise_on_release_error`` indicates whether to raise an exception when
|
||||
the lock is no longer owned when exiting the context manager. By default,
|
||||
this is True, meaning an exception will be raised. If False, the warning
|
||||
will be logged and the exception will be suppressed.
|
||||
|
||||
In some use cases it's necessary to disable thread local storage. For
|
||||
example, if you have code where one thread acquires a lock and passes
|
||||
that lock instance to a worker thread to release later. If thread
|
||||
local storage isn't disabled in this case, the worker thread won't see
|
||||
the token set by the thread that acquired the lock. Our assumption
|
||||
is that these cases aren't common and as such default to using
|
||||
thread local storage.
|
||||
"""
|
||||
self.redis = redis
|
||||
self.name = name
|
||||
self.timeout = timeout
|
||||
self.sleep = sleep
|
||||
self.blocking = blocking
|
||||
self.blocking_timeout = blocking_timeout
|
||||
self.thread_local = bool(thread_local)
|
||||
self.local = threading.local() if self.thread_local else SimpleNamespace()
|
||||
self.raise_on_release_error = raise_on_release_error
|
||||
self.local.token = None
|
||||
self.register_scripts()
|
||||
|
||||
def register_scripts(self):
|
||||
cls = self.__class__
|
||||
client = self.redis
|
||||
if cls.lua_release is None:
|
||||
cls.lua_release = client.register_script(cls.LUA_RELEASE_SCRIPT)
|
||||
if cls.lua_extend is None:
|
||||
cls.lua_extend = client.register_script(cls.LUA_EXTEND_SCRIPT)
|
||||
if cls.lua_reacquire is None:
|
||||
cls.lua_reacquire = client.register_script(cls.LUA_REACQUIRE_SCRIPT)
|
||||
|
||||
async def __aenter__(self):
|
||||
if await self.acquire():
|
||||
return self
|
||||
raise LockError("Unable to acquire lock within the time specified")
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
try:
|
||||
await self.release()
|
||||
except LockError:
|
||||
if self.raise_on_release_error:
|
||||
raise
|
||||
logger.warning(
|
||||
"Lock was unlocked or no longer owned when exiting context manager."
|
||||
)
|
||||
|
||||
async def acquire(
|
||||
self,
|
||||
blocking: Optional[bool] = None,
|
||||
blocking_timeout: Optional[Number] = None,
|
||||
token: Optional[Union[str, bytes]] = None,
|
||||
):
|
||||
"""
|
||||
Use Redis to hold a shared, distributed lock named ``name``.
|
||||
Returns True once the lock is acquired.
|
||||
|
||||
If ``blocking`` is False, always return immediately. If the lock
|
||||
was acquired, return True, otherwise return False.
|
||||
|
||||
``blocking_timeout`` specifies the maximum number of seconds to
|
||||
wait trying to acquire the lock.
|
||||
|
||||
``token`` specifies the token value to be used. If provided, token
|
||||
must be a bytes object or a string that can be encoded to a bytes
|
||||
object with the default encoding. If a token isn't specified, a UUID
|
||||
will be generated.
|
||||
"""
|
||||
sleep = self.sleep
|
||||
if token is None:
|
||||
token = uuid.uuid1().hex.encode()
|
||||
else:
|
||||
try:
|
||||
encoder = self.redis.connection_pool.get_encoder()
|
||||
except AttributeError:
|
||||
# Cluster
|
||||
encoder = self.redis.get_encoder()
|
||||
token = encoder.encode(token)
|
||||
if blocking is None:
|
||||
blocking = self.blocking
|
||||
if blocking_timeout is None:
|
||||
blocking_timeout = self.blocking_timeout
|
||||
stop_trying_at = None
|
||||
if blocking_timeout is not None:
|
||||
stop_trying_at = asyncio.get_running_loop().time() + blocking_timeout
|
||||
while True:
|
||||
if await self.do_acquire(token):
|
||||
self.local.token = token
|
||||
return True
|
||||
if not blocking:
|
||||
return False
|
||||
next_try_at = asyncio.get_running_loop().time() + sleep
|
||||
if stop_trying_at is not None and next_try_at > stop_trying_at:
|
||||
return False
|
||||
await asyncio.sleep(sleep)
|
||||
|
||||
async def do_acquire(self, token: Union[str, bytes]) -> bool:
|
||||
if self.timeout:
|
||||
# convert to milliseconds
|
||||
timeout = int(self.timeout * 1000)
|
||||
else:
|
||||
timeout = None
|
||||
if await self.redis.set(self.name, token, nx=True, px=timeout):
|
||||
return True
|
||||
return False
|
||||
|
||||
async def locked(self) -> bool:
|
||||
"""
|
||||
Returns True if this key is locked by any process, otherwise False.
|
||||
"""
|
||||
return await self.redis.get(self.name) is not None
|
||||
|
||||
async def owned(self) -> bool:
|
||||
"""
|
||||
Returns True if this key is locked by this lock, otherwise False.
|
||||
"""
|
||||
stored_token = await self.redis.get(self.name)
|
||||
# need to always compare bytes to bytes
|
||||
# TODO: this can be simplified when the context manager is finished
|
||||
if stored_token and not isinstance(stored_token, bytes):
|
||||
try:
|
||||
encoder = self.redis.connection_pool.get_encoder()
|
||||
except AttributeError:
|
||||
# Cluster
|
||||
encoder = self.redis.get_encoder()
|
||||
stored_token = encoder.encode(stored_token)
|
||||
return self.local.token is not None and stored_token == self.local.token
|
||||
|
||||
async def release(self) -> None:
|
||||
"""Releases the already acquired lock.
|
||||
|
||||
The token is only cleared after the Redis release operation completes
|
||||
successfully. This ensures that if the release is cancelled mid-operation,
|
||||
the lock state remains consistent and can be retried.
|
||||
"""
|
||||
expected_token = self.local.token
|
||||
if expected_token is None:
|
||||
raise LockError(
|
||||
"Cannot release a lock that's not owned or is already unlocked.",
|
||||
lock_name=self.name,
|
||||
)
|
||||
try:
|
||||
await self.do_release(expected_token)
|
||||
except LockNotOwnedError:
|
||||
# Lock doesn't exist in Redis, safe to clear token
|
||||
self.local.token = None
|
||||
raise
|
||||
# Only clear token after successful release
|
||||
self.local.token = None
|
||||
|
||||
async def do_release(self, expected_token: bytes) -> None:
|
||||
if not bool(
|
||||
await self.lua_release(
|
||||
keys=[self.name], args=[expected_token], client=self.redis
|
||||
)
|
||||
):
|
||||
raise LockNotOwnedError("Cannot release a lock that's no longer owned")
|
||||
|
||||
def extend(
|
||||
self, additional_time: Number, replace_ttl: bool = False
|
||||
) -> Awaitable[Literal[True]]:
|
||||
"""
|
||||
Adds more time to an already acquired lock.
|
||||
|
||||
``additional_time`` can be specified as an integer or a float, both
|
||||
representing the number of seconds to add.
|
||||
|
||||
``replace_ttl`` if False (the default), add `additional_time` to
|
||||
the lock's existing ttl. If True, replace the lock's ttl with
|
||||
`additional_time`.
|
||||
"""
|
||||
if self.local.token is None:
|
||||
raise LockError("Cannot extend an unlocked lock")
|
||||
if self.timeout is None:
|
||||
raise LockError("Cannot extend a lock with no timeout")
|
||||
return self.do_extend(additional_time, replace_ttl)
|
||||
|
||||
async def do_extend(self, additional_time, replace_ttl) -> Literal[True]:
|
||||
additional_time = int(additional_time * 1000)
|
||||
if not bool(
|
||||
await self.lua_extend(
|
||||
keys=[self.name],
|
||||
args=[self.local.token, additional_time, replace_ttl and "1" or "0"],
|
||||
client=self.redis,
|
||||
)
|
||||
):
|
||||
raise LockNotOwnedError("Cannot extend a lock that's no longer owned")
|
||||
return True
|
||||
|
||||
def reacquire(self) -> Awaitable[Literal[True]]:
|
||||
"""
|
||||
Resets a TTL of an already acquired lock back to a timeout value.
|
||||
"""
|
||||
if self.local.token is None:
|
||||
raise LockError("Cannot reacquire an unlocked lock")
|
||||
if self.timeout is None:
|
||||
raise LockError("Cannot reacquire a lock with no timeout")
|
||||
return self.do_reacquire()
|
||||
|
||||
async def do_reacquire(self) -> Literal[True]:
|
||||
timeout = int(self.timeout * 1000)
|
||||
if not bool(
|
||||
await self.lua_reacquire(
|
||||
keys=[self.name], args=[self.local.token, timeout], client=self.redis
|
||||
)
|
||||
):
|
||||
raise LockNotOwnedError("Cannot reacquire a lock that's no longer owned")
|
||||
return True
|
||||
@@ -0,0 +1,572 @@
|
||||
import asyncio
|
||||
import functools
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, Awaitable, Callable
|
||||
|
||||
from redis.asyncio.observability.recorder import (
|
||||
record_connection_handoff,
|
||||
record_connection_relaxed_timeout,
|
||||
record_maint_notification_count,
|
||||
)
|
||||
from redis.maint_notifications import (
|
||||
MaintenanceNotification,
|
||||
MaintenanceState,
|
||||
MaintNotificationsConfig,
|
||||
NodeMovingNotification,
|
||||
OSSNodeMigratedNotification,
|
||||
OSSNodeMigratingNotification,
|
||||
_get_maintenance_notification_name,
|
||||
_get_maintenance_notification_type,
|
||||
_should_skip_connection_timeout_update,
|
||||
)
|
||||
from redis.observability.attributes import get_pool_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from redis.asyncio.cluster import RedisCluster
|
||||
from redis.asyncio.connection import AsyncMaintNotificationsAbstractConnection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ScheduledCallback = Callable[..., Awaitable[None]]
|
||||
|
||||
|
||||
def _log_task_exception(message: str, task: "asyncio.Task[Any]") -> None:
|
||||
"""Task done-callback that surfaces a failed task's exception in the logs.
|
||||
|
||||
Without it, an unhandled exception in a fire-and-forget task is only
|
||||
reported by asyncio as a noisy "Task exception was never retrieved"
|
||||
warning. Retrieving the exception here suppresses that warning and logs a
|
||||
meaningful error instead. Cancellation is expected and left unlogged.
|
||||
|
||||
Bind ``message`` with ``functools.partial`` before passing to
|
||||
``add_done_callback`` (which supplies ``task``).
|
||||
"""
|
||||
try:
|
||||
exc = task.exception()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
if exc:
|
||||
logger.error(message, exc_info=exc)
|
||||
|
||||
|
||||
def add_debug_log_for_notification(
|
||||
connection: object,
|
||||
notification: str | MaintenanceNotification,
|
||||
) -> None:
|
||||
if not logger.isEnabledFor(logging.DEBUG):
|
||||
return
|
||||
|
||||
socket_address = None
|
||||
try:
|
||||
writer = getattr(connection, "_writer", None)
|
||||
socket_name = writer.get_extra_info("sockname") if writer else None
|
||||
socket_address = (
|
||||
socket_name[1] if socket_name and len(socket_name) > 1 else None
|
||||
)
|
||||
except (AttributeError, OSError, TypeError):
|
||||
pass
|
||||
|
||||
resolved_ip = None
|
||||
try:
|
||||
get_resolved_ip = getattr(connection, "get_resolved_ip", None)
|
||||
if callable(get_resolved_ip):
|
||||
resolved_ip = get_resolved_ip()
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
|
||||
logger.debug(
|
||||
f"Handling maintenance notification: {notification}, "
|
||||
f"with connection: {connection}, connected to ip {resolved_ip}, "
|
||||
f"local socket port: {socket_address}",
|
||||
)
|
||||
|
||||
|
||||
class AsyncMaintNotificationsPoolHandler:
|
||||
def __init__(
|
||||
self,
|
||||
pool: Any,
|
||||
config: MaintNotificationsConfig,
|
||||
) -> None:
|
||||
self.pool = pool
|
||||
self.config = config
|
||||
self._processed_notifications: set[MaintenanceNotification] = set()
|
||||
self._scheduled_tasks: set[asyncio.Task[None]] = set()
|
||||
self._lock = asyncio.Lock()
|
||||
self.connection: Any | None = None
|
||||
|
||||
def set_connection(
|
||||
self, connection: "AsyncMaintNotificationsAbstractConnection"
|
||||
) -> None:
|
||||
self.connection = connection
|
||||
|
||||
def get_handler_for_connection(self) -> "AsyncMaintNotificationsPoolHandler":
|
||||
# Copy all data that should be shared between connections, while each
|
||||
# connection gets its own handler instance and current connection state.
|
||||
copy = AsyncMaintNotificationsPoolHandler(self.pool, self.config)
|
||||
copy._processed_notifications = self._processed_notifications
|
||||
copy._scheduled_tasks = self._scheduled_tasks
|
||||
copy._lock = self._lock
|
||||
copy.connection = None
|
||||
return copy
|
||||
|
||||
async def remove_expired_notifications(self) -> None:
|
||||
async with self._lock:
|
||||
for notification in tuple(self._processed_notifications):
|
||||
if notification.is_expired():
|
||||
self._processed_notifications.remove(notification)
|
||||
|
||||
async def handle_notification(self, notification: MaintenanceNotification) -> None:
|
||||
await self.remove_expired_notifications()
|
||||
|
||||
if isinstance(notification, NodeMovingNotification):
|
||||
await self.handle_node_moving_notification(notification)
|
||||
else:
|
||||
logger.error(f"Unhandled notification type: {notification}")
|
||||
|
||||
async def handle_node_moving_notification(
|
||||
self, notification: NodeMovingNotification
|
||||
) -> None:
|
||||
if (
|
||||
not self.config.proactive_reconnect
|
||||
and not self.config.is_relaxed_timeouts_enabled()
|
||||
):
|
||||
return
|
||||
|
||||
async with self._lock:
|
||||
if notification in self._processed_notifications:
|
||||
# nothing to do in the connection pool handling
|
||||
# the notification has already been handled or is expired
|
||||
# just return
|
||||
return
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
f"Handling node MOVING notification: {notification}, "
|
||||
f"with connection: {self.connection}, connected to ip "
|
||||
f"{self.connection.get_resolved_ip() if self.connection else None}"
|
||||
)
|
||||
# Get the current connected address - if any
|
||||
# This is the address that is being moved
|
||||
# and we need to handle only connections
|
||||
# connected to the same address
|
||||
moving_address_src = (
|
||||
self.connection.getpeername() if self.connection else None
|
||||
)
|
||||
|
||||
# The async pool owns the active/free connection collections and
|
||||
# asyncio.Lock is not reentrant, so the whole MOVING pool mutation
|
||||
# has to be one pool-owned atomic operation. The handler still owns
|
||||
# the notification policy and passes the already-decided inputs.
|
||||
await self.pool.apply_moving_notification(
|
||||
notification=notification,
|
||||
config=self.config,
|
||||
moving_address_src=moving_address_src,
|
||||
run_proactive_reconnect=(
|
||||
self.config.proactive_reconnect
|
||||
and notification.new_node_host is not None
|
||||
),
|
||||
)
|
||||
|
||||
if self.config.proactive_reconnect and notification.new_node_host is None:
|
||||
self._schedule(
|
||||
notification.ttl / 2,
|
||||
self.run_proactive_reconnect,
|
||||
moving_address_src,
|
||||
)
|
||||
|
||||
self._schedule(
|
||||
notification.ttl,
|
||||
self.handle_node_moved_notification,
|
||||
notification,
|
||||
)
|
||||
|
||||
await record_connection_handoff(
|
||||
pool_name=get_pool_name(self.pool),
|
||||
)
|
||||
|
||||
self._processed_notifications.add(notification)
|
||||
|
||||
async def run_proactive_reconnect(
|
||||
self, moving_address_src: str | None = None
|
||||
) -> None:
|
||||
"""
|
||||
Run proactive reconnect for the pool.
|
||||
Active connections are marked for reconnect after they complete the current command.
|
||||
Inactive connections are disconnected and will be connected on next use.
|
||||
"""
|
||||
async with self._lock:
|
||||
# This delayed reconnect must be atomic for the same reason as the
|
||||
# initial MOVING mutation: a connection can move between active and
|
||||
# free lists while another task is acquiring or releasing it.
|
||||
await self.pool.run_proactive_reconnect(
|
||||
moving_address_src=moving_address_src,
|
||||
)
|
||||
|
||||
async def handle_node_moved_notification(
|
||||
self, notification: NodeMovingNotification
|
||||
) -> None:
|
||||
"""
|
||||
Handle the cleanup after a node moving notification expires.
|
||||
"""
|
||||
notification_hash = hash(notification)
|
||||
|
||||
async with self._lock:
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
f"Reverting temporary changes related to notification: {notification}, "
|
||||
f"with connection: {self.connection}, connected to ip "
|
||||
f"{self.connection.get_resolved_ip() if self.connection else None}"
|
||||
)
|
||||
reset_relaxed_timeout = self.config.is_relaxed_timeouts_enabled()
|
||||
reset_host_address = self.config.proactive_reconnect
|
||||
|
||||
# Cleanup has to reset future connection kwargs and existing
|
||||
# matching connections together under the pool lock. Splitting it
|
||||
# lets an acquire/release interleave and leaves stale MOVING state.
|
||||
await self.pool.cleanup_moving_notification(
|
||||
notification_hash=notification_hash,
|
||||
reset_relaxed_timeout=reset_relaxed_timeout,
|
||||
reset_host_address=reset_host_address,
|
||||
)
|
||||
|
||||
def _schedule(
|
||||
self,
|
||||
delay: float,
|
||||
callback: _ScheduledCallback,
|
||||
*args: Any,
|
||||
) -> None:
|
||||
# Record the absolute deadline now so that any lag between create_task
|
||||
# and the task's first execution slice does not push the fire time out.
|
||||
deadline = asyncio.get_running_loop().time() + delay
|
||||
task = asyncio.create_task(self._run_after(deadline, callback, *args))
|
||||
self._scheduled_tasks.add(task)
|
||||
task.add_done_callback(self._scheduled_tasks.discard)
|
||||
task.add_done_callback(
|
||||
functools.partial(
|
||||
_log_task_exception,
|
||||
"Error handling scheduled maintenance notification",
|
||||
)
|
||||
)
|
||||
|
||||
async def _run_after(
|
||||
self,
|
||||
deadline: float,
|
||||
callback: _ScheduledCallback,
|
||||
*args: Any,
|
||||
) -> None:
|
||||
remaining = deadline - asyncio.get_running_loop().time()
|
||||
if remaining > 0:
|
||||
await asyncio.sleep(remaining)
|
||||
await callback(*args)
|
||||
|
||||
async def cancel_scheduled_tasks(self) -> None:
|
||||
if not self._scheduled_tasks:
|
||||
return
|
||||
tasks = tuple(self._scheduled_tasks)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
class AsyncMaintNotificationsConnectionHandler:
|
||||
def __init__(
|
||||
self,
|
||||
connection: "AsyncMaintNotificationsAbstractConnection",
|
||||
config: MaintNotificationsConfig,
|
||||
) -> None:
|
||||
self.connection = connection
|
||||
self.config = config
|
||||
|
||||
def _get_pool_name(self) -> str:
|
||||
"""
|
||||
Get the pool name from the connection's pool handler.
|
||||
Falls back to connection representation if pool is not available.
|
||||
"""
|
||||
pool_handler = getattr(
|
||||
self.connection, "_maint_notifications_pool_handler", None
|
||||
)
|
||||
if pool_handler and getattr(pool_handler, "pool", None):
|
||||
return get_pool_name(pool_handler.pool)
|
||||
# Fallback for standalone connections without a pool
|
||||
return repr(self.connection)
|
||||
|
||||
async def handle_notification(self, notification: MaintenanceNotification) -> None:
|
||||
# 1 for start, 0 for end notification type, None for unknown.
|
||||
notification_type = _get_maintenance_notification_type(notification)
|
||||
maint_notification = _get_maintenance_notification_name(notification)
|
||||
|
||||
await record_maint_notification_count(
|
||||
server_address=self.connection.host,
|
||||
server_port=self.connection.port,
|
||||
network_peer_address=self.connection.host,
|
||||
network_peer_port=self.connection.port,
|
||||
maint_notification=maint_notification,
|
||||
)
|
||||
|
||||
if notification_type is None:
|
||||
logger.error(f"Unhandled notification type: {notification}")
|
||||
return
|
||||
|
||||
if notification_type:
|
||||
await self.handle_maintenance_start_notification(
|
||||
MaintenanceState.MAINTENANCE, notification
|
||||
)
|
||||
else:
|
||||
await self.handle_maintenance_completed_notification(
|
||||
notification=notification
|
||||
)
|
||||
|
||||
async def handle_maintenance_start_notification(
|
||||
self,
|
||||
maintenance_state: MaintenanceState,
|
||||
notification: MaintenanceNotification,
|
||||
) -> None:
|
||||
add_debug_log_for_notification(self.connection, notification)
|
||||
|
||||
if _should_skip_connection_timeout_update(
|
||||
self.connection.maintenance_state, self.config
|
||||
):
|
||||
return
|
||||
|
||||
self.connection.maintenance_state = maintenance_state
|
||||
self.connection.set_tmp_settings(
|
||||
tmp_relaxed_timeout=self.config.relaxed_timeout
|
||||
)
|
||||
self.connection.update_current_socket_timeout(self.config.relaxed_timeout)
|
||||
if isinstance(notification, OSSNodeMigratingNotification):
|
||||
# add the notification id to the set of processed start maint notifications
|
||||
# this is used to skip the unrelaxing of the timeouts if we have received more than
|
||||
# one start notification before the the final end notification
|
||||
self.connection.add_maint_start_notification(notification.id)
|
||||
|
||||
maint_notification = _get_maintenance_notification_name(notification)
|
||||
await record_connection_relaxed_timeout(
|
||||
connection_name=self._get_pool_name(),
|
||||
maint_notification=maint_notification,
|
||||
relaxed=True,
|
||||
)
|
||||
|
||||
async def handle_maintenance_completed_notification(self, **kwargs: Any) -> None:
|
||||
# Only reset timeouts if state is not MOVING and relaxed timeouts are enabled
|
||||
if _should_skip_connection_timeout_update(
|
||||
self.connection.maintenance_state, self.config
|
||||
):
|
||||
return
|
||||
|
||||
notification = None
|
||||
if kwargs.get("notification"):
|
||||
notification = kwargs["notification"]
|
||||
add_debug_log_for_notification(
|
||||
self.connection, notification if notification else "MAINTENANCE_COMPLETED"
|
||||
)
|
||||
self.connection.reset_tmp_settings(reset_relaxed_timeout=True)
|
||||
# Maintenance completed - reset the connection
|
||||
# timeouts by providing -1 as the relaxed timeout
|
||||
self.connection.update_current_socket_timeout(-1)
|
||||
self.connection.maintenance_state = MaintenanceState.NONE
|
||||
# reset the sets that keep track of received start maint
|
||||
# notifications and skipped end maint notifications
|
||||
self.connection.reset_received_notifications()
|
||||
|
||||
if notification:
|
||||
maint_notification = _get_maintenance_notification_name(notification)
|
||||
await record_connection_relaxed_timeout(
|
||||
connection_name=self._get_pool_name(),
|
||||
maint_notification=maint_notification,
|
||||
relaxed=False,
|
||||
)
|
||||
|
||||
|
||||
class AsyncOSSMaintNotificationsHandler:
|
||||
"""
|
||||
Cluster-wide handler for OSS (open-source) cluster maintenance notifications.
|
||||
|
||||
Reacts to SMIGRATED (slot migration completed) push notifications and
|
||||
triggers topology re-initialization via nodes_manager.initialize().
|
||||
|
||||
Lock discipline: _lock is held across the whole pool mutation, including
|
||||
await initialize(), so the topology refresh and the subsequent connection
|
||||
marking/disconnect happen as one atomic operation — releasing the lock
|
||||
mid-mutation would let other tasks observe partial state.
|
||||
|
||||
Re-entrancy is not a deadlock risk here even though asyncio.Lock is
|
||||
non-reentrant: initialize() may dispatch commands whose responses carry new
|
||||
push notifications, but handle_notification schedules the actual handling as
|
||||
a separate background task rather than calling into it inline, so the
|
||||
re-entrant arrival never tries to re-acquire the lock on this call stack.
|
||||
The cheap _in_progress/_processed dedup that gates that scheduling runs
|
||||
without the lock — those sets are only mutated from the single event loop.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cluster_client: "RedisCluster",
|
||||
config: MaintNotificationsConfig,
|
||||
) -> None:
|
||||
self.cluster_client = cluster_client
|
||||
self.config = config
|
||||
self._processed_notifications: set[MaintenanceNotification] = set()
|
||||
self._in_progress: set[MaintenanceNotification] = set()
|
||||
self._lock = asyncio.Lock()
|
||||
self._background_tasks: set[asyncio.Task] = set()
|
||||
|
||||
async def remove_expired_notifications(self) -> None:
|
||||
async with self._lock:
|
||||
for n in tuple(self._processed_notifications):
|
||||
if n.is_expired():
|
||||
self._processed_notifications.remove(n)
|
||||
|
||||
async def handle_notification(self, notification: MaintenanceNotification) -> None:
|
||||
# Synchronous pre-dedup BEFORE scheduling a task.
|
||||
#
|
||||
# The same SMIGRATED notification is delivered by the server on every
|
||||
# connection, so under load this callback fires many times for the same
|
||||
# notification. Without this guard we would schedule one background task
|
||||
# per arrival - a flood of tasks that each acquire the lock and dedup,
|
||||
# saturating the single event loop. Reserving the notification in
|
||||
# _in_progress here (and skipping if already in-progress/processed) caps
|
||||
# it at exactly ONE handling task per unique notification.
|
||||
#
|
||||
# These sets are only mutated from the single event loop, so this
|
||||
# check-and-add is race-free without holding the lock.
|
||||
if (
|
||||
notification in self._in_progress
|
||||
or notification in self._processed_notifications
|
||||
):
|
||||
return
|
||||
self._in_progress.add(notification)
|
||||
|
||||
# Schedule as a background task so the parser's read path is not blocked.
|
||||
# This also breaks the inline call chain that would otherwise deadlock:
|
||||
# initialize() dispatches commands whose responses may carry more push
|
||||
# notifications; as a separate task it can run while this one awaits.
|
||||
#
|
||||
# If scheduling the task (or wiring its callbacks) fails, release the
|
||||
# in-progress reservation made above. Otherwise the notification would be
|
||||
# stuck in _in_progress forever - the dedup guard would skip it on every
|
||||
# future arrival, and _do_handle_notification's finally (which normally
|
||||
# clears it) never runs because the task was never started.
|
||||
try:
|
||||
task = asyncio.get_running_loop().create_task(
|
||||
self._do_handle_notification(notification)
|
||||
)
|
||||
self._background_tasks.add(task)
|
||||
task.add_done_callback(self._background_tasks.discard)
|
||||
task.add_done_callback(
|
||||
functools.partial(
|
||||
_log_task_exception,
|
||||
"Error handling maintenance notification background task",
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
self._in_progress.discard(notification)
|
||||
raise
|
||||
|
||||
async def _do_handle_notification(
|
||||
self, notification: MaintenanceNotification
|
||||
) -> None:
|
||||
try:
|
||||
if isinstance(notification, OSSNodeMigratedNotification):
|
||||
await self.handle_oss_maintenance_completed_notification(notification)
|
||||
else:
|
||||
logger.error(f"Unhandled notification type: {notification}")
|
||||
finally:
|
||||
# Release the in-progress reservation. On success the notification is
|
||||
# also in _processed_notifications (so it won't be re-handled); on
|
||||
# failure it is not, allowing a later retry.
|
||||
self._in_progress.discard(notification)
|
||||
|
||||
async def handle_oss_maintenance_completed_notification(
|
||||
self, notification: OSSNodeMigratedNotification
|
||||
) -> None:
|
||||
await self.remove_expired_notifications()
|
||||
|
||||
async with self._lock:
|
||||
# handle_notification already reserved this notification in
|
||||
# _in_progress and guaranteed uniqueness; the processed check here is
|
||||
# defensive (e.g. across handler copies sharing the same sets).
|
||||
if notification in self._processed_notifications:
|
||||
return
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(f"Handling SMIGRATED notification: {notification}")
|
||||
|
||||
# Extract the information about the src and destination nodes that are
|
||||
# affected by the maintenance. nodes_to_slots_mapping structure:
|
||||
# {
|
||||
# "src_host:port": [
|
||||
# {"dest_host:port": "slot_range"},
|
||||
# ...
|
||||
# ],
|
||||
# ...
|
||||
# }
|
||||
additional_startup_nodes_info = []
|
||||
affected_nodes = set()
|
||||
for (
|
||||
src_address,
|
||||
dest_mappings,
|
||||
) in notification.nodes_to_slots_mapping.items():
|
||||
src_host, src_port = src_address.rsplit(":", 1)
|
||||
src_node = self.cluster_client.nodes_manager.get_node(
|
||||
host=src_host, port=int(src_port)
|
||||
)
|
||||
if src_node is not None:
|
||||
affected_nodes.add(src_node)
|
||||
for dest_mapping in dest_mappings:
|
||||
for dest_address in dest_mapping.keys():
|
||||
dest_host, dest_port = dest_address.rsplit(":", 1)
|
||||
additional_startup_nodes_info.append(
|
||||
(dest_host, int(dest_port))
|
||||
)
|
||||
# Updates the cluster slots cache with the new slots mapping
|
||||
# This will also update the nodes cache with the new nodes mapping
|
||||
await self.cluster_client.nodes_manager.initialize(
|
||||
additional_startup_nodes_info=additional_startup_nodes_info,
|
||||
)
|
||||
|
||||
all_nodes = set(affected_nodes)
|
||||
all_nodes = all_nodes.union(
|
||||
self.cluster_client.nodes_manager.nodes_cache.values()
|
||||
)
|
||||
for current_node in all_nodes:
|
||||
handoff_recorded = False
|
||||
if current_node in affected_nodes:
|
||||
# mark for reconnect all in-use connections to the node — this
|
||||
# forces them to disconnect after completing their current command
|
||||
free_set = set(current_node._free)
|
||||
for conn in current_node._connections:
|
||||
if conn not in free_set:
|
||||
add_debug_log_for_notification(
|
||||
conn, "SMIGRATED - mark for reconnect"
|
||||
)
|
||||
conn.mark_for_reconnect()
|
||||
await record_connection_handoff(
|
||||
pool_name=f"{current_node.host}:{current_node.port}"
|
||||
)
|
||||
handoff_recorded = True
|
||||
else:
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
f"SMIGRATED: Node {current_node.name} not affected "
|
||||
f"by maintenance, skipping mark for reconnect"
|
||||
)
|
||||
if (
|
||||
current_node
|
||||
not in self.cluster_client.nodes_manager.nodes_cache.values()
|
||||
):
|
||||
task = asyncio.get_running_loop().create_task(
|
||||
current_node.disconnect_free_connections()
|
||||
)
|
||||
self._background_tasks.add(task)
|
||||
task.add_done_callback(self._background_tasks.discard)
|
||||
task.add_done_callback(
|
||||
functools.partial(
|
||||
_log_task_exception,
|
||||
"Error disconnecting free connections after "
|
||||
"maintenance notification",
|
||||
)
|
||||
)
|
||||
if not handoff_recorded:
|
||||
await record_connection_handoff(
|
||||
pool_name=f"{current_node.host}:{current_node.port}"
|
||||
)
|
||||
|
||||
self._processed_notifications.add(notification)
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,661 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Awaitable, Callable, List, Literal, Optional, Union
|
||||
|
||||
from redis.asyncio.multidb.command_executor import DefaultCommandExecutor
|
||||
from redis.asyncio.multidb.config import (
|
||||
DEFAULT_GRACE_PERIOD,
|
||||
DatabaseConfig,
|
||||
InitialHealthCheck,
|
||||
MultiDbConfig,
|
||||
)
|
||||
from redis.asyncio.multidb.database import AsyncDatabase, Database, Databases
|
||||
from redis.asyncio.multidb.failure_detector import AsyncFailureDetector
|
||||
from redis.asyncio.multidb.healthcheck import HealthCheck, HealthCheckPolicy
|
||||
from redis.asyncio.retry import Retry
|
||||
from redis.background import BackgroundScheduler
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.commands import AsyncCoreCommands, AsyncRedisModuleCommands
|
||||
from redis.exceptions import DataError
|
||||
from redis.multidb.circuit import CircuitBreaker
|
||||
from redis.multidb.circuit import State as CBState
|
||||
from redis.multidb.exception import (
|
||||
InitialHealthCheckFailedError,
|
||||
NoValidDatabaseException,
|
||||
UnhealthyDatabaseException,
|
||||
)
|
||||
from redis.observability.attributes import GeoFailoverReason
|
||||
from redis.typing import ChannelT, EncodableT, KeyT, PubSubHandler, Subscription
|
||||
from redis.utils import experimental
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@experimental
|
||||
class MultiDBClient(AsyncRedisModuleCommands, AsyncCoreCommands):
|
||||
"""
|
||||
Client that operates on multiple logical Redis databases.
|
||||
Should be used in Client-side geographic failover database setups.
|
||||
"""
|
||||
|
||||
def __init__(self, config: MultiDbConfig):
|
||||
self._databases = config.databases()
|
||||
self._health_checks = (
|
||||
config.default_health_checks()
|
||||
if not config.health_checks
|
||||
else config.health_checks
|
||||
)
|
||||
self._health_check_interval = config.health_check_interval
|
||||
self._health_check_policy: HealthCheckPolicy = (
|
||||
config.health_check_policy.value()
|
||||
)
|
||||
self._failure_detectors = (
|
||||
config.default_failure_detectors()
|
||||
if not config.failure_detectors
|
||||
else config.failure_detectors
|
||||
)
|
||||
|
||||
self._failover_strategy = (
|
||||
config.default_failover_strategy()
|
||||
if config.failover_strategy is None
|
||||
else config.failover_strategy
|
||||
)
|
||||
self._failover_strategy.set_databases(self._databases)
|
||||
self._auto_fallback_interval = config.auto_fallback_interval
|
||||
self._event_dispatcher = config.event_dispatcher
|
||||
self._command_retry = config.command_retry
|
||||
self._command_retry.update_supported_errors([ConnectionRefusedError])
|
||||
self.command_executor = DefaultCommandExecutor(
|
||||
failure_detectors=self._failure_detectors,
|
||||
databases=self._databases,
|
||||
command_retry=self._command_retry,
|
||||
failover_strategy=self._failover_strategy,
|
||||
failover_attempts=config.failover_attempts,
|
||||
failover_delay=config.failover_delay,
|
||||
event_dispatcher=self._event_dispatcher,
|
||||
auto_fallback_interval=self._auto_fallback_interval,
|
||||
)
|
||||
self.initialized = False
|
||||
self._hc_lock = asyncio.Lock()
|
||||
self._bg_scheduler = BackgroundScheduler()
|
||||
self._config = config
|
||||
self._recurring_hc_task = None
|
||||
self._hc_tasks = []
|
||||
self._half_open_state_task = None
|
||||
|
||||
async def __aenter__(self: "MultiDBClient") -> "MultiDBClient":
|
||||
if not self.initialized:
|
||||
await self.initialize()
|
||||
return self
|
||||
|
||||
async def aclose(self):
|
||||
# Cancel background tasks
|
||||
if self._recurring_hc_task:
|
||||
self._recurring_hc_task.cancel()
|
||||
if self._half_open_state_task:
|
||||
self._half_open_state_task.cancel()
|
||||
for hc_task in self._hc_tasks:
|
||||
hc_task.cancel()
|
||||
|
||||
# Close health check connection pools
|
||||
await self._health_check_policy.close()
|
||||
|
||||
# Close database client
|
||||
if self.command_executor.active_database:
|
||||
await self.command_executor.active_database.client.aclose()
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
await self.aclose()
|
||||
|
||||
async def initialize(self):
|
||||
"""
|
||||
Perform initialization of databases to define their initial state.
|
||||
"""
|
||||
|
||||
# Initial databases check to define initial state
|
||||
await self._perform_initial_health_check()
|
||||
|
||||
# Starts recurring health checks on the background.
|
||||
self._recurring_hc_task = asyncio.create_task(
|
||||
self._bg_scheduler.run_recurring_async(
|
||||
self._health_check_interval,
|
||||
self._check_databases_health,
|
||||
)
|
||||
)
|
||||
|
||||
is_active_db_found = False
|
||||
|
||||
for database, weight in self._databases:
|
||||
# Set on state changed callback for each circuit.
|
||||
database.circuit.on_state_changed(self._on_circuit_state_change_callback)
|
||||
|
||||
# Set states according to a weights and circuit state
|
||||
if database.circuit.state == CBState.CLOSED and not is_active_db_found:
|
||||
# Directly set the active database during initialization
|
||||
# without recording a geo failover metric
|
||||
self.command_executor._active_database = database
|
||||
is_active_db_found = True
|
||||
|
||||
if not is_active_db_found:
|
||||
raise NoValidDatabaseException(
|
||||
"Initial connection failed - no active database found"
|
||||
)
|
||||
|
||||
self.initialized = True
|
||||
|
||||
def get_databases(self) -> Databases:
|
||||
"""
|
||||
Returns a sorted (by weight) list of all databases.
|
||||
"""
|
||||
return self._databases
|
||||
|
||||
async def set_active_database(self, database: AsyncDatabase) -> None:
|
||||
"""
|
||||
Promote one of the existing databases to become an active.
|
||||
"""
|
||||
exists = None
|
||||
|
||||
for existing_db, _ in self._databases:
|
||||
if existing_db == database:
|
||||
exists = True
|
||||
break
|
||||
|
||||
if not exists:
|
||||
raise ValueError("Given database is not a member of database list")
|
||||
|
||||
await self._check_db_health(database)
|
||||
|
||||
if database.circuit.state == CBState.CLOSED:
|
||||
highest_weighted_db, _ = self._databases.get_top_n(1)[0]
|
||||
await self.command_executor.set_active_database(
|
||||
database, GeoFailoverReason.MANUAL
|
||||
)
|
||||
return
|
||||
|
||||
raise NoValidDatabaseException(
|
||||
"Cannot set active database, database is unhealthy"
|
||||
)
|
||||
|
||||
async def add_database(
|
||||
self, config: DatabaseConfig, skip_initial_health_check: bool = True
|
||||
):
|
||||
"""
|
||||
Adds a new database to the database list.
|
||||
|
||||
Args:
|
||||
config: DatabaseConfig object that contains the database configuration.
|
||||
skip_initial_health_check: If True, adds the database even if it is unhealthy.
|
||||
"""
|
||||
# The retry object is not used in the lower level clients, so we can safely remove it.
|
||||
# We rely on command_retry in terms of global retries.
|
||||
config.client_kwargs.update({"retry": Retry(retries=0, backoff=NoBackoff())})
|
||||
|
||||
if config.from_url:
|
||||
client = self._config.client_class.from_url(
|
||||
config.from_url, **config.client_kwargs
|
||||
)
|
||||
elif config.from_pool:
|
||||
config.from_pool.set_retry(Retry(retries=0, backoff=NoBackoff()))
|
||||
client = self._config.client_class.from_pool(
|
||||
connection_pool=config.from_pool
|
||||
)
|
||||
else:
|
||||
client = self._config.client_class(**config.client_kwargs)
|
||||
|
||||
circuit = (
|
||||
config.default_circuit_breaker()
|
||||
if config.circuit is None
|
||||
else config.circuit
|
||||
)
|
||||
|
||||
database = Database(
|
||||
client=client,
|
||||
circuit=circuit,
|
||||
weight=config.weight,
|
||||
health_check_url=config.health_check_url,
|
||||
)
|
||||
|
||||
try:
|
||||
await self._check_db_health(database)
|
||||
except UnhealthyDatabaseException:
|
||||
if not skip_initial_health_check:
|
||||
raise
|
||||
|
||||
highest_weighted_db, highest_weight = self._databases.get_top_n(1)[0]
|
||||
self._databases.add(database, database.weight)
|
||||
await self._change_active_database(database, highest_weighted_db)
|
||||
|
||||
async def _change_active_database(
|
||||
self, new_database: AsyncDatabase, highest_weight_database: AsyncDatabase
|
||||
):
|
||||
if (
|
||||
new_database.weight > highest_weight_database.weight
|
||||
and new_database.circuit.state == CBState.CLOSED
|
||||
):
|
||||
await self.command_executor.set_active_database(
|
||||
new_database, GeoFailoverReason.AUTOMATIC
|
||||
)
|
||||
|
||||
async def remove_database(self, database: AsyncDatabase):
|
||||
"""
|
||||
Removes a database from the database list.
|
||||
"""
|
||||
weight = self._databases.remove(database)
|
||||
highest_weighted_db, highest_weight = self._databases.get_top_n(1)[0]
|
||||
|
||||
if (
|
||||
highest_weight <= weight
|
||||
and highest_weighted_db.circuit.state == CBState.CLOSED
|
||||
):
|
||||
await self.command_executor.set_active_database(
|
||||
highest_weighted_db, GeoFailoverReason.MANUAL
|
||||
)
|
||||
|
||||
async def update_database_weight(self, database: AsyncDatabase, weight: float):
|
||||
"""
|
||||
Updates a database from the database list.
|
||||
"""
|
||||
exists = None
|
||||
|
||||
for existing_db, _ in self._databases:
|
||||
if existing_db == database:
|
||||
exists = True
|
||||
break
|
||||
|
||||
if not exists:
|
||||
raise ValueError("Given database is not a member of database list")
|
||||
|
||||
highest_weighted_db, highest_weight = self._databases.get_top_n(1)[0]
|
||||
self._databases.update_weight(database, weight)
|
||||
database.weight = weight
|
||||
await self._change_active_database(database, highest_weighted_db)
|
||||
|
||||
def add_failure_detector(self, failure_detector: AsyncFailureDetector):
|
||||
"""
|
||||
Adds a new failure detector to the database.
|
||||
"""
|
||||
self._failure_detectors.append(failure_detector)
|
||||
|
||||
async def add_health_check(self, healthcheck: HealthCheck):
|
||||
"""
|
||||
Adds a new health check to the database.
|
||||
"""
|
||||
async with self._hc_lock:
|
||||
self._health_checks.append(healthcheck)
|
||||
|
||||
async def execute_command(self, *args, **options):
|
||||
"""
|
||||
Executes a single command and return its result.
|
||||
"""
|
||||
if not self.initialized:
|
||||
await self.initialize()
|
||||
|
||||
return await self.command_executor.execute_command(*args, **options)
|
||||
|
||||
# HIMPORT is not supported on the multi-database client. A HIMPORT fieldset is
|
||||
# per-connection server session state tracked by a single client's registry;
|
||||
# there is no coherent way to keep that state consistent across independent
|
||||
# database clients through failover. ``himport_set`` is inherited from
|
||||
# ``AsyncCoreCommands`` (and the lifecycle methods would otherwise be missing
|
||||
# entirely), so override them to fail early and clearly instead of surfacing a
|
||||
# confusing ``no such fieldset`` at runtime.
|
||||
_HIMPORT_UNSUPPORTED = (
|
||||
"HIMPORT is not supported on the multi-database (Active-Active) client"
|
||||
)
|
||||
|
||||
async def himport_prepare(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise DataError(self._HIMPORT_UNSUPPORTED)
|
||||
|
||||
async def himport_set(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise DataError(self._HIMPORT_UNSUPPORTED)
|
||||
|
||||
async def himport_discard(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise DataError(self._HIMPORT_UNSUPPORTED)
|
||||
|
||||
async def himport_discard_all(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise DataError(self._HIMPORT_UNSUPPORTED)
|
||||
|
||||
def pipeline(self):
|
||||
"""
|
||||
Enters into pipeline mode of the client.
|
||||
"""
|
||||
return Pipeline(self)
|
||||
|
||||
async def transaction(
|
||||
self,
|
||||
func: Callable[["Pipeline"], Union[Any, Awaitable[Any]]],
|
||||
*watches: KeyT,
|
||||
shard_hint: Optional[str] = None,
|
||||
value_from_callable: bool = False,
|
||||
watch_delay: Optional[float] = None,
|
||||
):
|
||||
"""
|
||||
Executes callable as transaction.
|
||||
"""
|
||||
if not self.initialized:
|
||||
await self.initialize()
|
||||
|
||||
return await self.command_executor.execute_transaction(
|
||||
func,
|
||||
*watches,
|
||||
shard_hint=shard_hint,
|
||||
value_from_callable=value_from_callable,
|
||||
watch_delay=watch_delay,
|
||||
)
|
||||
|
||||
async def pubsub(self, **kwargs):
|
||||
"""
|
||||
Return a Publish/Subscribe object. With this object, you can
|
||||
subscribe to channels and listen for messages that get published to
|
||||
them.
|
||||
"""
|
||||
if not self.initialized:
|
||||
await self.initialize()
|
||||
|
||||
return PubSub(self, **kwargs)
|
||||
|
||||
async def _check_databases_health(self) -> dict[Database, bool]:
|
||||
"""
|
||||
Runs health checks as a recurring task.
|
||||
Runs health checks against all databases.
|
||||
"""
|
||||
task_to_db: dict[asyncio.Task, Database] = {}
|
||||
|
||||
self._hc_tasks = []
|
||||
for database, _ in self._databases:
|
||||
task = asyncio.create_task(self._check_db_health(database))
|
||||
task_to_db[task] = database
|
||||
self._hc_tasks.append(task)
|
||||
|
||||
results = await asyncio.gather(*self._hc_tasks, return_exceptions=True)
|
||||
|
||||
# Map end results to databases
|
||||
db_results = {
|
||||
task_to_db[task]: result for task, result in zip(self._hc_tasks, results)
|
||||
}
|
||||
|
||||
for database, result in db_results.items():
|
||||
if isinstance(result, UnhealthyDatabaseException):
|
||||
unhealthy_db = result.database
|
||||
unhealthy_db.circuit.state = CBState.OPEN
|
||||
|
||||
logger.debug(
|
||||
"Health check failed, due to exception",
|
||||
exc_info=result.original_exception,
|
||||
)
|
||||
|
||||
db_results[unhealthy_db] = False
|
||||
|
||||
return db_results
|
||||
|
||||
async def _perform_initial_health_check(self):
|
||||
"""
|
||||
Runs initial health check and evaluate healthiness based on initial_health_check_policy.
|
||||
"""
|
||||
results = await self._check_databases_health()
|
||||
is_healthy = True
|
||||
|
||||
if self._config.initial_health_check_policy == InitialHealthCheck.ALL_AVAILABLE:
|
||||
is_healthy = False not in results.values()
|
||||
elif (
|
||||
self._config.initial_health_check_policy
|
||||
== InitialHealthCheck.MAJORITY_AVAILABLE
|
||||
):
|
||||
is_healthy = sum(results.values()) > len(results) / 2
|
||||
elif (
|
||||
self._config.initial_health_check_policy == InitialHealthCheck.ONE_AVAILABLE
|
||||
):
|
||||
is_healthy = True in results.values()
|
||||
|
||||
if not is_healthy:
|
||||
raise InitialHealthCheckFailedError(
|
||||
f"Initial health check failed. Initial health check policy: {self._config.initial_health_check_policy}"
|
||||
)
|
||||
|
||||
async def _check_db_health(self, database: AsyncDatabase) -> bool:
|
||||
"""
|
||||
Runs health checks on the given database until first failure.
|
||||
"""
|
||||
# Health check will setup circuit state
|
||||
is_healthy = await self._health_check_policy.execute(
|
||||
self._health_checks, database
|
||||
)
|
||||
|
||||
if not is_healthy:
|
||||
if database.circuit.state != CBState.OPEN:
|
||||
database.circuit.state = CBState.OPEN
|
||||
return is_healthy
|
||||
elif is_healthy and database.circuit.state != CBState.CLOSED:
|
||||
database.circuit.state = CBState.CLOSED
|
||||
|
||||
return is_healthy
|
||||
|
||||
def _on_circuit_state_change_callback(
|
||||
self, circuit: CircuitBreaker, old_state: CBState, new_state: CBState
|
||||
):
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
if new_state == CBState.HALF_OPEN:
|
||||
self._half_open_state_task = asyncio.create_task(
|
||||
self._check_db_health(circuit.database)
|
||||
)
|
||||
return
|
||||
|
||||
if old_state == CBState.CLOSED and new_state == CBState.OPEN:
|
||||
logger.warning(
|
||||
f"Database {circuit.database} is unreachable. Failover has been initiated."
|
||||
)
|
||||
loop.call_later(DEFAULT_GRACE_PERIOD, _half_open_circuit, circuit)
|
||||
|
||||
if old_state != CBState.CLOSED and new_state == CBState.CLOSED:
|
||||
logger.info(f"Database {circuit.database} is reachable again.")
|
||||
|
||||
|
||||
def _half_open_circuit(circuit: CircuitBreaker):
|
||||
circuit.state = CBState.HALF_OPEN
|
||||
|
||||
|
||||
class Pipeline(AsyncRedisModuleCommands, AsyncCoreCommands):
|
||||
"""
|
||||
Pipeline implementation for multiple logical Redis databases.
|
||||
"""
|
||||
|
||||
_is_async_client: Literal[True] = True
|
||||
|
||||
def __init__(self, client: MultiDBClient):
|
||||
self._command_stack = []
|
||||
self._client = client
|
||||
|
||||
async def __aenter__(self: "Pipeline") -> "Pipeline":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
await self.reset()
|
||||
await self._client.__aexit__(exc_type, exc_value, traceback)
|
||||
|
||||
def __await__(self):
|
||||
return self._async_self().__await__()
|
||||
|
||||
async def _async_self(self):
|
||||
return self
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._command_stack)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
"""Pipeline instances should always evaluate to True"""
|
||||
return True
|
||||
|
||||
async def reset(self) -> None:
|
||||
self._command_stack = []
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Close the pipeline"""
|
||||
await self.reset()
|
||||
|
||||
def pipeline_execute_command(self, *args, **options) -> "Pipeline":
|
||||
"""
|
||||
Stage a command to be executed when execute() is next called
|
||||
|
||||
Returns the current Pipeline object back so commands can be
|
||||
chained together, such as:
|
||||
|
||||
pipe = pipe.set('foo', 'bar').incr('baz').decr('bang')
|
||||
|
||||
At some other point, you can then run: pipe.execute(),
|
||||
which will execute all commands queued in the pipe.
|
||||
"""
|
||||
self._command_stack.append((args, options))
|
||||
return self
|
||||
|
||||
def execute_command(self, *args, **kwargs):
|
||||
"""Adds a command to the stack"""
|
||||
return self.pipeline_execute_command(*args, **kwargs)
|
||||
|
||||
# HIMPORT is unsupported on the multi-database client (see MultiDBClient). A
|
||||
# pipeline inherits ``himport_set`` from ``AsyncCoreCommands`` and would otherwise
|
||||
# queue an unprepared ``HIMPORT SET`` that fails with a confusing server-side
|
||||
# error, bypassing the client-level guard. Block all HIMPORT methods here too.
|
||||
def himport_prepare(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise DataError(MultiDBClient._HIMPORT_UNSUPPORTED)
|
||||
|
||||
def himport_set(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise DataError(MultiDBClient._HIMPORT_UNSUPPORTED)
|
||||
|
||||
def himport_discard(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise DataError(MultiDBClient._HIMPORT_UNSUPPORTED)
|
||||
|
||||
def himport_discard_all(self, *args: Any, **kwargs: Any) -> Any:
|
||||
raise DataError(MultiDBClient._HIMPORT_UNSUPPORTED)
|
||||
|
||||
async def execute(self) -> List[Any]:
|
||||
"""Execute all the commands in the current pipeline"""
|
||||
if not self._client.initialized:
|
||||
await self._client.initialize()
|
||||
|
||||
try:
|
||||
return await self._client.command_executor.execute_pipeline(
|
||||
tuple(self._command_stack)
|
||||
)
|
||||
finally:
|
||||
await self.reset()
|
||||
|
||||
|
||||
class PubSub:
|
||||
"""
|
||||
PubSub object for multi database client.
|
||||
"""
|
||||
|
||||
def __init__(self, client: MultiDBClient, **kwargs):
|
||||
"""Initialize the PubSub object for a multi-database client.
|
||||
|
||||
Args:
|
||||
client: MultiDBClient instance to use for pub/sub operations
|
||||
**kwargs: Additional keyword arguments to pass to the underlying pubsub implementation
|
||||
"""
|
||||
|
||||
self._client = client
|
||||
self._client.command_executor.pubsub(**kwargs)
|
||||
|
||||
async def __aenter__(self) -> "PubSub":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback) -> None:
|
||||
await self.aclose()
|
||||
|
||||
async def aclose(self):
|
||||
return await self._client.command_executor.execute_pubsub_method("aclose")
|
||||
|
||||
@property
|
||||
def subscribed(self) -> bool:
|
||||
return self._client.command_executor.active_pubsub.subscribed
|
||||
|
||||
async def execute_command(self, *args: EncodableT):
|
||||
return await self._client.command_executor.execute_pubsub_method(
|
||||
"execute_command", *args
|
||||
)
|
||||
|
||||
async def psubscribe(
|
||||
self, *args: ChannelT | Subscription, **kwargs: PubSubHandler
|
||||
) -> None:
|
||||
"""
|
||||
Subscribe to channel patterns. Patterns supplied as keyword arguments
|
||||
expect a pattern name as the key and a callable as the value. A
|
||||
pattern's callable will be invoked automatically when a message is
|
||||
received on that pattern rather than producing a message via
|
||||
``listen()``.
|
||||
"""
|
||||
return await self._client.command_executor.execute_pubsub_method(
|
||||
"psubscribe", *args, **kwargs
|
||||
)
|
||||
|
||||
async def punsubscribe(self, *args: ChannelT):
|
||||
"""
|
||||
Unsubscribe from the supplied patterns. If empty, unsubscribe from
|
||||
all patterns.
|
||||
"""
|
||||
return await self._client.command_executor.execute_pubsub_method(
|
||||
"punsubscribe", *args
|
||||
)
|
||||
|
||||
async def subscribe(
|
||||
self, *args: ChannelT | Subscription, **kwargs: PubSubHandler
|
||||
) -> None:
|
||||
"""
|
||||
Subscribe to channels. Channels supplied as keyword arguments expect
|
||||
a channel name as the key and a callable as the value. A channel's
|
||||
callable will be invoked automatically when a message is received on
|
||||
that channel rather than producing a message via ``listen()`` or
|
||||
``get_message()``.
|
||||
"""
|
||||
return await self._client.command_executor.execute_pubsub_method(
|
||||
"subscribe", *args, **kwargs
|
||||
)
|
||||
|
||||
async def unsubscribe(self, *args):
|
||||
"""
|
||||
Unsubscribe from the supplied channels. If empty, unsubscribe from
|
||||
all channels
|
||||
"""
|
||||
return await self._client.command_executor.execute_pubsub_method(
|
||||
"unsubscribe", *args
|
||||
)
|
||||
|
||||
async def get_message(
|
||||
self, ignore_subscribe_messages: bool = False, timeout: Optional[float] = 0.0
|
||||
):
|
||||
"""
|
||||
Get the next message if one is available, otherwise None.
|
||||
|
||||
If timeout is specified, the system will wait for `timeout` seconds
|
||||
before returning. Timeout should be specified as a floating point
|
||||
number or None to wait indefinitely.
|
||||
"""
|
||||
return await self._client.command_executor.execute_pubsub_method(
|
||||
"get_message",
|
||||
ignore_subscribe_messages=ignore_subscribe_messages,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
*,
|
||||
exception_handler=None,
|
||||
poll_timeout: float = 1.0,
|
||||
) -> None:
|
||||
"""Process pub/sub messages using registered callbacks.
|
||||
|
||||
This is the equivalent of :py:meth:`redis.PubSub.run_in_thread` in
|
||||
redis-py, but it is a coroutine. To launch it as a separate task, use
|
||||
``asyncio.create_task``:
|
||||
|
||||
>>> task = asyncio.create_task(pubsub.run())
|
||||
|
||||
To shut it down, use asyncio cancellation:
|
||||
|
||||
>>> task.cancel()
|
||||
>>> await task
|
||||
"""
|
||||
return await self._client.command_executor.execute_pubsub_run(
|
||||
sleep_time=poll_timeout, exception_handler=exception_handler, pubsub=self
|
||||
)
|
||||
@@ -0,0 +1,356 @@
|
||||
from abc import abstractmethod
|
||||
from asyncio import iscoroutinefunction
|
||||
from datetime import datetime
|
||||
from typing import Any, Awaitable, Callable, List, Optional, Union
|
||||
|
||||
from redis.asyncio import RedisCluster
|
||||
from redis.asyncio.client import Pipeline, PubSub
|
||||
from redis.asyncio.multidb.database import AsyncDatabase, Database, Databases
|
||||
from redis.asyncio.multidb.event import (
|
||||
AsyncActiveDatabaseChanged,
|
||||
CloseConnectionOnActiveDatabaseChanged,
|
||||
RegisterCommandFailure,
|
||||
ResubscribeOnActiveDatabaseChanged,
|
||||
)
|
||||
from redis.asyncio.multidb.failover import (
|
||||
DEFAULT_FAILOVER_ATTEMPTS,
|
||||
DEFAULT_FAILOVER_DELAY,
|
||||
AsyncFailoverStrategy,
|
||||
DefaultFailoverStrategyExecutor,
|
||||
FailoverStrategyExecutor,
|
||||
)
|
||||
from redis.asyncio.multidb.failure_detector import AsyncFailureDetector
|
||||
from redis.asyncio.observability.recorder import record_geo_failover
|
||||
from redis.asyncio.retry import Retry
|
||||
from redis.event import AsyncOnCommandsFailEvent, EventDispatcherInterface
|
||||
from redis.multidb.circuit import State as CBState
|
||||
from redis.multidb.command_executor import BaseCommandExecutor, CommandExecutor
|
||||
from redis.multidb.config import DEFAULT_AUTO_FALLBACK_INTERVAL
|
||||
from redis.observability.attributes import GeoFailoverReason
|
||||
from redis.typing import KeyT
|
||||
|
||||
|
||||
class AsyncCommandExecutor(CommandExecutor):
|
||||
@property
|
||||
@abstractmethod
|
||||
def databases(self) -> Databases:
|
||||
"""Returns a list of databases."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def failure_detectors(self) -> List[AsyncFailureDetector]:
|
||||
"""Returns a list of failure detectors."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def add_failure_detector(self, failure_detector: AsyncFailureDetector) -> None:
|
||||
"""Adds a new failure detector to the list of failure detectors."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def active_database(self) -> Optional[AsyncDatabase]:
|
||||
"""Returns currently active database."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def set_active_database(
|
||||
self, database: AsyncDatabase, reason: GeoFailoverReason
|
||||
) -> None:
|
||||
"""Sets the currently active database.
|
||||
|
||||
Args:
|
||||
database: The new active database.
|
||||
reason: The reason for the failover.
|
||||
"""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def active_pubsub(self) -> Optional[PubSub]:
|
||||
"""Returns currently active pubsub."""
|
||||
pass
|
||||
|
||||
@active_pubsub.setter
|
||||
@abstractmethod
|
||||
def active_pubsub(self, pubsub: PubSub) -> None:
|
||||
"""Sets currently active pubsub."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def failover_strategy_executor(self) -> FailoverStrategyExecutor:
|
||||
"""Returns failover strategy executor."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def command_retry(self) -> Retry:
|
||||
"""Returns command retry object."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def pubsub(self, **kwargs):
|
||||
"""Initializes a PubSub object on a currently active database"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def execute_command(self, *args, **options):
|
||||
"""Executes a command and returns the result."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def execute_pipeline(self, command_stack: tuple):
|
||||
"""Executes a stack of commands in pipeline."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def execute_transaction(
|
||||
self, transaction: Callable[[Pipeline], None], *watches, **options
|
||||
):
|
||||
"""Executes a transaction block wrapped in callback."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def execute_pubsub_method(self, method_name: str, *args, **kwargs):
|
||||
"""Executes a given method on active pub/sub."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def execute_pubsub_run(self, sleep_time: float, **kwargs) -> Any:
|
||||
"""Executes pub/sub run in a thread."""
|
||||
pass
|
||||
|
||||
|
||||
class DefaultCommandExecutor(BaseCommandExecutor, AsyncCommandExecutor):
|
||||
def __init__(
|
||||
self,
|
||||
failure_detectors: List[AsyncFailureDetector],
|
||||
databases: Databases,
|
||||
command_retry: Retry,
|
||||
failover_strategy: AsyncFailoverStrategy,
|
||||
event_dispatcher: EventDispatcherInterface,
|
||||
failover_attempts: int = DEFAULT_FAILOVER_ATTEMPTS,
|
||||
failover_delay: float = DEFAULT_FAILOVER_DELAY,
|
||||
auto_fallback_interval: float = DEFAULT_AUTO_FALLBACK_INTERVAL,
|
||||
):
|
||||
"""
|
||||
Initialize the DefaultCommandExecutor instance.
|
||||
|
||||
Args:
|
||||
failure_detectors: List of failure detector instances to monitor database health
|
||||
databases: Collection of available databases to execute commands on
|
||||
command_retry: Retry policy for failed command execution
|
||||
failover_strategy: Strategy for handling database failover
|
||||
event_dispatcher: Interface for dispatching events
|
||||
failover_attempts: Number of failover attempts
|
||||
failover_delay: Delay between failover attempts
|
||||
auto_fallback_interval: Time interval in seconds between attempts to fall back to a primary database
|
||||
"""
|
||||
super().__init__(auto_fallback_interval)
|
||||
|
||||
for fd in failure_detectors:
|
||||
fd.set_command_executor(command_executor=self)
|
||||
|
||||
self._databases = databases
|
||||
self._failure_detectors = failure_detectors
|
||||
self._command_retry = command_retry
|
||||
self._failover_strategy_executor = DefaultFailoverStrategyExecutor(
|
||||
failover_strategy, failover_attempts, failover_delay
|
||||
)
|
||||
self._event_dispatcher = event_dispatcher
|
||||
self._active_database: Optional[Database] = None
|
||||
self._active_pubsub: Optional[PubSub] = None
|
||||
self._active_pubsub_kwargs = {}
|
||||
self._setup_event_dispatcher()
|
||||
self._schedule_next_fallback()
|
||||
|
||||
@property
|
||||
def databases(self) -> Databases:
|
||||
return self._databases
|
||||
|
||||
@property
|
||||
def failure_detectors(self) -> List[AsyncFailureDetector]:
|
||||
return self._failure_detectors
|
||||
|
||||
def add_failure_detector(self, failure_detector: AsyncFailureDetector) -> None:
|
||||
self._failure_detectors.append(failure_detector)
|
||||
|
||||
@property
|
||||
def active_database(self) -> Optional[AsyncDatabase]:
|
||||
return self._active_database
|
||||
|
||||
async def set_active_database(
|
||||
self, database: AsyncDatabase, reason: GeoFailoverReason
|
||||
) -> None:
|
||||
old_active = self._active_database
|
||||
self._active_database = database
|
||||
|
||||
if old_active is not None and old_active is not database:
|
||||
await record_geo_failover(
|
||||
fail_from=old_active,
|
||||
fail_to=database,
|
||||
reason=reason,
|
||||
)
|
||||
await self._event_dispatcher.dispatch_async(
|
||||
AsyncActiveDatabaseChanged(
|
||||
old_active,
|
||||
self._active_database,
|
||||
self,
|
||||
**self._active_pubsub_kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def active_pubsub(self) -> Optional[PubSub]:
|
||||
return self._active_pubsub
|
||||
|
||||
@active_pubsub.setter
|
||||
def active_pubsub(self, pubsub: PubSub) -> None:
|
||||
self._active_pubsub = pubsub
|
||||
|
||||
@property
|
||||
def failover_strategy_executor(self) -> FailoverStrategyExecutor:
|
||||
return self._failover_strategy_executor
|
||||
|
||||
@property
|
||||
def command_retry(self) -> Retry:
|
||||
return self._command_retry
|
||||
|
||||
def pubsub(self, **kwargs):
|
||||
if self._active_pubsub is None:
|
||||
if isinstance(self._active_database.client, RedisCluster):
|
||||
raise ValueError("PubSub is not supported for RedisCluster")
|
||||
|
||||
self._active_pubsub = self._active_database.client.pubsub(**kwargs)
|
||||
self._active_pubsub_kwargs = kwargs
|
||||
|
||||
async def execute_command(self, *args, **options):
|
||||
async def callback():
|
||||
response = await self._active_database.client.execute_command(
|
||||
*args, **options
|
||||
)
|
||||
await self._register_command_execution(args)
|
||||
return response
|
||||
|
||||
return await self._execute_with_failure_detection(callback, args)
|
||||
|
||||
async def execute_pipeline(self, command_stack: tuple):
|
||||
async def callback():
|
||||
async with self._active_database.client.pipeline() as pipe:
|
||||
for command, options in command_stack:
|
||||
pipe.execute_command(*command, **options)
|
||||
|
||||
response = await pipe.execute()
|
||||
await self._register_command_execution(command_stack)
|
||||
return response
|
||||
|
||||
return await self._execute_with_failure_detection(callback, command_stack)
|
||||
|
||||
async def execute_transaction(
|
||||
self,
|
||||
func: Callable[["Pipeline"], Union[Any, Awaitable[Any]]],
|
||||
*watches: KeyT,
|
||||
shard_hint: Optional[str] = None,
|
||||
value_from_callable: bool = False,
|
||||
watch_delay: Optional[float] = None,
|
||||
):
|
||||
async def callback():
|
||||
response = await self._active_database.client.transaction(
|
||||
func,
|
||||
*watches,
|
||||
shard_hint=shard_hint,
|
||||
value_from_callable=value_from_callable,
|
||||
watch_delay=watch_delay,
|
||||
)
|
||||
await self._register_command_execution(())
|
||||
return response
|
||||
|
||||
return await self._execute_with_failure_detection(callback)
|
||||
|
||||
async def execute_pubsub_method(self, method_name: str, *args, **kwargs):
|
||||
async def callback():
|
||||
method = getattr(self.active_pubsub, method_name)
|
||||
if iscoroutinefunction(method):
|
||||
response = await method(*args, **kwargs)
|
||||
else:
|
||||
response = method(*args, **kwargs)
|
||||
|
||||
await self._register_command_execution(args)
|
||||
return response
|
||||
|
||||
return await self._execute_with_failure_detection(callback, *args)
|
||||
|
||||
async def execute_pubsub_run(
|
||||
self, sleep_time: float, exception_handler=None, pubsub=None
|
||||
) -> Any:
|
||||
async def callback():
|
||||
return await self._active_pubsub.run(
|
||||
poll_timeout=sleep_time,
|
||||
exception_handler=exception_handler,
|
||||
pubsub=pubsub,
|
||||
)
|
||||
|
||||
return await self._execute_with_failure_detection(callback)
|
||||
|
||||
async def _execute_with_failure_detection(
|
||||
self, callback: Callable, cmds: tuple = ()
|
||||
):
|
||||
"""
|
||||
Execute a commands execution callback with failure detection.
|
||||
"""
|
||||
|
||||
async def wrapper():
|
||||
# On each retry we need to check active database as it might change.
|
||||
await self._check_active_database()
|
||||
return await callback()
|
||||
|
||||
return await self._command_retry.call_with_retry(
|
||||
lambda: wrapper(),
|
||||
lambda error: self._on_command_fail(error, *cmds),
|
||||
)
|
||||
|
||||
async def _check_active_database(self):
|
||||
"""
|
||||
Checks if active a database needs to be updated.
|
||||
"""
|
||||
if (
|
||||
self._active_database is None
|
||||
or self._active_database.circuit.state != CBState.CLOSED
|
||||
or (
|
||||
self._auto_fallback_interval > 0
|
||||
and self._next_fallback_attempt <= datetime.now()
|
||||
)
|
||||
):
|
||||
await self.set_active_database(
|
||||
await self._failover_strategy_executor.execute(),
|
||||
GeoFailoverReason.AUTOMATIC,
|
||||
)
|
||||
self._schedule_next_fallback()
|
||||
|
||||
async def _on_command_fail(self, error, *args):
|
||||
await self._event_dispatcher.dispatch_async(
|
||||
AsyncOnCommandsFailEvent(args, error)
|
||||
)
|
||||
|
||||
async def _register_command_execution(self, cmd: tuple):
|
||||
for detector in self._failure_detectors:
|
||||
await detector.register_command_execution(cmd)
|
||||
|
||||
def _setup_event_dispatcher(self):
|
||||
"""
|
||||
Registers necessary listeners.
|
||||
"""
|
||||
failure_listener = RegisterCommandFailure(self._failure_detectors)
|
||||
resubscribe_listener = ResubscribeOnActiveDatabaseChanged()
|
||||
close_connection_listener = CloseConnectionOnActiveDatabaseChanged()
|
||||
self._event_dispatcher.register_listeners(
|
||||
{
|
||||
AsyncOnCommandsFailEvent: [failure_listener],
|
||||
AsyncActiveDatabaseChanged: [
|
||||
close_connection_listener,
|
||||
resubscribe_listener,
|
||||
],
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,232 @@
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import List, Optional, Type, Union
|
||||
|
||||
import pybreaker
|
||||
|
||||
from redis._defaults import DEFAULT_RETRY_BASE, DEFAULT_RETRY_CAP, DEFAULT_RETRY_COUNT
|
||||
from redis.asyncio import ConnectionPool, Redis, RedisCluster
|
||||
from redis.asyncio.multidb.database import Database, Databases
|
||||
from redis.asyncio.multidb.failover import (
|
||||
DEFAULT_FAILOVER_ATTEMPTS,
|
||||
DEFAULT_FAILOVER_DELAY,
|
||||
AsyncFailoverStrategy,
|
||||
WeightBasedFailoverStrategy,
|
||||
)
|
||||
from redis.asyncio.multidb.failure_detector import (
|
||||
AsyncFailureDetector,
|
||||
FailureDetectorAsyncWrapper,
|
||||
)
|
||||
from redis.asyncio.multidb.healthcheck import (
|
||||
DEFAULT_HEALTH_CHECK_DELAY,
|
||||
DEFAULT_HEALTH_CHECK_INTERVAL,
|
||||
DEFAULT_HEALTH_CHECK_POLICY,
|
||||
DEFAULT_HEALTH_CHECK_PROBES,
|
||||
DEFAULT_HEALTH_CHECK_TIMEOUT,
|
||||
HealthCheck,
|
||||
HealthCheckPolicies,
|
||||
PingHealthCheck,
|
||||
)
|
||||
from redis.asyncio.retry import Retry
|
||||
from redis.backoff import ExponentialWithJitterBackoff, NoBackoff
|
||||
from redis.data_structure import WeightedList
|
||||
from redis.event import EventDispatcher, EventDispatcherInterface
|
||||
from redis.multidb.circuit import (
|
||||
DEFAULT_GRACE_PERIOD,
|
||||
CircuitBreaker,
|
||||
PBCircuitBreakerAdapter,
|
||||
)
|
||||
from redis.multidb.failure_detector import (
|
||||
DEFAULT_FAILURE_RATE_THRESHOLD,
|
||||
DEFAULT_FAILURES_DETECTION_WINDOW,
|
||||
DEFAULT_MIN_NUM_FAILURES,
|
||||
CommandFailureDetector,
|
||||
)
|
||||
|
||||
DEFAULT_AUTO_FALLBACK_INTERVAL = 120
|
||||
|
||||
|
||||
class InitialHealthCheck(Enum):
|
||||
ALL_AVAILABLE = "all_available"
|
||||
MAJORITY_AVAILABLE = "majority_available"
|
||||
ONE_AVAILABLE = "one_available"
|
||||
|
||||
|
||||
def default_event_dispatcher() -> EventDispatcherInterface:
|
||||
return EventDispatcher()
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatabaseConfig:
|
||||
"""
|
||||
Dataclass representing the configuration for a database connection.
|
||||
|
||||
This class is used to store configuration settings for a database connection,
|
||||
including client options, connection sourcing details, circuit breaker settings,
|
||||
and cluster-specific properties. It provides a structure for defining these
|
||||
attributes and allows for the creation of customized configurations for various
|
||||
database setups.
|
||||
|
||||
Attributes:
|
||||
weight (float): Weight of the database to define the active one.
|
||||
client_kwargs (dict): Additional parameters for the database client connection.
|
||||
from_url (Optional[str]): Redis URL way of connecting to the database.
|
||||
from_pool (Optional[ConnectionPool]): A pre-configured connection pool to use.
|
||||
circuit (Optional[CircuitBreaker]): Custom circuit breaker implementation.
|
||||
grace_period (float): Grace period after which we need to check if the circuit could be closed again.
|
||||
health_check_url (Optional[str]): URL for health checks. Cluster FQDN is typically used
|
||||
on public Redis Enterprise endpoints.
|
||||
|
||||
Methods:
|
||||
default_circuit_breaker:
|
||||
Generates and returns a default CircuitBreaker instance adapted for use.
|
||||
"""
|
||||
|
||||
weight: float = 1.0
|
||||
client_kwargs: dict = field(default_factory=dict)
|
||||
from_url: Optional[str] = None
|
||||
from_pool: Optional[ConnectionPool] = None
|
||||
circuit: Optional[CircuitBreaker] = None
|
||||
grace_period: float = DEFAULT_GRACE_PERIOD
|
||||
health_check_url: Optional[str] = None
|
||||
|
||||
def default_circuit_breaker(self) -> CircuitBreaker:
|
||||
circuit_breaker = pybreaker.CircuitBreaker(reset_timeout=self.grace_period)
|
||||
return PBCircuitBreakerAdapter(circuit_breaker)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MultiDbConfig:
|
||||
"""
|
||||
Configuration class for managing multiple database connections in a resilient and fail-safe manner.
|
||||
|
||||
Attributes:
|
||||
databases_config: A list of database configurations.
|
||||
client_class: The client class used to manage database connections.
|
||||
command_retry: Retry strategy for executing database commands.
|
||||
failure_detectors: Optional list of additional failure detectors for monitoring database failures.
|
||||
min_num_failures: Minimal count of failures required for failover
|
||||
failure_rate_threshold: Percentage of failures required for failover
|
||||
failures_detection_window: Time interval for tracking database failures.
|
||||
health_checks: Optional list of additional health checks performed on databases.
|
||||
health_check_interval: Time interval for executing health checks.
|
||||
health_check_probes: Number of attempts to evaluate the health of a database.
|
||||
health_check_delay: Delay between health check attempts.
|
||||
health_check_timeout: Timeout for the full health check operation (including all probes).
|
||||
health_check_policy: Policy for determining database health based on health checks.
|
||||
failover_strategy: Optional strategy for handling database failover scenarios.
|
||||
failover_attempts: Number of retries allowed for failover operations.
|
||||
failover_delay: Delay between failover attempts.
|
||||
auto_fallback_interval: Time interval to trigger automatic fallback.
|
||||
event_dispatcher: Interface for dispatching events related to database operations.
|
||||
initial_health_check_policy: Defines the policy used to determine whether the databases setup is
|
||||
healthy during the initial health check.
|
||||
|
||||
Methods:
|
||||
databases:
|
||||
Retrieves a collection of database clients managed by weighted configurations.
|
||||
Initializes database clients based on the provided configuration and removes
|
||||
redundant retry objects for lower-level clients to rely on global retry logic.
|
||||
|
||||
default_failure_detectors:
|
||||
Returns the default list of failure detectors used to monitor database failures.
|
||||
|
||||
default_health_checks:
|
||||
Returns the default list of health checks used to monitor database health
|
||||
with specific retry and backoff strategies.
|
||||
|
||||
default_failover_strategy:
|
||||
Provides the default failover strategy used for handling failover scenarios
|
||||
with defined retry and backoff configurations.
|
||||
"""
|
||||
|
||||
databases_config: List[DatabaseConfig]
|
||||
client_class: Type[Union[Redis, RedisCluster]] = Redis
|
||||
command_retry: Retry = Retry(
|
||||
backoff=ExponentialWithJitterBackoff(
|
||||
base=DEFAULT_RETRY_BASE, cap=DEFAULT_RETRY_CAP
|
||||
),
|
||||
retries=DEFAULT_RETRY_COUNT,
|
||||
)
|
||||
failure_detectors: Optional[List[AsyncFailureDetector]] = None
|
||||
min_num_failures: int = DEFAULT_MIN_NUM_FAILURES
|
||||
failure_rate_threshold: float = DEFAULT_FAILURE_RATE_THRESHOLD
|
||||
failures_detection_window: float = DEFAULT_FAILURES_DETECTION_WINDOW
|
||||
health_checks: Optional[List[HealthCheck]] = None
|
||||
health_check_interval: float = DEFAULT_HEALTH_CHECK_INTERVAL
|
||||
health_check_probes: int = DEFAULT_HEALTH_CHECK_PROBES
|
||||
health_check_delay: float = DEFAULT_HEALTH_CHECK_DELAY
|
||||
health_check_timeout: float = DEFAULT_HEALTH_CHECK_TIMEOUT
|
||||
health_check_policy: HealthCheckPolicies = DEFAULT_HEALTH_CHECK_POLICY
|
||||
failover_strategy: Optional[AsyncFailoverStrategy] = None
|
||||
failover_attempts: int = DEFAULT_FAILOVER_ATTEMPTS
|
||||
failover_delay: float = DEFAULT_FAILOVER_DELAY
|
||||
auto_fallback_interval: float = DEFAULT_AUTO_FALLBACK_INTERVAL
|
||||
event_dispatcher: EventDispatcherInterface = field(
|
||||
default_factory=default_event_dispatcher
|
||||
)
|
||||
initial_health_check_policy: InitialHealthCheck = InitialHealthCheck.ALL_AVAILABLE
|
||||
|
||||
def databases(self) -> Databases:
|
||||
databases = WeightedList()
|
||||
|
||||
for database_config in self.databases_config:
|
||||
# The retry object is not used in the lower level clients, so we can safely remove it.
|
||||
# We rely on command_retry in terms of global retries.
|
||||
database_config.client_kwargs.update(
|
||||
{"retry": Retry(retries=0, backoff=NoBackoff())}
|
||||
)
|
||||
|
||||
if database_config.from_url:
|
||||
client = self.client_class.from_url(
|
||||
database_config.from_url, **database_config.client_kwargs
|
||||
)
|
||||
elif database_config.from_pool:
|
||||
database_config.from_pool.set_retry(
|
||||
Retry(retries=0, backoff=NoBackoff())
|
||||
)
|
||||
client = self.client_class.from_pool(
|
||||
connection_pool=database_config.from_pool
|
||||
)
|
||||
else:
|
||||
client = self.client_class(**database_config.client_kwargs)
|
||||
|
||||
circuit = (
|
||||
database_config.default_circuit_breaker()
|
||||
if database_config.circuit is None
|
||||
else database_config.circuit
|
||||
)
|
||||
databases.add(
|
||||
Database(
|
||||
client=client,
|
||||
circuit=circuit,
|
||||
weight=database_config.weight,
|
||||
health_check_url=database_config.health_check_url,
|
||||
),
|
||||
database_config.weight,
|
||||
)
|
||||
|
||||
return databases
|
||||
|
||||
def default_failure_detectors(self) -> List[AsyncFailureDetector]:
|
||||
return [
|
||||
FailureDetectorAsyncWrapper(
|
||||
CommandFailureDetector(
|
||||
min_num_failures=self.min_num_failures,
|
||||
failure_rate_threshold=self.failure_rate_threshold,
|
||||
failure_detection_window=self.failures_detection_window,
|
||||
)
|
||||
),
|
||||
]
|
||||
|
||||
def default_health_checks(self) -> List[HealthCheck]:
|
||||
return [
|
||||
PingHealthCheck(
|
||||
health_check_probes=self.health_check_probes,
|
||||
health_check_delay=self.health_check_delay,
|
||||
health_check_timeout=self.health_check_timeout,
|
||||
),
|
||||
]
|
||||
|
||||
def default_failover_strategy(self) -> AsyncFailoverStrategy:
|
||||
return WeightBasedFailoverStrategy()
|
||||
@@ -0,0 +1,72 @@
|
||||
from abc import abstractmethod
|
||||
from typing import Optional, Union
|
||||
|
||||
from redis.asyncio import Redis, RedisCluster
|
||||
from redis.data_structure import WeightedList
|
||||
from redis.multidb.circuit import CircuitBreaker
|
||||
from redis.multidb.database import AbstractDatabase, BaseDatabase
|
||||
from redis.typing import Number
|
||||
|
||||
|
||||
class AsyncDatabase(AbstractDatabase):
|
||||
"""Database with an underlying asynchronous redis client."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def client(self) -> Union[Redis, RedisCluster]:
|
||||
"""The underlying redis client."""
|
||||
pass
|
||||
|
||||
@client.setter
|
||||
@abstractmethod
|
||||
def client(self, client: Union[Redis, RedisCluster]):
|
||||
"""Set the underlying redis client."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def circuit(self) -> CircuitBreaker:
|
||||
"""Circuit breaker for the current database."""
|
||||
pass
|
||||
|
||||
@circuit.setter
|
||||
@abstractmethod
|
||||
def circuit(self, circuit: CircuitBreaker):
|
||||
"""Set the circuit breaker for the current database."""
|
||||
pass
|
||||
|
||||
|
||||
Databases = WeightedList[tuple[AsyncDatabase, Number]]
|
||||
|
||||
|
||||
class Database(BaseDatabase, AsyncDatabase):
|
||||
def __init__(
|
||||
self,
|
||||
client: Union[Redis, RedisCluster],
|
||||
circuit: CircuitBreaker,
|
||||
weight: float,
|
||||
health_check_url: Optional[str] = None,
|
||||
):
|
||||
self._client = client
|
||||
self._cb = circuit
|
||||
self._cb.database = self
|
||||
super().__init__(weight, health_check_url)
|
||||
|
||||
@property
|
||||
def client(self) -> Union[Redis, RedisCluster]:
|
||||
return self._client
|
||||
|
||||
@client.setter
|
||||
def client(self, client: Union[Redis, RedisCluster]):
|
||||
self._client = client
|
||||
|
||||
@property
|
||||
def circuit(self) -> CircuitBreaker:
|
||||
return self._cb
|
||||
|
||||
@circuit.setter
|
||||
def circuit(self, circuit: CircuitBreaker):
|
||||
self._cb = circuit
|
||||
|
||||
def __repr__(self):
|
||||
return f"Database(client={self.client}, weight={self.weight})"
|
||||
@@ -0,0 +1,84 @@
|
||||
from typing import List
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from redis.asyncio.multidb.database import AsyncDatabase
|
||||
from redis.asyncio.multidb.failure_detector import AsyncFailureDetector
|
||||
from redis.event import AsyncEventListenerInterface, AsyncOnCommandsFailEvent
|
||||
|
||||
|
||||
class AsyncActiveDatabaseChanged:
|
||||
"""
|
||||
Event fired when an async active database has been changed.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
old_database: AsyncDatabase,
|
||||
new_database: AsyncDatabase,
|
||||
command_executor,
|
||||
**kwargs,
|
||||
):
|
||||
self._old_database = old_database
|
||||
self._new_database = new_database
|
||||
self._command_executor = command_executor
|
||||
self._kwargs = kwargs
|
||||
|
||||
@property
|
||||
def old_database(self) -> AsyncDatabase:
|
||||
return self._old_database
|
||||
|
||||
@property
|
||||
def new_database(self) -> AsyncDatabase:
|
||||
return self._new_database
|
||||
|
||||
@property
|
||||
def command_executor(self):
|
||||
return self._command_executor
|
||||
|
||||
@property
|
||||
def kwargs(self):
|
||||
return self._kwargs
|
||||
|
||||
|
||||
class ResubscribeOnActiveDatabaseChanged(AsyncEventListenerInterface):
|
||||
"""
|
||||
Re-subscribe the currently active pub / sub to a new active database.
|
||||
"""
|
||||
|
||||
async def listen(self, event: AsyncActiveDatabaseChanged):
|
||||
old_pubsub = event.command_executor.active_pubsub
|
||||
|
||||
if old_pubsub is not None:
|
||||
# Re-assign old channels and patterns so they will be automatically subscribed on connection.
|
||||
new_pubsub = event.new_database.client.pubsub(**event.kwargs)
|
||||
new_pubsub.channels = old_pubsub.channels
|
||||
new_pubsub.patterns = old_pubsub.patterns
|
||||
await new_pubsub.on_connect(None)
|
||||
event.command_executor.active_pubsub = new_pubsub
|
||||
await old_pubsub.aclose()
|
||||
|
||||
|
||||
class CloseConnectionOnActiveDatabaseChanged(AsyncEventListenerInterface):
|
||||
"""
|
||||
Close connection to the old active database.
|
||||
"""
|
||||
|
||||
async def listen(self, event: AsyncActiveDatabaseChanged):
|
||||
await event.old_database.client.aclose()
|
||||
|
||||
if isinstance(event.old_database.client, Redis):
|
||||
await event.old_database.client.connection_pool.update_active_connections_for_reconnect()
|
||||
await event.old_database.client.connection_pool.disconnect()
|
||||
|
||||
|
||||
class RegisterCommandFailure(AsyncEventListenerInterface):
|
||||
"""
|
||||
Event listener that registers command failures and passing it to the failure detectors.
|
||||
"""
|
||||
|
||||
def __init__(self, failure_detectors: List[AsyncFailureDetector]):
|
||||
self._failure_detectors = failure_detectors
|
||||
|
||||
async def listen(self, event: AsyncOnCommandsFailEvent) -> None:
|
||||
for failure_detector in self._failure_detectors:
|
||||
await failure_detector.register_failure(event.exception, event.commands)
|
||||
@@ -0,0 +1,125 @@
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from redis.asyncio.multidb.database import AsyncDatabase, Databases
|
||||
from redis.data_structure import WeightedList
|
||||
from redis.multidb.circuit import State as CBState
|
||||
from redis.multidb.exception import (
|
||||
NoValidDatabaseException,
|
||||
TemporaryUnavailableException,
|
||||
)
|
||||
|
||||
DEFAULT_FAILOVER_ATTEMPTS = 10
|
||||
DEFAULT_FAILOVER_DELAY = 12
|
||||
|
||||
|
||||
class AsyncFailoverStrategy(ABC):
|
||||
@abstractmethod
|
||||
async def database(self) -> AsyncDatabase:
|
||||
"""Select the database according to the strategy."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def set_databases(self, databases: Databases) -> None:
|
||||
"""Set the database strategy operates on."""
|
||||
pass
|
||||
|
||||
|
||||
class FailoverStrategyExecutor(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
def failover_attempts(self) -> int:
|
||||
"""The number of failover attempts."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def failover_delay(self) -> float:
|
||||
"""The delay between failover attempts."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def strategy(self) -> AsyncFailoverStrategy:
|
||||
"""The strategy to execute."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def execute(self) -> AsyncDatabase:
|
||||
"""Execute the failover strategy."""
|
||||
pass
|
||||
|
||||
|
||||
class WeightBasedFailoverStrategy(AsyncFailoverStrategy):
|
||||
"""
|
||||
Failover strategy based on database weights.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._databases = WeightedList()
|
||||
|
||||
async def database(self) -> AsyncDatabase:
|
||||
for database, _ in self._databases:
|
||||
if database.circuit.state == CBState.CLOSED:
|
||||
return database
|
||||
|
||||
raise NoValidDatabaseException("No valid database available for communication")
|
||||
|
||||
def set_databases(self, databases: Databases) -> None:
|
||||
self._databases = databases
|
||||
|
||||
|
||||
class DefaultFailoverStrategyExecutor(FailoverStrategyExecutor):
|
||||
"""
|
||||
Executes given failover strategy.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
strategy: AsyncFailoverStrategy,
|
||||
failover_attempts: int = DEFAULT_FAILOVER_ATTEMPTS,
|
||||
failover_delay: float = DEFAULT_FAILOVER_DELAY,
|
||||
):
|
||||
self._strategy = strategy
|
||||
self._failover_attempts = failover_attempts
|
||||
self._failover_delay = failover_delay
|
||||
self._next_attempt_ts: int = 0
|
||||
self._failover_counter: int = 0
|
||||
|
||||
@property
|
||||
def failover_attempts(self) -> int:
|
||||
return self._failover_attempts
|
||||
|
||||
@property
|
||||
def failover_delay(self) -> float:
|
||||
return self._failover_delay
|
||||
|
||||
@property
|
||||
def strategy(self) -> AsyncFailoverStrategy:
|
||||
return self._strategy
|
||||
|
||||
async def execute(self) -> AsyncDatabase:
|
||||
try:
|
||||
database = await self._strategy.database()
|
||||
self._reset()
|
||||
return database
|
||||
except NoValidDatabaseException as e:
|
||||
if self._next_attempt_ts == 0:
|
||||
self._next_attempt_ts = time.time() + self._failover_delay
|
||||
self._failover_counter += 1
|
||||
elif time.time() >= self._next_attempt_ts:
|
||||
self._next_attempt_ts += self._failover_delay
|
||||
self._failover_counter += 1
|
||||
|
||||
if self._failover_counter > self._failover_attempts:
|
||||
self._reset()
|
||||
raise e
|
||||
else:
|
||||
raise TemporaryUnavailableException(
|
||||
"No database connections currently available. "
|
||||
"This is a temporary condition - please retry the operation."
|
||||
)
|
||||
|
||||
def _reset(self) -> None:
|
||||
self._next_attempt_ts = 0
|
||||
self._failover_counter = 0
|
||||
@@ -0,0 +1,38 @@
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from redis.multidb.failure_detector import FailureDetector
|
||||
|
||||
|
||||
class AsyncFailureDetector(ABC):
|
||||
@abstractmethod
|
||||
async def register_failure(self, exception: Exception, cmd: tuple) -> None:
|
||||
"""Register a failure that occurred during command execution."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def register_command_execution(self, cmd: tuple) -> None:
|
||||
"""Register a command execution."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def set_command_executor(self, command_executor) -> None:
|
||||
"""Set the command executor for this failure."""
|
||||
pass
|
||||
|
||||
|
||||
class FailureDetectorAsyncWrapper(AsyncFailureDetector):
|
||||
"""
|
||||
Async wrapper for the failure detector.
|
||||
"""
|
||||
|
||||
def __init__(self, failure_detector: FailureDetector) -> None:
|
||||
self._failure_detector = failure_detector
|
||||
|
||||
async def register_failure(self, exception: Exception, cmd: tuple) -> None:
|
||||
self._failure_detector.register_failure(exception, cmd)
|
||||
|
||||
async def register_command_execution(self, cmd: tuple) -> None:
|
||||
self._failure_detector.register_command_execution(cmd)
|
||||
|
||||
def set_command_executor(self, command_executor) -> None:
|
||||
self._failure_detector.set_command_executor(command_executor)
|
||||
@@ -0,0 +1,508 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from enum import Enum
|
||||
from typing import List, Optional, Tuple, Type, Union
|
||||
|
||||
from redis.asyncio import Redis as AsyncRedis
|
||||
from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster
|
||||
from redis.asyncio.http.http_client import DEFAULT_TIMEOUT, AsyncHTTPClientWrapper
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.client import Redis as SyncRedis
|
||||
from redis.cluster import RedisCluster as SyncRedisCluster
|
||||
from redis.http.http_client import HttpClient
|
||||
from redis.multidb.exception import UnhealthyDatabaseException
|
||||
from redis.retry import Retry
|
||||
|
||||
# Type alias for async Redis clients (standalone or cluster)
|
||||
AsyncRedisClientT = Union[AsyncRedis, AsyncRedisCluster]
|
||||
|
||||
|
||||
def _get_init_params(cls: Type) -> frozenset:
|
||||
"""Extract parameter names from a class's __init__ method."""
|
||||
sig = inspect.signature(cls.__init__)
|
||||
return frozenset(
|
||||
name
|
||||
for name, param in sig.parameters.items()
|
||||
if name != "self"
|
||||
and param.kind
|
||||
in (
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.KEYWORD_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _filter_kwargs(kwargs: dict, cls: Type) -> dict:
|
||||
"""Filter kwargs to only include parameters accepted by the class's __init__."""
|
||||
allowed = _get_init_params(cls)
|
||||
return {k: v for k, v in kwargs.items() if k in allowed}
|
||||
|
||||
|
||||
DEFAULT_HEALTH_CHECK_PROBES = 3
|
||||
DEFAULT_HEALTH_CHECK_INTERVAL = 5
|
||||
DEFAULT_HEALTH_CHECK_TIMEOUT = 3
|
||||
DEFAULT_HEALTH_CHECK_DELAY = 0.5
|
||||
DEFAULT_LAG_AWARE_TOLERANCE = 5000
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class HealthCheck(ABC):
|
||||
"""
|
||||
Health check interface.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def health_check_probes(self) -> int:
|
||||
"""Number of probes to execute health checks."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def health_check_delay(self) -> float:
|
||||
"""Delay between health check probes."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def health_check_timeout(self) -> float:
|
||||
"""Timeout for the full health check operation (including all probes)."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def check_health(self, database, hc_client: AsyncRedisClientT) -> bool:
|
||||
"""
|
||||
Function to determine the health status.
|
||||
|
||||
Args:
|
||||
database: The database being checked
|
||||
hc_client: A Redis client (AsyncRedis or AsyncRedisCluster) to use for
|
||||
health checks. This client follows topology changes automatically.
|
||||
|
||||
Returns:
|
||||
True if the database is healthy, False otherwise.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class HealthCheckPolicy(ABC):
|
||||
"""
|
||||
Health checks execution policy.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def execute(self, health_checks: List[HealthCheck], database) -> bool:
|
||||
"""Execute health checks and return database health status."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def _execute(self, health_check: HealthCheck, database) -> bool:
|
||||
"""
|
||||
Executes health check against given database.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_client(self, database) -> AsyncRedisClientT:
|
||||
"""
|
||||
Get a health check client for the database.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def close(self) -> None:
|
||||
"""Close all health check clients."""
|
||||
pass
|
||||
|
||||
|
||||
class AbstractHealthCheckPolicy(HealthCheckPolicy):
|
||||
"""
|
||||
Abstract health check policy.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Single client per database, keyed by database id
|
||||
self._clients: dict[int, AsyncRedisClientT] = {}
|
||||
|
||||
async def execute(self, health_checks: List[HealthCheck], database) -> bool:
|
||||
"""
|
||||
Execute all health checks concurrently with individual timeouts.
|
||||
Each health check runs with its own timeout, and all run in parallel.
|
||||
|
||||
All exception handling is centralized here - _execute() methods just
|
||||
propagate exceptions naturally.
|
||||
"""
|
||||
|
||||
# Create wrapper tasks that apply individual timeouts
|
||||
async def execute_with_timeout(health_check: HealthCheck):
|
||||
return await asyncio.wait_for(
|
||||
self._execute(health_check, database),
|
||||
timeout=health_check.health_check_timeout,
|
||||
)
|
||||
|
||||
# Run all health checks concurrently and collect results/exceptions
|
||||
results = await asyncio.gather(
|
||||
*[execute_with_timeout(hc) for hc in health_checks],
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
# Check results - handle exceptions and failures
|
||||
for result in results:
|
||||
if isinstance(result, Exception):
|
||||
# Any exception (including TimeoutError) makes the database unhealthy
|
||||
raise UnhealthyDatabaseException("Unhealthy database", database, result)
|
||||
elif not result:
|
||||
# Health check returned False
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
async def get_client(self, database) -> AsyncRedisClientT:
|
||||
"""
|
||||
Get or create a health check client for the database.
|
||||
|
||||
Creates a single client instance per database that follows topology
|
||||
changes automatically. For cluster databases, the client handles
|
||||
node discovery and slot mapping internally.
|
||||
"""
|
||||
db_id = id(database)
|
||||
client = self._clients.get(db_id)
|
||||
|
||||
if client is None:
|
||||
# Check for both sync and async standalone Redis clients
|
||||
if isinstance(database.client, (AsyncRedis, SyncRedis)):
|
||||
conn_kwargs = database.client.get_connection_kwargs()
|
||||
filtered_kwargs = _filter_kwargs(conn_kwargs, AsyncRedis)
|
||||
client = AsyncRedis(**filtered_kwargs)
|
||||
elif isinstance(database.client, (AsyncRedisCluster, SyncRedisCluster)):
|
||||
# Cluster client - create a single cluster client that handles
|
||||
# topology changes internally
|
||||
conn_kwargs = database.client.get_connection_kwargs().copy()
|
||||
filtered_kwargs = _filter_kwargs(conn_kwargs, AsyncRedisCluster)
|
||||
startup_nodes = database.client.startup_nodes
|
||||
# Use the first node as the startup node
|
||||
if startup_nodes:
|
||||
first_node = startup_nodes[0]
|
||||
nodes_manager = database.client.nodes_manager
|
||||
# The sync and async NodesManager expose this setting under
|
||||
# different names (``_require_full_coverage`` vs
|
||||
# ``require_full_coverage``), so resolve it defensively to
|
||||
# support a sync RedisCluster underlying client too.
|
||||
require_full_coverage = getattr(
|
||||
nodes_manager,
|
||||
"require_full_coverage",
|
||||
getattr(nodes_manager, "_require_full_coverage", True),
|
||||
)
|
||||
client = AsyncRedisCluster(
|
||||
host=first_node.host,
|
||||
port=first_node.port,
|
||||
dynamic_startup_nodes=nodes_manager._dynamic_startup_nodes,
|
||||
address_remap=nodes_manager.address_remap,
|
||||
require_full_coverage=require_full_coverage,
|
||||
retry=database.client.retry,
|
||||
**filtered_kwargs,
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Cluster client has no nodes - cannot create health check client"
|
||||
)
|
||||
else:
|
||||
raise TypeError(f"Unsupported client type: {type(database.client)}")
|
||||
self._clients[db_id] = client
|
||||
|
||||
return client
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close all health check clients."""
|
||||
close_tasks = [
|
||||
asyncio.create_task(client.aclose()) for client in self._clients.values()
|
||||
]
|
||||
|
||||
if close_tasks:
|
||||
await asyncio.gather(*close_tasks, return_exceptions=True)
|
||||
|
||||
self._clients.clear()
|
||||
|
||||
@abstractmethod
|
||||
async def _execute(self, health_check: HealthCheck, database) -> bool:
|
||||
"""
|
||||
Executes health check against given database.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class HealthyAllPolicy(AbstractHealthCheckPolicy):
|
||||
"""
|
||||
Policy that returns True if all health check probes are successful.
|
||||
"""
|
||||
|
||||
async def _execute(self, health_check: HealthCheck, database) -> bool:
|
||||
"""
|
||||
Executes health check against given database.
|
||||
|
||||
Uses a single client that handles topology changes automatically.
|
||||
"""
|
||||
client = await self.get_client(database)
|
||||
probes = health_check.health_check_probes
|
||||
|
||||
for attempt in range(probes):
|
||||
result = await health_check.check_health(database, client)
|
||||
if not result:
|
||||
return False
|
||||
|
||||
if attempt < probes - 1:
|
||||
await asyncio.sleep(health_check.health_check_delay)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class HealthyMajorityPolicy(AbstractHealthCheckPolicy):
|
||||
"""
|
||||
Policy that returns True if a majority of health check probes are successful.
|
||||
|
||||
Majority means more than half must pass:
|
||||
- 3 probes: need 2+ to pass (1 failure allowed)
|
||||
- 4 probes: need 3+ to pass (1 failure allowed, tie = unhealthy)
|
||||
- 5 probes: need 3+ to pass (2 failures allowed)
|
||||
"""
|
||||
|
||||
async def _execute(self, health_check: HealthCheck, database) -> bool:
|
||||
"""
|
||||
Executes health check against given database.
|
||||
|
||||
Uses a single client that handles topology changes automatically.
|
||||
"""
|
||||
probes = health_check.health_check_probes
|
||||
# Strict majority: more than half must pass
|
||||
# (probes - 1) // 2 gives the max allowed failures
|
||||
allowed_unsuccessful_probes = (probes - 1) // 2
|
||||
client = await self.get_client(database)
|
||||
last_exception = None
|
||||
|
||||
for attempt in range(probes):
|
||||
try:
|
||||
result = await health_check.check_health(database, client)
|
||||
if not result:
|
||||
# Probe failed (returned False)
|
||||
allowed_unsuccessful_probes -= 1
|
||||
if allowed_unsuccessful_probes < 0:
|
||||
return False
|
||||
except Exception as e:
|
||||
# Probe failed (exception)
|
||||
last_exception = e
|
||||
allowed_unsuccessful_probes -= 1
|
||||
if allowed_unsuccessful_probes < 0:
|
||||
raise last_exception
|
||||
|
||||
if attempt < probes - 1:
|
||||
await asyncio.sleep(health_check.health_check_delay)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class HealthyAnyPolicy(AbstractHealthCheckPolicy):
|
||||
"""
|
||||
Policy that returns True if at least one health check probe is successful.
|
||||
"""
|
||||
|
||||
async def _execute(self, health_check: HealthCheck, database) -> bool:
|
||||
"""
|
||||
Executes health check against given database.
|
||||
|
||||
Uses a single client that handles topology changes automatically.
|
||||
"""
|
||||
probes = health_check.health_check_probes
|
||||
last_exception = None
|
||||
client = await self.get_client(database)
|
||||
|
||||
for attempt in range(probes):
|
||||
try:
|
||||
result = await health_check.check_health(database, client)
|
||||
if result:
|
||||
# At least one probe succeeded
|
||||
return True
|
||||
except Exception as e:
|
||||
last_exception = e
|
||||
|
||||
if attempt < probes - 1:
|
||||
await asyncio.sleep(health_check.health_check_delay)
|
||||
|
||||
# All probes failed
|
||||
if last_exception:
|
||||
raise last_exception
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class HealthCheckPolicies(Enum):
|
||||
HEALTHY_ALL = HealthyAllPolicy
|
||||
HEALTHY_MAJORITY = HealthyMajorityPolicy
|
||||
HEALTHY_ANY = HealthyAnyPolicy
|
||||
|
||||
|
||||
DEFAULT_HEALTH_CHECK_POLICY: HealthCheckPolicies = HealthCheckPolicies.HEALTHY_ALL
|
||||
|
||||
|
||||
class AbstractHealthCheck(HealthCheck):
|
||||
def __init__(
|
||||
self,
|
||||
health_check_probes: int = DEFAULT_HEALTH_CHECK_PROBES,
|
||||
health_check_delay: float = DEFAULT_HEALTH_CHECK_DELAY,
|
||||
health_check_timeout: float = DEFAULT_HEALTH_CHECK_TIMEOUT,
|
||||
):
|
||||
if health_check_probes < 1:
|
||||
raise ValueError("health_check_probes must be greater than 0")
|
||||
self._health_check_probes = health_check_probes
|
||||
self._health_check_delay = health_check_delay
|
||||
self._health_check_timeout = health_check_timeout
|
||||
|
||||
@property
|
||||
def health_check_probes(self) -> int:
|
||||
return self._health_check_probes
|
||||
|
||||
@property
|
||||
def health_check_delay(self) -> float:
|
||||
return self._health_check_delay
|
||||
|
||||
@property
|
||||
def health_check_timeout(self) -> float:
|
||||
return self._health_check_timeout
|
||||
|
||||
@abstractmethod
|
||||
async def check_health(self, database, hc_client: AsyncRedisClientT) -> bool:
|
||||
pass
|
||||
|
||||
|
||||
class PingHealthCheck(AbstractHealthCheck):
|
||||
"""
|
||||
Health check based on PING command.
|
||||
"""
|
||||
|
||||
async def check_health(self, database, hc_client: AsyncRedisClientT) -> bool:
|
||||
if isinstance(hc_client, AsyncRedis):
|
||||
return await hc_client.execute_command("PING")
|
||||
else:
|
||||
# For a cluster checks if all nodes are healthy.
|
||||
all_nodes = hc_client.get_nodes()
|
||||
for node in all_nodes:
|
||||
if not await node.redis_connection.execute_command("PING"):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class LagAwareHealthCheck(AbstractHealthCheck):
|
||||
"""
|
||||
Health check available for Redis Enterprise deployments.
|
||||
Verify via REST API that the database is healthy based on different lags.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
rest_api_port: int = 9443,
|
||||
lag_aware_tolerance: int = DEFAULT_LAG_AWARE_TOLERANCE,
|
||||
http_timeout: float = DEFAULT_TIMEOUT,
|
||||
auth_basic: Optional[Tuple[str, str]] = None,
|
||||
verify_tls: bool = True,
|
||||
# TLS verification (server) options
|
||||
ca_file: Optional[str] = None,
|
||||
ca_path: Optional[str] = None,
|
||||
ca_data: Optional[Union[str, bytes]] = None,
|
||||
# Mutual TLS (client cert) options
|
||||
client_cert_file: Optional[str] = None,
|
||||
client_key_file: Optional[str] = None,
|
||||
client_key_password: Optional[str] = None,
|
||||
# Health check configuration
|
||||
health_check_probes: int = DEFAULT_HEALTH_CHECK_PROBES,
|
||||
health_check_delay: float = DEFAULT_HEALTH_CHECK_DELAY,
|
||||
health_check_timeout: float = DEFAULT_HEALTH_CHECK_TIMEOUT,
|
||||
):
|
||||
"""
|
||||
Initialize LagAwareHealthCheck with the specified parameters.
|
||||
|
||||
Args:
|
||||
rest_api_port: Port number for Redis Enterprise REST API (default: 9443)
|
||||
lag_aware_tolerance: Tolerance in lag between databases in MS (default: 100)
|
||||
http_timeout: Request timeout in seconds (default: DEFAULT_TIMEOUT)
|
||||
auth_basic: Tuple of (username, password) for basic authentication
|
||||
verify_tls: Whether to verify TLS certificates (default: True)
|
||||
ca_file: Path to CA certificate file for TLS verification
|
||||
ca_path: Path to CA certificates directory for TLS verification
|
||||
ca_data: CA certificate data as string or bytes
|
||||
client_cert_file: Path to client certificate file for mutual TLS
|
||||
client_key_file: Path to client private key file for mutual TLS
|
||||
client_key_password: Password for encrypted client private key
|
||||
"""
|
||||
self._http_client = AsyncHTTPClientWrapper(
|
||||
HttpClient(
|
||||
timeout=http_timeout,
|
||||
auth_basic=auth_basic,
|
||||
retry=Retry(NoBackoff(), retries=0),
|
||||
verify_tls=verify_tls,
|
||||
ca_file=ca_file,
|
||||
ca_path=ca_path,
|
||||
ca_data=ca_data,
|
||||
client_cert_file=client_cert_file,
|
||||
client_key_file=client_key_file,
|
||||
client_key_password=client_key_password,
|
||||
)
|
||||
)
|
||||
self._rest_api_port = rest_api_port
|
||||
self._lag_aware_tolerance = lag_aware_tolerance
|
||||
super().__init__(
|
||||
health_check_probes=health_check_probes,
|
||||
health_check_delay=health_check_delay,
|
||||
health_check_timeout=health_check_timeout,
|
||||
)
|
||||
|
||||
async def check_health(self, database, hc_client: AsyncRedisClientT) -> bool:
|
||||
"""
|
||||
Check database health via Redis Enterprise REST API.
|
||||
|
||||
Note: The client parameter is not used for this health check as it
|
||||
relies on the REST API instead of Redis protocol. The client is
|
||||
accepted for interface compatibility.
|
||||
"""
|
||||
if database.health_check_url is None:
|
||||
raise ValueError(
|
||||
"Database health check url is not set. Please check DatabaseConfig for the current database."
|
||||
)
|
||||
|
||||
if isinstance(database.client, (AsyncRedis, SyncRedis)):
|
||||
db_host = database.client.get_connection_kwargs()["host"]
|
||||
else:
|
||||
# Cluster client
|
||||
db_host = database.client.get_nodes()[0].host
|
||||
|
||||
base_url = f"{database.health_check_url}:{self._rest_api_port}"
|
||||
self._http_client.client.base_url = base_url
|
||||
|
||||
# Find bdb matching to the current database host
|
||||
matching_bdb = None
|
||||
for bdb in await self._http_client.get("/v1/bdbs"):
|
||||
for endpoint in bdb["endpoints"]:
|
||||
if endpoint["dns_name"] == db_host:
|
||||
matching_bdb = bdb
|
||||
break
|
||||
|
||||
# In case if the host was set as public IP
|
||||
for addr in endpoint["addr"]:
|
||||
if addr == db_host:
|
||||
matching_bdb = bdb
|
||||
break
|
||||
|
||||
if matching_bdb is None:
|
||||
logger.warning("LagAwareHealthCheck failed: Couldn't find a matching bdb")
|
||||
raise ValueError("Could not find a matching bdb")
|
||||
|
||||
url = (
|
||||
f"/v1/bdbs/{matching_bdb['uid']}/availability"
|
||||
f"?extend_check=lag&availability_lag_tolerance_ms={self._lag_aware_tolerance}"
|
||||
)
|
||||
await self._http_client.get(url, expect_json=False)
|
||||
|
||||
# Status checked in an http client, otherwise HttpError will be raised
|
||||
return True
|
||||
@@ -0,0 +1,14 @@
|
||||
"""
|
||||
Async observability module for Redis async clients.
|
||||
|
||||
This module provides async-safe APIs for recording Redis metrics using OpenTelemetry.
|
||||
|
||||
Usage:
|
||||
from redis.asyncio.observability.recorder import record_operation_duration
|
||||
|
||||
Configuration is shared with the sync observability module:
|
||||
from redis.observability import get_observability_instance, OTelConfig
|
||||
|
||||
otel = get_observability_instance()
|
||||
otel.init(OTelConfig())
|
||||
"""
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,635 @@
|
||||
"""
|
||||
Async-compatible API for recording observability metrics.
|
||||
|
||||
This module provides an async-safe interface for Redis async client code to record
|
||||
metrics without needing to know about OpenTelemetry internals. It reuses the same
|
||||
RedisMetricsCollector and configuration as the sync recorder.
|
||||
|
||||
Usage in Redis async client code:
|
||||
from redis.asyncio.observability.recorder import record_operation_duration
|
||||
|
||||
start_time = time.monotonic()
|
||||
# ... execute Redis command ...
|
||||
await record_operation_duration(
|
||||
command_name='SET',
|
||||
duration_seconds=time.monotonic() - start_time,
|
||||
server_address='localhost',
|
||||
server_port=6379,
|
||||
db_namespace='0',
|
||||
error=None
|
||||
)
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, List, Optional
|
||||
|
||||
from redis.observability.attributes import (
|
||||
ConnectionState,
|
||||
GeoFailoverReason,
|
||||
PubSubDirection,
|
||||
)
|
||||
from redis.observability.metrics import CloseReason, RedisMetricsCollector
|
||||
from redis.observability.providers import get_observability_instance
|
||||
from redis.observability.registry import get_observables_registry_instance
|
||||
from redis.utils import deprecated_function, str_if_bytes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from redis.asyncio.connection import ConnectionPool
|
||||
from redis.asyncio.multidb.database import AsyncDatabase
|
||||
from redis.observability.config import OTelConfig
|
||||
|
||||
# Global metrics collector instance (lazy-initialized)
|
||||
_async_metrics_collector: Optional[RedisMetricsCollector] = None
|
||||
|
||||
CONNECTION_COUNT_REGISTRY_KEY = "connection_count"
|
||||
|
||||
|
||||
def _get_or_create_collector() -> Optional[RedisMetricsCollector]:
|
||||
"""
|
||||
Get or create the global metrics collector.
|
||||
|
||||
Returns:
|
||||
RedisMetricsCollector instance if observability is enabled, None otherwise
|
||||
"""
|
||||
global _async_metrics_collector
|
||||
|
||||
if _async_metrics_collector is not None:
|
||||
return _async_metrics_collector
|
||||
|
||||
try:
|
||||
manager = get_observability_instance().get_provider_manager()
|
||||
if manager is None or not manager.config.enabled_telemetry:
|
||||
return None
|
||||
|
||||
# Get meter from the global MeterProvider
|
||||
meter = manager.get_meter_provider().get_meter(
|
||||
RedisMetricsCollector.METER_NAME, RedisMetricsCollector.METER_VERSION
|
||||
)
|
||||
|
||||
_async_metrics_collector = RedisMetricsCollector(meter, manager.config)
|
||||
return _async_metrics_collector
|
||||
|
||||
except ImportError:
|
||||
# Observability module not available
|
||||
return None
|
||||
except Exception:
|
||||
# Any other error - don't break Redis operations
|
||||
return None
|
||||
|
||||
|
||||
async def _get_config() -> Optional["OTelConfig"]:
|
||||
"""
|
||||
Get the OTel configuration from the observability manager.
|
||||
|
||||
Returns:
|
||||
OTelConfig instance if observability is enabled, None otherwise
|
||||
"""
|
||||
try:
|
||||
manager = get_observability_instance().get_provider_manager()
|
||||
if manager is None:
|
||||
return None
|
||||
return manager.config
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def record_operation_duration(
|
||||
command_name: str,
|
||||
duration_seconds: float,
|
||||
server_address: Optional[str] = None,
|
||||
server_port: Optional[int] = None,
|
||||
db_namespace: Optional[str] = None,
|
||||
error: Optional[Exception] = None,
|
||||
is_blocking: Optional[bool] = None,
|
||||
retry_attempts: Optional[int] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Record a Redis command execution duration.
|
||||
|
||||
This is an async-safe API that Redis async client code can call directly.
|
||||
If observability is not enabled, this returns immediately with zero overhead.
|
||||
|
||||
Args:
|
||||
command_name: Redis command name (e.g., 'GET', 'SET')
|
||||
duration_seconds: Command execution time in seconds
|
||||
server_address: Redis server address
|
||||
server_port: Redis server port
|
||||
db_namespace: Redis database index
|
||||
error: Exception if command failed, None if successful
|
||||
is_blocking: Whether the operation is a blocking command
|
||||
retry_attempts: Number of retry attempts made
|
||||
|
||||
Example:
|
||||
>>> start = time.monotonic()
|
||||
>>> # ... execute command ...
|
||||
>>> await record_operation_duration('SET', time.monotonic() - start, 'localhost', 6379, '0')
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_operation_duration(
|
||||
command_name=command_name,
|
||||
duration_seconds=duration_seconds,
|
||||
server_address=server_address,
|
||||
server_port=server_port,
|
||||
db_namespace=db_namespace,
|
||||
error_type=error,
|
||||
network_peer_address=server_address,
|
||||
network_peer_port=server_port,
|
||||
is_blocking=is_blocking,
|
||||
retry_attempts=retry_attempts,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_connection_create_time(
|
||||
connection_pool: "ConnectionPool",
|
||||
duration_seconds: float,
|
||||
) -> None:
|
||||
"""
|
||||
Record connection creation time.
|
||||
|
||||
Args:
|
||||
connection_pool: Connection pool implementation
|
||||
duration_seconds: Time taken to create connection in seconds
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_connection_create_time(
|
||||
connection_pool=connection_pool,
|
||||
duration_seconds=duration_seconds,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_connection_count(
|
||||
pool_name: str,
|
||||
connection_state: ConnectionState,
|
||||
counter: int = 1,
|
||||
) -> None:
|
||||
"""
|
||||
Record a connection count change for a single state.
|
||||
|
||||
Args:
|
||||
pool_name: Connection pool identifier
|
||||
connection_state: State to update (IDLE or USED)
|
||||
counter: Number to add (positive) or subtract (negative)
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_connection_count(
|
||||
pool_name=pool_name,
|
||||
connection_state=connection_state,
|
||||
counter=counter,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@deprecated_function(
|
||||
reason="Connection count is now tracked via record_connection_count(). "
|
||||
"This functionality will be removed in the next major version",
|
||||
version="7.4.0",
|
||||
)
|
||||
async def init_connection_count() -> None:
|
||||
"""
|
||||
Initialize observable gauge for connection count metric.
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
def observable_callback(__):
|
||||
observables_registry = get_observables_registry_instance()
|
||||
callbacks = observables_registry.get(CONNECTION_COUNT_REGISTRY_KEY)
|
||||
observations = []
|
||||
|
||||
for callback in callbacks:
|
||||
observations.extend(callback())
|
||||
|
||||
return observations
|
||||
|
||||
try:
|
||||
collector.init_connection_count(
|
||||
callback=observable_callback,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@deprecated_function(
|
||||
reason="Connection count is now tracked via record_connection_count(). "
|
||||
"This functionality will be removed in the next major version",
|
||||
version="7.4.0",
|
||||
)
|
||||
async def register_pools_connection_count(
|
||||
connection_pools: List["ConnectionPool"],
|
||||
) -> None:
|
||||
"""
|
||||
Add connection pools to connection count observable registry.
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
# Lazy import
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
def connection_count_callback():
|
||||
observations = []
|
||||
for connection_pool in connection_pools:
|
||||
for count, attributes in connection_pool.get_connection_count():
|
||||
observations.append(Observation(count, attributes=attributes))
|
||||
return observations
|
||||
|
||||
observables_registry = get_observables_registry_instance()
|
||||
observables_registry.register(
|
||||
CONNECTION_COUNT_REGISTRY_KEY, connection_count_callback
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_connection_timeout(
|
||||
pool_name: str,
|
||||
) -> None:
|
||||
"""
|
||||
Record a connection timeout event.
|
||||
|
||||
Args:
|
||||
pool_name: Connection pool identifier
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_connection_timeout(
|
||||
pool_name=pool_name,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_connection_wait_time(
|
||||
pool_name: str,
|
||||
duration_seconds: float,
|
||||
) -> None:
|
||||
"""
|
||||
Record time taken to obtain a connection from the pool.
|
||||
|
||||
Args:
|
||||
pool_name: Connection pool identifier
|
||||
duration_seconds: Wait time in seconds
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_connection_wait_time(
|
||||
pool_name=pool_name,
|
||||
duration_seconds=duration_seconds,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_connection_closed(
|
||||
close_reason: Optional[CloseReason] = None,
|
||||
error_type: Optional[Exception] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Record a connection closed event.
|
||||
|
||||
Args:
|
||||
close_reason: Reason for closing (e.g. 'error', 'application_close')
|
||||
error_type: Error type if closed due to error
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_connection_closed(
|
||||
close_reason=close_reason,
|
||||
error_type=error_type,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_connection_relaxed_timeout(
|
||||
connection_name: str,
|
||||
maint_notification: str,
|
||||
relaxed: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Record a connection timeout relaxation event.
|
||||
|
||||
Args:
|
||||
connection_name: Connection identifier (pool name)
|
||||
maint_notification: Maintenance notification type
|
||||
relaxed: True to count up (relaxed), False to count down (unrelaxed)
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_connection_relaxed_timeout(
|
||||
connection_name=connection_name,
|
||||
maint_notification=maint_notification,
|
||||
relaxed=relaxed,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_connection_handoff(
|
||||
pool_name: str,
|
||||
) -> None:
|
||||
"""
|
||||
Record a connection handoff event (e.g., after MOVING notification).
|
||||
|
||||
Args:
|
||||
pool_name: Connection pool identifier
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_connection_handoff(
|
||||
pool_name=pool_name,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_error_count(
|
||||
server_address: str,
|
||||
server_port: int,
|
||||
network_peer_address: str,
|
||||
network_peer_port: int,
|
||||
error_type: Exception,
|
||||
retry_attempts: int,
|
||||
is_internal: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Record error count.
|
||||
|
||||
Args:
|
||||
server_address: Server address
|
||||
server_port: Server port
|
||||
network_peer_address: Network peer address
|
||||
network_peer_port: Network peer port
|
||||
error_type: Error type (Exception)
|
||||
retry_attempts: Retry attempts
|
||||
is_internal: Whether the error is internal (e.g., timeout, network error)
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_error_count(
|
||||
server_address=server_address,
|
||||
server_port=server_port,
|
||||
network_peer_address=network_peer_address,
|
||||
network_peer_port=network_peer_port,
|
||||
error_type=error_type,
|
||||
retry_attempts=retry_attempts,
|
||||
is_internal=is_internal,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_pubsub_message(
|
||||
direction: PubSubDirection,
|
||||
channel: Optional[str] = None,
|
||||
sharded: Optional[bool] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Record a PubSub message (published or received).
|
||||
|
||||
Args:
|
||||
direction: Message direction ('publish' or 'receive')
|
||||
channel: Pub/Sub channel name
|
||||
sharded: True if sharded Pub/Sub channel
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
# Check if channel names should be hidden
|
||||
effective_channel = channel
|
||||
if channel is not None:
|
||||
config = await _get_config()
|
||||
if config is not None and config.hide_pubsub_channel_names:
|
||||
effective_channel = None
|
||||
else:
|
||||
# Normalize bytes to str for OTel attributes
|
||||
effective_channel = str_if_bytes(channel)
|
||||
|
||||
try:
|
||||
collector.record_pubsub_message(
|
||||
direction=direction,
|
||||
channel=effective_channel,
|
||||
sharded=sharded,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_streaming_lag(
|
||||
lag_seconds: float,
|
||||
stream_name: Optional[str] = None,
|
||||
consumer_group: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Record the lag of a streaming message.
|
||||
|
||||
Args:
|
||||
lag_seconds: Lag in seconds
|
||||
stream_name: Stream name
|
||||
consumer_group: Consumer group name
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
# Check if stream names should be hidden
|
||||
effective_stream_name = stream_name
|
||||
if stream_name is not None:
|
||||
config = await _get_config()
|
||||
if config is not None and config.hide_stream_names:
|
||||
effective_stream_name = None
|
||||
|
||||
try:
|
||||
collector.record_streaming_lag(
|
||||
lag_seconds=lag_seconds,
|
||||
stream_name=effective_stream_name,
|
||||
consumer_group=consumer_group,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_streaming_lag_from_response(
|
||||
response,
|
||||
consumer_group: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Record streaming lag from XREAD/XREADGROUP response.
|
||||
|
||||
Parses the response and calculates lag for each message based on message ID timestamp.
|
||||
|
||||
Args:
|
||||
response: Response from XREAD/XREADGROUP command
|
||||
consumer_group: Consumer group name (for XREADGROUP)
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
if not response:
|
||||
return
|
||||
|
||||
try:
|
||||
now = datetime.now().timestamp()
|
||||
|
||||
# Check if stream names should be hidden
|
||||
config = await _get_config()
|
||||
hide_stream_names = config is not None and config.hide_stream_names
|
||||
|
||||
# RESP3 format: dict
|
||||
if isinstance(response, dict):
|
||||
for stream_name, stream_messages in response.items():
|
||||
effective_stream_name = (
|
||||
None if hide_stream_names else str_if_bytes(stream_name)
|
||||
)
|
||||
for messages in stream_messages:
|
||||
for message in messages:
|
||||
message_id, _ = message
|
||||
message_id = str_if_bytes(message_id)
|
||||
timestamp, _ = message_id.split("-")
|
||||
# Ensure lag is non-negative (clock skew can cause negative values)
|
||||
lag_seconds = max(0.0, now - int(timestamp) / 1000)
|
||||
|
||||
collector.record_streaming_lag(
|
||||
lag_seconds=lag_seconds,
|
||||
stream_name=effective_stream_name,
|
||||
consumer_group=consumer_group,
|
||||
)
|
||||
else:
|
||||
# RESP2 format: list
|
||||
for stream_entry in response:
|
||||
stream_name = str_if_bytes(stream_entry[0])
|
||||
effective_stream_name = None if hide_stream_names else stream_name
|
||||
|
||||
for message in stream_entry[1]:
|
||||
message_id, _ = message
|
||||
message_id = str_if_bytes(message_id)
|
||||
timestamp, _ = message_id.split("-")
|
||||
# Ensure lag is non-negative (clock skew can cause negative values)
|
||||
lag_seconds = max(0.0, now - int(timestamp) / 1000)
|
||||
|
||||
collector.record_streaming_lag(
|
||||
lag_seconds=lag_seconds,
|
||||
stream_name=effective_stream_name,
|
||||
consumer_group=consumer_group,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_maint_notification_count(
|
||||
server_address: str,
|
||||
server_port: int,
|
||||
network_peer_address: str,
|
||||
network_peer_port: int,
|
||||
maint_notification: str,
|
||||
) -> None:
|
||||
"""
|
||||
Record a maintenance notification count.
|
||||
|
||||
Args:
|
||||
server_address: Server address
|
||||
server_port: Server port
|
||||
network_peer_address: Network peer address
|
||||
network_peer_port: Network peer port
|
||||
maint_notification: Maintenance notification type (e.g., 'MOVING', 'MIGRATING')
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_maint_notification_count(
|
||||
server_address=server_address,
|
||||
server_port=server_port,
|
||||
network_peer_address=network_peer_address,
|
||||
network_peer_port=network_peer_port,
|
||||
maint_notification=maint_notification,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def record_geo_failover(
|
||||
fail_from: "AsyncDatabase",
|
||||
fail_to: "AsyncDatabase",
|
||||
reason: GeoFailoverReason,
|
||||
) -> None:
|
||||
"""
|
||||
Record a geo failover.
|
||||
|
||||
Args:
|
||||
fail_from: Database failed from
|
||||
fail_to: Database failed to
|
||||
reason: Reason for the failover
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
if collector is None:
|
||||
return
|
||||
|
||||
try:
|
||||
collector.record_geo_failover(
|
||||
fail_from=fail_from,
|
||||
fail_to=fail_to,
|
||||
reason=reason,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def reset_collector() -> None:
|
||||
"""
|
||||
Reset the global async collector (used for testing or re-initialization).
|
||||
"""
|
||||
global _async_metrics_collector
|
||||
_async_metrics_collector = None
|
||||
|
||||
|
||||
async def is_enabled() -> bool:
|
||||
"""
|
||||
Check if observability is enabled.
|
||||
|
||||
Returns:
|
||||
True if metrics are being collected, False otherwise
|
||||
"""
|
||||
collector = _get_or_create_collector()
|
||||
return collector is not None
|
||||
@@ -0,0 +1,84 @@
|
||||
from asyncio import sleep
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Awaitable,
|
||||
Callable,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
)
|
||||
|
||||
from redis.exceptions import ConnectionError, RedisError, TimeoutError
|
||||
from redis.retry import AbstractRetry
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from redis.backoff import AbstractBackoff
|
||||
|
||||
|
||||
class Retry(AbstractRetry[RedisError]):
|
||||
__hash__ = AbstractRetry.__hash__
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
backoff: "AbstractBackoff",
|
||||
retries: int,
|
||||
supported_errors: Tuple[Type[RedisError], ...] = (
|
||||
ConnectionError,
|
||||
TimeoutError,
|
||||
),
|
||||
):
|
||||
super().__init__(backoff, retries, supported_errors)
|
||||
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
if not isinstance(other, Retry):
|
||||
return NotImplemented
|
||||
|
||||
return (
|
||||
self._backoff == other._backoff
|
||||
and self._retries == other._retries
|
||||
and set(self._supported_errors) == set(other._supported_errors)
|
||||
)
|
||||
|
||||
async def call_with_retry(
|
||||
self,
|
||||
do: Callable[[], Awaitable[T]],
|
||||
fail: Union[
|
||||
Callable[[Exception], Any],
|
||||
Callable[[Exception, int], Any],
|
||||
],
|
||||
is_retryable: Optional[Callable[[Exception], bool]] = None,
|
||||
with_failure_count: bool = False,
|
||||
) -> T:
|
||||
"""
|
||||
Execute an operation that might fail and returns its result, or
|
||||
raise the exception that was thrown depending on the `Backoff` object.
|
||||
`do`: the operation to call. Expects no argument.
|
||||
`fail`: the failure handler, expects the last error that was thrown
|
||||
``is_retryable``: optional function to determine if an error is retryable
|
||||
``with_failure_count``: if True, the failure count is passed to the failure handler
|
||||
"""
|
||||
self._backoff.reset()
|
||||
failures = 0
|
||||
while True:
|
||||
try:
|
||||
return await do()
|
||||
except self._supported_errors as error:
|
||||
if is_retryable and not is_retryable(error):
|
||||
raise
|
||||
failures += 1
|
||||
|
||||
if with_failure_count:
|
||||
await fail(error, failures)
|
||||
else:
|
||||
await fail(error)
|
||||
|
||||
if self._retries >= 0 and failures > self._retries:
|
||||
raise error
|
||||
backoff = self._backoff.compute(failures)
|
||||
if backoff > 0:
|
||||
await sleep(backoff)
|
||||
@@ -0,0 +1,504 @@
|
||||
import asyncio
|
||||
import random
|
||||
import weakref
|
||||
from typing import AsyncIterator, Iterable, Mapping, Optional, Sequence, Tuple, Type
|
||||
|
||||
from redis.asyncio.client import Redis
|
||||
from redis.asyncio.connection import (
|
||||
Connection,
|
||||
ConnectionPool,
|
||||
EncodableT,
|
||||
SSLConnection,
|
||||
)
|
||||
from redis.commands import AsyncSentinelCommands
|
||||
from redis.exceptions import (
|
||||
ConnectionError,
|
||||
ReadOnlyError,
|
||||
ResponseError,
|
||||
TimeoutError,
|
||||
)
|
||||
|
||||
|
||||
class MasterNotFoundError(ConnectionError):
|
||||
pass
|
||||
|
||||
|
||||
class SlaveNotFoundError(ConnectionError):
|
||||
pass
|
||||
|
||||
|
||||
ReplicaNotFoundError = SlaveNotFoundError
|
||||
|
||||
|
||||
class SentinelManagedConnection(Connection):
|
||||
def __init__(self, **kwargs):
|
||||
self.connection_pool = kwargs.pop("connection_pool")
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def __repr__(self):
|
||||
s = f"<{self.__class__.__module__}.{self.__class__.__name__}"
|
||||
if self.host:
|
||||
host_info = f",host={self.host},port={self.port}"
|
||||
s += host_info
|
||||
return s + ")>"
|
||||
|
||||
async def connect_to(self, address):
|
||||
self.host, self.port = address
|
||||
await self.connect_check_health(
|
||||
check_health=self.connection_pool.check_connection,
|
||||
retry_socket_connect=False,
|
||||
)
|
||||
|
||||
async def _connect_retry(self):
|
||||
if self._reader:
|
||||
return # already connected
|
||||
if self.connection_pool.is_master:
|
||||
await self.connect_to(await self.connection_pool.get_master_address())
|
||||
else:
|
||||
async for slave in self.connection_pool.rotate_slaves():
|
||||
try:
|
||||
return await self.connect_to(slave)
|
||||
except ConnectionError:
|
||||
continue
|
||||
raise SlaveNotFoundError # Never be here
|
||||
|
||||
async def connect(self):
|
||||
return await self.retry.call_with_retry(
|
||||
self._connect_retry,
|
||||
lambda error: asyncio.sleep(0),
|
||||
)
|
||||
|
||||
async def read_response(
|
||||
self,
|
||||
disable_decoding: bool = False,
|
||||
timeout: Optional[float] = None,
|
||||
*,
|
||||
disconnect_on_error: Optional[float] = True,
|
||||
push_request: Optional[bool] = False,
|
||||
):
|
||||
try:
|
||||
return await super().read_response(
|
||||
disable_decoding=disable_decoding,
|
||||
timeout=timeout,
|
||||
disconnect_on_error=disconnect_on_error,
|
||||
push_request=push_request,
|
||||
)
|
||||
except ReadOnlyError:
|
||||
if self.connection_pool.is_master:
|
||||
# When talking to a master, a ReadOnlyError when likely
|
||||
# indicates that the previous master that we're still connected
|
||||
# to has been demoted to a slave and there's a new master.
|
||||
# calling disconnect will force the connection to re-query
|
||||
# sentinel during the next connect() attempt.
|
||||
await self.disconnect()
|
||||
raise ConnectionError("The previous master is now a slave")
|
||||
raise
|
||||
|
||||
|
||||
class SentinelManagedSSLConnection(SentinelManagedConnection, SSLConnection):
|
||||
pass
|
||||
|
||||
|
||||
class SentinelConnectionPool(ConnectionPool):
|
||||
"""
|
||||
Sentinel backed connection pool.
|
||||
|
||||
If ``check_connection`` flag is set to True, SentinelManagedConnection
|
||||
sends a PING command right after establishing the connection.
|
||||
"""
|
||||
|
||||
def __init__(self, service_name, sentinel_manager, **kwargs):
|
||||
kwargs["connection_class"] = kwargs.get(
|
||||
"connection_class",
|
||||
(
|
||||
SentinelManagedSSLConnection
|
||||
if kwargs.pop("ssl", False)
|
||||
else SentinelManagedConnection
|
||||
),
|
||||
)
|
||||
self.is_master = kwargs.pop("is_master", True)
|
||||
self.check_connection = kwargs.pop("check_connection", False)
|
||||
super().__init__(**kwargs)
|
||||
self.connection_kwargs["connection_pool"] = weakref.proxy(self)
|
||||
self.service_name = service_name
|
||||
self.sentinel_manager = sentinel_manager
|
||||
self.master_address = None
|
||||
self.slave_rr_counter = None
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"<{self.__class__.__module__}.{self.__class__.__name__}"
|
||||
f"(service={self.service_name}({self.is_master and 'master' or 'slave'}))>"
|
||||
)
|
||||
|
||||
def reset(self):
|
||||
super().reset()
|
||||
self.master_address = None
|
||||
self.slave_rr_counter = None
|
||||
|
||||
def owns_connection(self, connection: Connection):
|
||||
check = not self.is_master or (
|
||||
self.is_master and self.master_address == (connection.host, connection.port)
|
||||
)
|
||||
return check and super().owns_connection(connection)
|
||||
|
||||
async def get_master_address(self):
|
||||
master_address = await self.sentinel_manager.discover_master(self.service_name)
|
||||
if self.is_master:
|
||||
if self.master_address != master_address:
|
||||
self.master_address = master_address
|
||||
# disconnect any idle connections so that they reconnect
|
||||
# to the new master the next time that they are used.
|
||||
await self.disconnect(inuse_connections=False)
|
||||
return master_address
|
||||
|
||||
async def rotate_slaves(self) -> AsyncIterator:
|
||||
"""Round-robin slave balancer"""
|
||||
slaves = await self.sentinel_manager.discover_slaves(self.service_name)
|
||||
if slaves:
|
||||
if self.slave_rr_counter is None:
|
||||
self.slave_rr_counter = random.randint(0, len(slaves) - 1)
|
||||
for _ in range(len(slaves)):
|
||||
self.slave_rr_counter = (self.slave_rr_counter + 1) % len(slaves)
|
||||
slave = slaves[self.slave_rr_counter]
|
||||
yield slave
|
||||
# Fallback to the master connection
|
||||
try:
|
||||
yield await self.get_master_address()
|
||||
except MasterNotFoundError:
|
||||
pass
|
||||
raise SlaveNotFoundError(f"No slave found for {self.service_name!r}")
|
||||
|
||||
def rotate_replicas(self) -> AsyncIterator:
|
||||
"""Round-robin replica balancer.
|
||||
|
||||
This is an alias for :py:meth:`rotate_slaves`,
|
||||
using the preferred Redis 5.0+ terminology.
|
||||
"""
|
||||
return self.rotate_slaves()
|
||||
|
||||
|
||||
class Sentinel(AsyncSentinelCommands):
|
||||
"""
|
||||
Redis Sentinel cluster client
|
||||
|
||||
>>> from redis.sentinel import Sentinel
|
||||
>>> sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1)
|
||||
>>> master = sentinel.master_for('mymaster', socket_timeout=0.1)
|
||||
>>> await master.set('foo', 'bar')
|
||||
>>> slave = sentinel.slave_for('mymaster', socket_timeout=0.1)
|
||||
>>> await slave.get('foo')
|
||||
b'bar'
|
||||
|
||||
``sentinels`` is a list of sentinel nodes. Each node is represented by
|
||||
a pair (hostname, port).
|
||||
|
||||
``min_other_sentinels`` defined a minimum number of peers for a sentinel.
|
||||
When querying a sentinel, if it doesn't meet this threshold, responses
|
||||
from that sentinel won't be considered valid.
|
||||
|
||||
``sentinel_kwargs`` is a dictionary of connection arguments used when
|
||||
connecting to sentinel instances. Any argument that can be passed to
|
||||
a normal Redis connection can be specified here. If ``sentinel_kwargs`` is
|
||||
not specified, any socket_timeout and socket_keepalive options specified
|
||||
in ``connection_kwargs`` will be used.
|
||||
|
||||
``connection_kwargs`` are keyword arguments that will be used when
|
||||
establishing a connection to a Redis server.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sentinels,
|
||||
min_other_sentinels=0,
|
||||
sentinel_kwargs=None,
|
||||
force_master_ip=None,
|
||||
**connection_kwargs,
|
||||
):
|
||||
# if sentinel_kwargs isn't defined, use the socket_* options from
|
||||
# connection_kwargs
|
||||
if sentinel_kwargs is None:
|
||||
sentinel_kwargs = {
|
||||
k: v for k, v in connection_kwargs.items() if k.startswith("socket_")
|
||||
}
|
||||
self.sentinel_kwargs = sentinel_kwargs
|
||||
|
||||
self.sentinels = [
|
||||
Redis(host=hostname, port=port, **self.sentinel_kwargs)
|
||||
for hostname, port in sentinels
|
||||
]
|
||||
self.min_other_sentinels = min_other_sentinels
|
||||
self.connection_kwargs = connection_kwargs
|
||||
self._force_master_ip = force_master_ip
|
||||
|
||||
async def execute_command(self, *args, **kwargs):
|
||||
"""
|
||||
Execute Sentinel command in sentinel nodes.
|
||||
once - If set to True, then execute the resulting command on a single
|
||||
node at random, rather than across the entire sentinel cluster.
|
||||
"""
|
||||
once = bool(kwargs.pop("once", False))
|
||||
|
||||
# Check if command is supposed to return the original
|
||||
# responses instead of boolean value.
|
||||
return_responses = bool(kwargs.pop("return_responses", False))
|
||||
|
||||
if once:
|
||||
response = await random.choice(self.sentinels).execute_command(
|
||||
*args, **kwargs
|
||||
)
|
||||
if return_responses:
|
||||
return [response]
|
||||
else:
|
||||
return True if response else False
|
||||
|
||||
tasks = [
|
||||
asyncio.Task(sentinel.execute_command(*args, **kwargs))
|
||||
for sentinel in self.sentinels
|
||||
]
|
||||
responses = await asyncio.gather(*tasks)
|
||||
|
||||
if return_responses:
|
||||
return responses
|
||||
|
||||
return all(responses)
|
||||
|
||||
def __repr__(self):
|
||||
sentinel_addresses = []
|
||||
for sentinel in self.sentinels:
|
||||
sentinel_addresses.append(
|
||||
f"{sentinel.connection_pool.connection_kwargs['host']}:"
|
||||
f"{sentinel.connection_pool.connection_kwargs['port']}"
|
||||
)
|
||||
return (
|
||||
f"<{self.__class__}.{self.__class__.__name__}"
|
||||
f"(sentinels=[{','.join(sentinel_addresses)}])>"
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""
|
||||
Close all sentinel clients created by this Sentinel and their
|
||||
connection pools.
|
||||
|
||||
Each client is closed independently: if one raises, the remaining
|
||||
clients are still closed and the first error is re-raised afterwards.
|
||||
|
||||
Clients returned by ``master_for``/``slave_for`` are owned by the
|
||||
caller and are not closed here.
|
||||
"""
|
||||
resp = await asyncio.gather(
|
||||
*(sentinel.aclose() for sentinel in self.sentinels),
|
||||
return_exceptions=True,
|
||||
)
|
||||
exc = next((r for r in resp if isinstance(r, BaseException)), None)
|
||||
if exc:
|
||||
raise exc
|
||||
|
||||
async def __aenter__(self) -> "Sentinel":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback) -> None:
|
||||
await self.aclose()
|
||||
|
||||
def check_master_state(self, state: dict, service_name: str) -> bool:
|
||||
if not state["is_master"] or state["is_sdown"] or state["is_odown"]:
|
||||
return False
|
||||
# Check if our sentinel doesn't see other nodes
|
||||
if state["num-other-sentinels"] < self.min_other_sentinels:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def discover_master(self, service_name: str):
|
||||
"""
|
||||
Asks sentinel servers for the Redis master's address corresponding
|
||||
to the service labeled ``service_name``.
|
||||
|
||||
Returns a pair (address, port) or raises MasterNotFoundError if no
|
||||
master is found.
|
||||
"""
|
||||
collected_errors = list()
|
||||
for sentinel_no, sentinel in enumerate(self.sentinels):
|
||||
try:
|
||||
masters = await sentinel.sentinel_masters()
|
||||
except (ConnectionError, TimeoutError) as e:
|
||||
collected_errors.append(f"{sentinel} - {e!r}")
|
||||
continue
|
||||
state = masters.get(service_name)
|
||||
if state and self.check_master_state(state, service_name):
|
||||
# Put this sentinel at the top of the list
|
||||
self.sentinels[0], self.sentinels[sentinel_no] = (
|
||||
sentinel,
|
||||
self.sentinels[0],
|
||||
)
|
||||
|
||||
ip = (
|
||||
self._force_master_ip
|
||||
if self._force_master_ip is not None
|
||||
else state["ip"]
|
||||
)
|
||||
return ip, state["port"]
|
||||
|
||||
error_info = ""
|
||||
if len(collected_errors) > 0:
|
||||
error_info = f" : {', '.join(collected_errors)}"
|
||||
raise MasterNotFoundError(f"No master found for {service_name!r}{error_info}")
|
||||
|
||||
def filter_slaves(
|
||||
self, slaves: Iterable[Mapping]
|
||||
) -> Sequence[Tuple[EncodableT, EncodableT]]:
|
||||
"""Remove slaves that are in an ODOWN or SDOWN state"""
|
||||
slaves_alive = []
|
||||
for slave in slaves:
|
||||
if slave["is_odown"] or slave["is_sdown"]:
|
||||
continue
|
||||
slaves_alive.append((slave["ip"], slave["port"]))
|
||||
return slaves_alive
|
||||
|
||||
def filter_replicas(
|
||||
self, replicas: Iterable[Mapping]
|
||||
) -> Sequence[Tuple[EncodableT, EncodableT]]:
|
||||
"""Remove replicas that are in an ODOWN or SDOWN state.
|
||||
|
||||
This is an alias for :py:meth:`filter_slaves`,
|
||||
using the preferred Redis 5.0+ terminology.
|
||||
"""
|
||||
return self.filter_slaves(replicas)
|
||||
|
||||
async def discover_slaves(
|
||||
self, service_name: str
|
||||
) -> Sequence[Tuple[EncodableT, EncodableT]]:
|
||||
"""Returns a list of alive slaves for service ``service_name``"""
|
||||
for sentinel in self.sentinels:
|
||||
try:
|
||||
slaves = await sentinel.sentinel_slaves(service_name)
|
||||
except (ConnectionError, ResponseError, TimeoutError):
|
||||
continue
|
||||
slaves = self.filter_slaves(slaves)
|
||||
if slaves:
|
||||
return slaves
|
||||
return []
|
||||
|
||||
async def discover_replicas(
|
||||
self, service_name: str
|
||||
) -> Sequence[Tuple[EncodableT, EncodableT]]:
|
||||
"""Returns a list of alive replicas for service ``service_name``.
|
||||
|
||||
This is an alias for :py:meth:`discover_slaves`,
|
||||
using the preferred Redis 5.0+ terminology.
|
||||
"""
|
||||
return await self.discover_slaves(service_name)
|
||||
|
||||
def master_for(
|
||||
self,
|
||||
service_name: str,
|
||||
redis_class: Type[Redis] = Redis,
|
||||
connection_pool_class: Type[SentinelConnectionPool] = SentinelConnectionPool,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Returns a redis client instance for the ``service_name`` master.
|
||||
Sentinel client will detect failover and reconnect Redis clients
|
||||
automatically.
|
||||
|
||||
A :py:class:`~redis.sentinel.SentinelConnectionPool` class is
|
||||
used to retrieve the master's address before establishing a new
|
||||
connection.
|
||||
|
||||
NOTE: If the master's address has changed, any cached connections to
|
||||
the old master are closed.
|
||||
|
||||
By default clients will be a :py:class:`~redis.Redis` instance.
|
||||
Specify a different class to the ``redis_class`` argument if you
|
||||
desire something different.
|
||||
|
||||
The ``connection_pool_class`` specifies the connection pool to
|
||||
use. The :py:class:`~redis.sentinel.SentinelConnectionPool`
|
||||
will be used by default.
|
||||
|
||||
All other keyword arguments are merged with any connection_kwargs
|
||||
passed to this class and passed to the connection pool as keyword
|
||||
arguments to be used to initialize Redis connections.
|
||||
|
||||
HIMPORT note: the returned client exposes the HIMPORT command family
|
||||
(``himport_prepare``, ``himport_set``, ``himport_discard``,
|
||||
``himport_discard_all``). Fieldsets are declared at runtime with
|
||||
``himport_prepare`` on the client and live on that client's shared registry,
|
||||
so they survive Sentinel failover automatically: the pool re-points to the
|
||||
new master and the fieldset is re-prepared lazily on the next
|
||||
``himport_set``. Each call to current command returns a *new* client with its *own* empty
|
||||
registry, so call ``himport_prepare`` on the long-lived client you reuse
|
||||
rather than creating a fresh one per operation; otherwise ``himport_set``
|
||||
fails with ``no such fieldset``. (``himport_set`` is a write and is served by
|
||||
the master.)
|
||||
"""
|
||||
kwargs["is_master"] = True
|
||||
connection_kwargs = dict(self.connection_kwargs)
|
||||
connection_kwargs.update(kwargs)
|
||||
|
||||
connection_pool = connection_pool_class(service_name, self, **connection_kwargs)
|
||||
# The Redis object "owns" the pool
|
||||
return redis_class.from_pool(connection_pool)
|
||||
|
||||
def slave_for(
|
||||
self,
|
||||
service_name: str,
|
||||
redis_class: Type[Redis] = Redis,
|
||||
connection_pool_class: Type[SentinelConnectionPool] = SentinelConnectionPool,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Returns redis client instance for the ``service_name`` slave(s).
|
||||
|
||||
A SentinelConnectionPool class is used to retrieve the slave's
|
||||
address before establishing a new connection.
|
||||
|
||||
By default clients will be a :py:class:`~redis.Redis` instance.
|
||||
Specify a different class to the ``redis_class`` argument if you
|
||||
desire something different.
|
||||
|
||||
The ``connection_pool_class`` specifies the connection pool to use.
|
||||
The SentinelConnectionPool will be used by default.
|
||||
|
||||
All other keyword arguments are merged with any connection_kwargs
|
||||
passed to this class and passed to the connection pool as keyword
|
||||
arguments to be used to initialize Redis connections.
|
||||
|
||||
HIMPORT note: the returned client exposes the HIMPORT command family
|
||||
(``himport_prepare``, ``himport_set``, ``himport_discard``,
|
||||
``himport_discard_all``). Fieldsets are declared at runtime with
|
||||
``himport_prepare`` on the client and live on that client's shared registry,
|
||||
so they survive Sentinel failover automatically: the pool re-points to the
|
||||
new master and the fieldset is re-prepared lazily on the next
|
||||
``himport_set``. Each call to current command returns a *new* client with its *own* empty
|
||||
registry, so call ``himport_prepare`` on the long-lived client you reuse
|
||||
rather than creating a fresh one per operation; otherwise ``himport_set``
|
||||
fails with ``no such fieldset``. (``himport_set`` is a write and is served by
|
||||
the master.)
|
||||
"""
|
||||
kwargs["is_master"] = False
|
||||
connection_kwargs = dict(self.connection_kwargs)
|
||||
connection_kwargs.update(kwargs)
|
||||
|
||||
connection_pool = connection_pool_class(service_name, self, **connection_kwargs)
|
||||
# The Redis object "owns" the pool
|
||||
return redis_class.from_pool(connection_pool)
|
||||
|
||||
def replica_for(
|
||||
self,
|
||||
service_name: str,
|
||||
redis_class: Type[Redis] = Redis,
|
||||
connection_pool_class: Type[SentinelConnectionPool] = SentinelConnectionPool,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Returns redis client instance for the ``service_name`` replica(s).
|
||||
|
||||
This is an alias for :py:meth:`slave_for`,
|
||||
using the preferred Redis 5.0+ terminology.
|
||||
"""
|
||||
return self.slave_for(
|
||||
service_name,
|
||||
redis_class=redis_class,
|
||||
connection_pool_class=connection_pool_class,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -0,0 +1,28 @@
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from redis.asyncio.client import Pipeline, Redis
|
||||
|
||||
|
||||
def from_url(url: str, **kwargs: Any) -> "Redis":
|
||||
"""
|
||||
Returns an active Redis client generated from the given database URL.
|
||||
|
||||
Will attempt to extract the database id from the path url fragment, if
|
||||
none is provided.
|
||||
"""
|
||||
from redis.asyncio.client import Redis
|
||||
|
||||
return Redis.from_url(url, **kwargs)
|
||||
|
||||
|
||||
class pipeline: # noqa: N801
|
||||
def __init__(self, redis_obj: "Redis"):
|
||||
self.p: "Pipeline" = redis_obj.pipeline()
|
||||
|
||||
async def __aenter__(self) -> "Pipeline":
|
||||
return self.p
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
await self.p.execute()
|
||||
del self.p
|
||||
Reference in New Issue
Block a user