add
This commit is contained in:
macbook_max
@@ -0,0 +1,59 @@
|
||||
# dialects/sqlite/__init__.py
|
||||
# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of SQLAlchemy and is released under
|
||||
# the MIT License: https://www.opensource.org/licenses/mit-license.php
|
||||
# mypy: ignore-errors
|
||||
|
||||
|
||||
from . import aiosqlite # noqa
|
||||
from . import base # noqa
|
||||
from . import pysqlcipher # noqa
|
||||
from . import pysqlite # noqa
|
||||
from .base import BLOB
|
||||
from .base import BOOLEAN
|
||||
from .base import CHAR
|
||||
from .base import DATE
|
||||
from .base import DATETIME
|
||||
from .base import DECIMAL
|
||||
from .base import FLOAT
|
||||
from .base import INTEGER
|
||||
from .base import JSON
|
||||
from .base import JSONB
|
||||
from .base import NUMERIC
|
||||
from .base import REAL
|
||||
from .base import SMALLINT
|
||||
from .base import TEXT
|
||||
from .base import TIME
|
||||
from .base import TIMESTAMP
|
||||
from .base import VARCHAR
|
||||
from .dml import Insert
|
||||
from .dml import insert
|
||||
|
||||
# default dialect
|
||||
base.dialect = dialect = pysqlite.dialect
|
||||
|
||||
|
||||
__all__ = (
|
||||
"BLOB",
|
||||
"BOOLEAN",
|
||||
"CHAR",
|
||||
"DATE",
|
||||
"DATETIME",
|
||||
"DECIMAL",
|
||||
"FLOAT",
|
||||
"INTEGER",
|
||||
"JSON",
|
||||
"JSONB",
|
||||
"NUMERIC",
|
||||
"SMALLINT",
|
||||
"TEXT",
|
||||
"TIME",
|
||||
"TIMESTAMP",
|
||||
"VARCHAR",
|
||||
"REAL",
|
||||
"Insert",
|
||||
"insert",
|
||||
"dialect",
|
||||
)
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,375 @@
|
||||
# dialects/sqlite/aiosqlite.py
|
||||
# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of SQLAlchemy and is released under
|
||||
# the MIT License: https://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
|
||||
r"""
|
||||
|
||||
.. dialect:: sqlite+aiosqlite
|
||||
:name: aiosqlite
|
||||
:dbapi: aiosqlite
|
||||
:connectstring: sqlite+aiosqlite:///file_path
|
||||
:url: https://pypi.org/project/aiosqlite/
|
||||
|
||||
The aiosqlite dialect provides support for the SQLAlchemy asyncio interface
|
||||
running on top of pysqlite.
|
||||
|
||||
aiosqlite is a wrapper around pysqlite that uses a background thread for
|
||||
each connection. It does not actually use non-blocking IO, as SQLite
|
||||
databases are not socket-based. However it does provide a working asyncio
|
||||
interface that's useful for testing and prototyping purposes.
|
||||
|
||||
Using a special asyncio mediation layer, the aiosqlite dialect is usable
|
||||
as the backend for the :ref:`SQLAlchemy asyncio <asyncio_toplevel>`
|
||||
extension package.
|
||||
|
||||
This dialect should normally be used only with the
|
||||
:func:`_asyncio.create_async_engine` engine creation function::
|
||||
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
engine = create_async_engine("sqlite+aiosqlite:///filename")
|
||||
|
||||
The URL passes through all arguments to the ``pysqlite`` driver, so all
|
||||
connection arguments are the same as they are for that of :ref:`pysqlite`.
|
||||
|
||||
.. _aiosqlite_udfs:
|
||||
|
||||
User-Defined Functions
|
||||
----------------------
|
||||
|
||||
aiosqlite extends pysqlite to support async, so we can create our own user-defined functions (UDFs)
|
||||
in Python and use them directly in SQLite queries as described here: :ref:`pysqlite_udfs`.
|
||||
|
||||
.. _aiosqlite_serializable:
|
||||
|
||||
Serializable isolation / Savepoints / Transactional DDL (asyncio version)
|
||||
-------------------------------------------------------------------------
|
||||
|
||||
A newly revised version of this important section is now available
|
||||
at the top level of the SQLAlchemy SQLite documentation, in the section
|
||||
:ref:`sqlite_transactions`.
|
||||
|
||||
|
||||
.. _aiosqlite_pooling:
|
||||
|
||||
Pooling Behavior
|
||||
----------------
|
||||
|
||||
The SQLAlchemy ``aiosqlite`` DBAPI establishes the connection pool differently
|
||||
based on the kind of SQLite database that's requested:
|
||||
|
||||
* When a ``:memory:`` SQLite database is specified, the dialect by default
|
||||
will use :class:`.StaticPool`. This pool maintains a single
|
||||
connection, so that all access to the engine
|
||||
use the same ``:memory:`` database.
|
||||
* When a file-based database is specified, the dialect will use
|
||||
:class:`.AsyncAdaptedQueuePool` as the source of connections.
|
||||
|
||||
.. versionchanged:: 2.0.38
|
||||
|
||||
SQLite file database engines now use :class:`.AsyncAdaptedQueuePool` by default.
|
||||
Previously, :class:`.NullPool` were used. The :class:`.NullPool` class
|
||||
may be used by specifying it via the
|
||||
:paramref:`_sa.create_engine.poolclass` parameter.
|
||||
|
||||
As with the pysqlite dialect, this selection is made based on the database
|
||||
name alone, and the ``mode=memory`` query string argument is deprecated as
|
||||
a means of influencing it; see :ref:`pysqlite_threading_pooling` for
|
||||
background.
|
||||
|
||||
.. _aiosqlite_memory:
|
||||
|
||||
Using a Memory Database with Multiple Coroutines
|
||||
-------------------------------------------------
|
||||
|
||||
The default :class:`.StaticPool` used for ``:memory:`` databases forces all
|
||||
coroutines to share a single DBAPI connection. Because SQLite maintains only
|
||||
one transaction state per connection, concurrent coroutines can interfere
|
||||
with each other — a ``ROLLBACK`` in one coroutine will also discard
|
||||
uncommitted work from any other coroutine using the same engine.
|
||||
|
||||
For async workloads where multiple :class:`.AsyncSession` or
|
||||
:class:`.AsyncConnection` objects may be active simultaneously, use SQLite's
|
||||
shared-cache URI mode instead. This gives each checkout its own DBAPI
|
||||
connection with independent transaction state while still sharing one
|
||||
in-memory database::
|
||||
|
||||
engine = create_async_engine(
|
||||
"sqlite+aiosqlite:///file::memory:?cache=shared&uri=true"
|
||||
)
|
||||
|
||||
Because this URL form is treated as a file-based database by the dialect,
|
||||
:class:`.AsyncAdaptedQueuePool` is used automatically and no additional
|
||||
configuration is needed.
|
||||
|
||||
Note that a shared-cache database is discarded once its last connection is
|
||||
closed, so that operations such as :meth:`_asyncio.AsyncEngine.dispose` or
|
||||
the use of :paramref:`_sa.create_engine.pool_recycle` will destroy its
|
||||
contents; see :ref:`pysqlite_shared_cache_lifespan` for background and for
|
||||
how to hold such a database open.
|
||||
|
||||
See the pysqlite documentation at
|
||||
:ref:`pysqlite_uri_shared_cache` for full details on shared-cache memory
|
||||
databases, including how to use named databases to maintain multiple
|
||||
independent in-memory databases within the same process.
|
||||
|
||||
""" # noqa
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from functools import partial
|
||||
from threading import Thread
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
from typing import cast
|
||||
from typing import NoReturn
|
||||
from typing import Optional
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Union
|
||||
|
||||
from .base import SQLiteExecutionContext
|
||||
from .pysqlite import SQLiteDialect_pysqlite
|
||||
from ... import pool
|
||||
from ... import util
|
||||
from ...connectors.asyncio import AsyncAdapt_dbapi_connection
|
||||
from ...connectors.asyncio import AsyncAdapt_dbapi_cursor
|
||||
from ...connectors.asyncio import AsyncAdapt_dbapi_module
|
||||
from ...connectors.asyncio import AsyncAdapt_dbapi_ss_cursor
|
||||
from ...connectors.asyncio import AsyncAdapt_terminate
|
||||
from ...util.concurrency import await_
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...connectors.asyncio import AsyncIODBAPIConnection
|
||||
from ...engine.interfaces import DBAPIConnection
|
||||
from ...engine.interfaces import DBAPICursor
|
||||
from ...engine.interfaces import DBAPIModule
|
||||
from ...engine.url import URL
|
||||
from ...pool.base import PoolProxiedConnection
|
||||
|
||||
|
||||
class AsyncAdapt_aiosqlite_cursor(AsyncAdapt_dbapi_cursor):
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class AsyncAdapt_aiosqlite_ss_cursor(AsyncAdapt_dbapi_ss_cursor):
|
||||
__slots__ = ()
|
||||
|
||||
|
||||
class AsyncAdapt_aiosqlite_connection(
|
||||
AsyncAdapt_terminate, AsyncAdapt_dbapi_connection
|
||||
):
|
||||
__slots__ = ()
|
||||
|
||||
_cursor_cls = AsyncAdapt_aiosqlite_cursor
|
||||
_ss_cursor_cls = AsyncAdapt_aiosqlite_ss_cursor
|
||||
|
||||
@property
|
||||
def isolation_level(self) -> Optional[str]:
|
||||
return cast(str, self._connection.isolation_level)
|
||||
|
||||
@isolation_level.setter
|
||||
def isolation_level(self, value: Optional[str]) -> None:
|
||||
# aiosqlite's isolation_level setter works outside the Thread
|
||||
# that it's supposed to, necessitating setting check_same_thread=False.
|
||||
# for improved stability, we instead invent our own awaitable version
|
||||
# using aiosqlite's async queue directly.
|
||||
|
||||
def set_iso(
|
||||
connection: AsyncAdapt_aiosqlite_connection, value: Optional[str]
|
||||
) -> None:
|
||||
connection.isolation_level = value
|
||||
|
||||
function = partial(set_iso, self._connection._conn, value)
|
||||
future = asyncio.get_event_loop().create_future()
|
||||
|
||||
self._connection._tx.put_nowait((future, function))
|
||||
|
||||
try:
|
||||
await_(future)
|
||||
except Exception as error:
|
||||
self._handle_exception(error)
|
||||
|
||||
def create_function(self, *args: Any, **kw: Any) -> None:
|
||||
try:
|
||||
await_(self._connection.create_function(*args, **kw))
|
||||
except Exception as error:
|
||||
self._handle_exception(error)
|
||||
|
||||
def rollback(self) -> None:
|
||||
if self._connection._connection:
|
||||
super().rollback()
|
||||
|
||||
def commit(self) -> None:
|
||||
if self._connection._connection:
|
||||
super().commit()
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
await_(self._connection.close())
|
||||
except ValueError:
|
||||
# this is undocumented for aiosqlite, that ValueError
|
||||
# was raised if .close() was called more than once, which is
|
||||
# both not customary for DBAPI and is also not a DBAPI.Error
|
||||
# exception. This is now fixed in aiosqlite via my PR
|
||||
# https://github.com/omnilib/aiosqlite/pull/238, so we can be
|
||||
# assured this will not become some other kind of exception,
|
||||
# since it doesn't raise anymore.
|
||||
|
||||
pass
|
||||
except Exception as error:
|
||||
self._handle_exception(error)
|
||||
|
||||
@classmethod
|
||||
def _handle_exception_no_connection(
|
||||
cls, dbapi: Any, error: Exception
|
||||
) -> NoReturn:
|
||||
if isinstance(error, ValueError) and error.args[0].lower() in (
|
||||
"no active connection",
|
||||
"connection closed",
|
||||
):
|
||||
raise dbapi.sqlite.OperationalError(error.args[0]) from error
|
||||
else:
|
||||
super()._handle_exception_no_connection(dbapi, error)
|
||||
|
||||
async def _terminate_graceful_close(self) -> None:
|
||||
"""Try to close connection gracefully"""
|
||||
await self._connection.close()
|
||||
|
||||
def _terminate_force_close(self) -> None:
|
||||
"""Terminate the connection"""
|
||||
|
||||
# this was added in aiosqlite 0.22.1. if stop() is not present,
|
||||
# the dialect should indicate has_terminate=False
|
||||
try:
|
||||
meth = self._connection.stop
|
||||
except AttributeError as ae:
|
||||
raise NotImplementedError(
|
||||
"terminate_force_close() not implemented by this DBAPI shim"
|
||||
) from ae
|
||||
else:
|
||||
meth()
|
||||
|
||||
|
||||
class AsyncAdapt_aiosqlite_dbapi(AsyncAdapt_dbapi_module):
|
||||
def __init__(self, aiosqlite: ModuleType, sqlite: ModuleType):
|
||||
super().__init__(aiosqlite, dbapi_module=sqlite)
|
||||
self.aiosqlite = aiosqlite
|
||||
self.sqlite = sqlite
|
||||
self.paramstyle = "qmark"
|
||||
self.has_stop = hasattr(aiosqlite.Connection, "stop")
|
||||
self._init_dbapi_attributes()
|
||||
|
||||
def _init_dbapi_attributes(self) -> None:
|
||||
for name in (
|
||||
"DatabaseError",
|
||||
"Error",
|
||||
"IntegrityError",
|
||||
"NotSupportedError",
|
||||
"OperationalError",
|
||||
"ProgrammingError",
|
||||
"sqlite_version",
|
||||
"sqlite_version_info",
|
||||
):
|
||||
setattr(self, name, getattr(self.aiosqlite, name))
|
||||
|
||||
for name in ("PARSE_COLNAMES", "PARSE_DECLTYPES"):
|
||||
setattr(self, name, getattr(self.sqlite, name))
|
||||
|
||||
for name in ("Binary",):
|
||||
setattr(self, name, getattr(self.sqlite, name))
|
||||
|
||||
def connect(self, *arg: Any, **kw: Any) -> AsyncAdapt_aiosqlite_connection:
|
||||
creator_fn = kw.pop("async_creator_fn", None)
|
||||
if creator_fn:
|
||||
connection = creator_fn(*arg, **kw)
|
||||
else:
|
||||
connection = self.aiosqlite.connect(*arg, **kw)
|
||||
|
||||
# aiosqlite uses a Thread. you'll thank us later
|
||||
if isinstance(connection, Thread):
|
||||
# Connection itself was a thread in version prior to 0.22
|
||||
connection.daemon = True
|
||||
else:
|
||||
# in 0.22+ instead it contains a thread.
|
||||
connection._thread.daemon = True
|
||||
|
||||
return AsyncAdapt_aiosqlite_connection(self, await_(connection))
|
||||
|
||||
|
||||
class SQLiteExecutionContext_aiosqlite(SQLiteExecutionContext):
|
||||
def create_server_side_cursor(self) -> DBAPICursor:
|
||||
return self._dbapi_connection.cursor(server_side=True)
|
||||
|
||||
|
||||
class SQLiteDialect_aiosqlite(SQLiteDialect_pysqlite):
|
||||
driver = "aiosqlite"
|
||||
supports_statement_cache = True
|
||||
|
||||
is_async = True
|
||||
has_terminate = True
|
||||
|
||||
supports_server_side_cursors = True
|
||||
|
||||
execution_ctx_cls = SQLiteExecutionContext_aiosqlite
|
||||
|
||||
def __init__(self, **kwargs: Any):
|
||||
super().__init__(**kwargs)
|
||||
if self.dbapi and not self.dbapi.has_stop:
|
||||
self.has_terminate = False
|
||||
|
||||
@classmethod
|
||||
def import_dbapi(cls) -> AsyncAdapt_aiosqlite_dbapi:
|
||||
return AsyncAdapt_aiosqlite_dbapi(
|
||||
__import__("aiosqlite"), __import__("sqlite3")
|
||||
)
|
||||
|
||||
def retrieve_dbapi_version(self, dbapi: DBAPIModule) -> util.VersionInfo:
|
||||
# the version of aiosqlite, rather than the Python version
|
||||
# reported by the pysqlite dialect
|
||||
aiosqlite = getattr(dbapi, "aiosqlite", None)
|
||||
return util.parse_version_string(
|
||||
getattr(aiosqlite, "__version__", None)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_pool_class(cls, url: URL) -> type[pool.Pool]:
|
||||
if cls._is_url_file_db(url):
|
||||
return pool.AsyncAdaptedQueuePool
|
||||
else:
|
||||
cls._warn_memory_mode_pool_selection(
|
||||
url, pool.StaticPool, pool.AsyncAdaptedQueuePool
|
||||
)
|
||||
return pool.StaticPool
|
||||
|
||||
def is_disconnect(
|
||||
self,
|
||||
e: DBAPIModule.Error,
|
||||
connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]],
|
||||
cursor: Optional[DBAPICursor],
|
||||
) -> bool:
|
||||
self.dbapi = cast("DBAPIModule", self.dbapi)
|
||||
if isinstance(e, self.dbapi.OperationalError):
|
||||
err_lower = str(e).lower()
|
||||
if (
|
||||
"no active connection" in err_lower
|
||||
or "connection closed" in err_lower
|
||||
):
|
||||
return True
|
||||
|
||||
return super().is_disconnect(e, connection, cursor)
|
||||
|
||||
def get_driver_connection(
|
||||
self, connection: DBAPIConnection
|
||||
) -> AsyncIODBAPIConnection:
|
||||
return connection._connection # type: ignore[no-any-return]
|
||||
|
||||
def do_terminate(self, dbapi_connection: DBAPIConnection) -> None:
|
||||
dbapi_connection.terminate()
|
||||
|
||||
|
||||
dialect = SQLiteDialect_aiosqlite
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,314 @@
|
||||
# dialects/sqlite/dml.py
|
||||
# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of SQLAlchemy and is released under
|
||||
# the MIT License: https://www.opensource.org/licenses/mit-license.php
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Sequence
|
||||
from typing import Union
|
||||
|
||||
from .._typing import _OnConflictIndexElementsT
|
||||
from .._typing import _OnConflictIndexWhereT
|
||||
from .._typing import _OnConflictSetT
|
||||
from .._typing import _OnConflictWhereT
|
||||
from ... import exc
|
||||
from ... import util
|
||||
from ...sql import coercions
|
||||
from ...sql import roles
|
||||
from ...sql import schema
|
||||
from ...sql._typing import _DMLTableArgument
|
||||
from ...sql.base import ColumnCollection
|
||||
from ...sql.base import ReadOnlyColumnCollection
|
||||
from ...sql.base import SyntaxExtension
|
||||
from ...sql.dml import _DMLColumnElement
|
||||
from ...sql.dml import Insert as StandardInsert
|
||||
from ...sql.elements import ClauseElement
|
||||
from ...sql.elements import ColumnElement
|
||||
from ...sql.elements import KeyedColumnElement
|
||||
from ...sql.elements import TextClause
|
||||
from ...sql.expression import alias
|
||||
from ...sql.sqltypes import NULLTYPE
|
||||
from ...sql.visitors import InternalTraversal
|
||||
from ...util.typing import Self
|
||||
|
||||
__all__ = ("Insert", "insert")
|
||||
|
||||
|
||||
def insert(table: _DMLTableArgument) -> Insert:
|
||||
"""Construct a sqlite-specific variant :class:`_sqlite.Insert`
|
||||
construct.
|
||||
|
||||
.. container:: inherited_member
|
||||
|
||||
The :func:`sqlalchemy.dialects.sqlite.insert` function creates
|
||||
a :class:`sqlalchemy.dialects.sqlite.Insert`. This class is based
|
||||
on the dialect-agnostic :class:`_sql.Insert` construct which may
|
||||
be constructed using the :func:`_sql.insert` function in
|
||||
SQLAlchemy Core.
|
||||
|
||||
The :class:`_sqlite.Insert` construct includes additional methods
|
||||
:meth:`_sqlite.Insert.on_conflict_do_update`,
|
||||
:meth:`_sqlite.Insert.on_conflict_do_nothing`.
|
||||
|
||||
"""
|
||||
return Insert(table)
|
||||
|
||||
|
||||
class Insert(StandardInsert):
|
||||
"""SQLite-specific implementation of INSERT.
|
||||
|
||||
Adds methods for SQLite-specific syntaxes such as ON CONFLICT.
|
||||
|
||||
The :class:`_sqlite.Insert` object is created using the
|
||||
:func:`sqlalchemy.dialects.sqlite.insert` function.
|
||||
|
||||
.. versionadded:: 1.4
|
||||
|
||||
.. seealso::
|
||||
|
||||
:ref:`sqlite_on_conflict_insert`
|
||||
|
||||
"""
|
||||
|
||||
stringify_dialect = "sqlite"
|
||||
inherit_cache = True
|
||||
|
||||
@util.memoized_property
|
||||
def excluded(
|
||||
self,
|
||||
) -> ReadOnlyColumnCollection[str, KeyedColumnElement[Any]]:
|
||||
"""Provide the ``excluded`` namespace for an ON CONFLICT statement
|
||||
|
||||
SQLite's ON CONFLICT clause allows reference to the row that would
|
||||
be inserted, known as ``excluded``. This attribute provides
|
||||
all columns in this row to be referenceable.
|
||||
|
||||
.. tip:: The :attr:`_sqlite.Insert.excluded` attribute is an instance
|
||||
of :class:`_expression.ColumnCollection`, which provides an
|
||||
interface the same as that of the :attr:`_schema.Table.c`
|
||||
collection described at :ref:`metadata_tables_and_columns`.
|
||||
With this collection, ordinary names are accessible like attributes
|
||||
(e.g. ``stmt.excluded.some_column``), but special names and
|
||||
dictionary method names should be accessed using indexed access,
|
||||
such as ``stmt.excluded["column name"]`` or
|
||||
``stmt.excluded["values"]``. See the docstring for
|
||||
:class:`_expression.ColumnCollection` for further examples.
|
||||
|
||||
"""
|
||||
return alias(self.table, name="excluded").columns
|
||||
|
||||
def on_conflict_do_update(
|
||||
self,
|
||||
index_elements: _OnConflictIndexElementsT = None,
|
||||
index_where: _OnConflictIndexWhereT = None,
|
||||
set_: _OnConflictSetT = None,
|
||||
where: _OnConflictWhereT = None,
|
||||
) -> Self:
|
||||
r"""
|
||||
Specifies a DO UPDATE SET action for ON CONFLICT clause.
|
||||
|
||||
This method may be invoked more than once against the same
|
||||
:class:`_sqlite.Insert` construct, where each ``ON CONFLICT`` clause
|
||||
renders in the order in which it was established.
|
||||
|
||||
.. versionadded:: 2.1 Multiple ``ON CONFLICT`` clauses may be
|
||||
established on a single :class:`_sqlite.Insert` construct.
|
||||
|
||||
.. seealso::
|
||||
|
||||
:ref:`sqlite_on_conflict_multiple`
|
||||
|
||||
:param index_elements:
|
||||
A sequence consisting of string column names, :class:`_schema.Column`
|
||||
objects, or other column expression objects that will be used
|
||||
to infer a target index or unique constraint.
|
||||
|
||||
:param index_where:
|
||||
Additional WHERE criterion that can be used to infer a
|
||||
conditional target index.
|
||||
|
||||
:param set\_:
|
||||
A dictionary or other mapping object
|
||||
where the keys are either names of columns in the target table,
|
||||
or :class:`_schema.Column` objects or other ORM-mapped columns
|
||||
matching that of the target table, and expressions or literals
|
||||
as values, specifying the ``SET`` actions to take.
|
||||
|
||||
.. versionadded:: 1.4 The
|
||||
:paramref:`_sqlite.Insert.on_conflict_do_update.set_`
|
||||
parameter supports :class:`_schema.Column` objects from the target
|
||||
:class:`_schema.Table` as keys.
|
||||
|
||||
.. warning:: This dictionary does **not** take into account
|
||||
Python-specified default UPDATE values or generation functions,
|
||||
e.g. those specified using :paramref:`_schema.Column.onupdate`.
|
||||
These values will not be exercised for an ON CONFLICT style of
|
||||
UPDATE, unless they are manually specified in the
|
||||
:paramref:`.Insert.on_conflict_do_update.set_` dictionary.
|
||||
|
||||
:param where:
|
||||
Optional argument. An expression object representing a ``WHERE``
|
||||
clause that restricts the rows affected by ``DO UPDATE SET``. Rows not
|
||||
meeting the ``WHERE`` condition will not be updated (effectively a
|
||||
``DO NOTHING`` for those rows).
|
||||
|
||||
"""
|
||||
|
||||
return self.ext(
|
||||
OnConflictDoUpdate(index_elements, index_where, set_, where)
|
||||
)
|
||||
|
||||
def on_conflict_do_nothing(
|
||||
self,
|
||||
index_elements: _OnConflictIndexElementsT = None,
|
||||
index_where: _OnConflictIndexWhereT = None,
|
||||
) -> Self:
|
||||
"""
|
||||
Specifies a DO NOTHING action for ON CONFLICT clause.
|
||||
|
||||
This method may be invoked more than once against the same
|
||||
:class:`_sqlite.Insert` construct, and may be combined with
|
||||
:meth:`_sqlite.Insert.on_conflict_do_update`, where each
|
||||
``ON CONFLICT`` clause renders in the order in which it was
|
||||
established. As SQLite allows only the last ``ON CONFLICT`` clause
|
||||
of a statement to omit its conflict target, a call that omits
|
||||
:paramref:`_sqlite.Insert.on_conflict_do_nothing.index_elements`
|
||||
must be the last clause established, else
|
||||
:class:`.InvalidRequestError` is raised.
|
||||
|
||||
.. versionadded:: 2.1 Multiple ``ON CONFLICT`` clauses may be
|
||||
established on a single :class:`_sqlite.Insert` construct.
|
||||
|
||||
.. seealso::
|
||||
|
||||
:ref:`sqlite_on_conflict_multiple`
|
||||
|
||||
:param index_elements:
|
||||
A sequence consisting of string column names, :class:`_schema.Column`
|
||||
objects, or other column expression objects that will be used
|
||||
to infer a target index or unique constraint.
|
||||
|
||||
:param index_where:
|
||||
Additional WHERE criterion that can be used to infer a
|
||||
conditional target index.
|
||||
|
||||
"""
|
||||
|
||||
return self.ext(OnConflictDoNothing(index_elements, index_where))
|
||||
|
||||
|
||||
class OnConflictClause(SyntaxExtension, ClauseElement):
|
||||
stringify_dialect = "sqlite"
|
||||
|
||||
inferred_target_elements: Optional[List[Union[str, schema.Column[Any]]]]
|
||||
inferred_target_whereclause: Optional[
|
||||
Union[ColumnElement[Any], TextClause]
|
||||
]
|
||||
|
||||
_traverse_internals = [
|
||||
("inferred_target_elements", InternalTraversal.dp_multi_list),
|
||||
("inferred_target_whereclause", InternalTraversal.dp_clauseelement),
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
index_elements: _OnConflictIndexElementsT = None,
|
||||
index_where: _OnConflictIndexWhereT = None,
|
||||
):
|
||||
if index_elements is not None:
|
||||
self.inferred_target_elements = [
|
||||
coercions.expect(roles.DDLConstraintColumnRole, column)
|
||||
for column in index_elements
|
||||
]
|
||||
self.inferred_target_whereclause = (
|
||||
coercions.expect(
|
||||
roles.WhereHavingRole,
|
||||
index_where,
|
||||
)
|
||||
if index_where is not None
|
||||
else None
|
||||
)
|
||||
else:
|
||||
self.inferred_target_elements = (
|
||||
self.inferred_target_whereclause
|
||||
) = None
|
||||
|
||||
def _append_to_existing(
|
||||
self, existing: Sequence[ClauseElement]
|
||||
) -> Sequence[ClauseElement]:
|
||||
if existing:
|
||||
last = existing[-1]
|
||||
if (
|
||||
isinstance(last, OnConflictClause)
|
||||
and last.inferred_target_elements is None
|
||||
):
|
||||
raise exc.InvalidRequestError(
|
||||
"This Insert construct already has an ON CONFLICT "
|
||||
"clause that omits a conflict target; such a clause "
|
||||
"must be the last ON CONFLICT clause in the statement"
|
||||
)
|
||||
return [*existing, self]
|
||||
|
||||
def apply_to_insert(self, insert_stmt: StandardInsert) -> None:
|
||||
insert_stmt.apply_syntax_extension_point(
|
||||
self._append_to_existing, "post_values"
|
||||
)
|
||||
|
||||
|
||||
class OnConflictDoNothing(OnConflictClause):
|
||||
__visit_name__ = "on_conflict_do_nothing"
|
||||
|
||||
inherit_cache = True
|
||||
|
||||
|
||||
class OnConflictDoUpdate(OnConflictClause):
|
||||
__visit_name__ = "on_conflict_do_update"
|
||||
|
||||
update_values_to_set: Dict[_DMLColumnElement, ColumnElement[Any]]
|
||||
update_whereclause: Optional[ColumnElement[Any]]
|
||||
|
||||
_traverse_internals = OnConflictClause._traverse_internals + [
|
||||
("update_values_to_set", InternalTraversal.dp_dml_values),
|
||||
("update_whereclause", InternalTraversal.dp_clauseelement),
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
index_elements: _OnConflictIndexElementsT = None,
|
||||
index_where: _OnConflictIndexWhereT = None,
|
||||
set_: _OnConflictSetT = None,
|
||||
where: _OnConflictWhereT = None,
|
||||
):
|
||||
super().__init__(
|
||||
index_elements=index_elements,
|
||||
index_where=index_where,
|
||||
)
|
||||
|
||||
if isinstance(set_, dict):
|
||||
if not set_:
|
||||
raise ValueError("set parameter dictionary must not be empty")
|
||||
elif isinstance(set_, ColumnCollection):
|
||||
set_ = dict(set_)
|
||||
else:
|
||||
raise ValueError(
|
||||
"set parameter must be a non-empty dictionary "
|
||||
"or a ColumnCollection such as the `.c.` collection "
|
||||
"of a Table object"
|
||||
)
|
||||
self.update_values_to_set = {
|
||||
coercions.expect(roles.DMLColumnRole, k): coercions.expect(
|
||||
roles.ExpressionElementRole, v, type_=NULLTYPE, is_crud=True
|
||||
)
|
||||
for k, v in set_.items()
|
||||
}
|
||||
self.update_whereclause = (
|
||||
coercions.expect(roles.WhereHavingRole, where)
|
||||
if where is not None
|
||||
else None
|
||||
)
|
||||
@@ -0,0 +1,134 @@
|
||||
# dialects/sqlite/json.py
|
||||
# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of SQLAlchemy and is released under
|
||||
# the MIT License: https://www.opensource.org/licenses/mit-license.php
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ... import types as sqltypes
|
||||
from ...sql import func
|
||||
from ...sql.sqltypes import _T_JSON
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...engine.interfaces import Dialect
|
||||
from ...sql.elements import ColumnElement
|
||||
from ...sql.type_api import _BindProcessorType
|
||||
from ...sql.type_api import _LiteralProcessorType
|
||||
|
||||
|
||||
class JSON(sqltypes.JSON[_T_JSON]):
|
||||
"""SQLite JSON type.
|
||||
|
||||
SQLite supports JSON as of version 3.9 through its JSON1_ extension. Note
|
||||
that JSON1_ is a
|
||||
`loadable extension <https://www.sqlite.org/loadext.html>`_ and as such
|
||||
may not be available, or may require run-time loading.
|
||||
|
||||
:class:`_sqlite.JSON` is used automatically whenever the base
|
||||
:class:`_types.JSON` datatype is used against a SQLite backend.
|
||||
|
||||
.. seealso::
|
||||
|
||||
:class:`_types.JSON` - main documentation for the generic
|
||||
cross-platform JSON datatype.
|
||||
|
||||
The :class:`_sqlite.JSON` type supports persistence of JSON values
|
||||
as well as the core index operations provided by :class:`_types.JSON`
|
||||
datatype, by adapting the operations to render the ``JSON_EXTRACT``
|
||||
function wrapped in the ``JSON_QUOTE`` function at the database level.
|
||||
Extracted values are quoted in order to ensure that the results are
|
||||
always JSON string values.
|
||||
|
||||
|
||||
.. _JSON1: https://www.sqlite.org/json1.html
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class JSONB(JSON[_T_JSON]):
|
||||
"""SQLite JSONB type.
|
||||
|
||||
Stores JSON data in SQLite's binary JSONB format, available as of
|
||||
SQLite version 3.45.0. The binary format is more compact and faster
|
||||
to parse than the text-based :class:`_sqlite.JSON` type.
|
||||
|
||||
Values are transparently stored using the ``jsonb()`` SQL function and
|
||||
retrieved as text JSON via the ``json()`` SQL function, so the Python
|
||||
side behaves identically to :class:`_sqlite.JSON`.
|
||||
|
||||
.. versionadded:: 2.1b3
|
||||
|
||||
.. seealso::
|
||||
|
||||
:class:`_sqlite.JSON`
|
||||
|
||||
https://sqlite.org/jsonb.html
|
||||
|
||||
"""
|
||||
|
||||
__visit_name__ = "JSONB"
|
||||
|
||||
def bind_expression(
|
||||
self, bindvalue: ColumnElement[Any]
|
||||
) -> ColumnElement[Any]:
|
||||
return func.jsonb(bindvalue, type_=self)
|
||||
|
||||
def column_expression(self, col: ColumnElement[Any]) -> ColumnElement[Any]:
|
||||
return func.json(col, type_=self)
|
||||
|
||||
|
||||
# Note: these objects currently match exactly those of MySQL, however since
|
||||
# these are not generalizable to all JSON implementations, remain separately
|
||||
# implemented for each dialect.
|
||||
class _FormatTypeMixin:
|
||||
def _format_value(self, value: Any) -> str:
|
||||
raise NotImplementedError()
|
||||
|
||||
def bind_processor(self, dialect: Dialect) -> _BindProcessorType[Any]:
|
||||
super_proc = self.string_bind_processor(dialect) # type: ignore[attr-defined] # noqa: E501
|
||||
|
||||
def process(value: Any) -> Any:
|
||||
value = self._format_value(value)
|
||||
if super_proc:
|
||||
value = super_proc(value)
|
||||
return value
|
||||
|
||||
return process
|
||||
|
||||
def literal_processor(
|
||||
self, dialect: Dialect
|
||||
) -> _LiteralProcessorType[Any]:
|
||||
super_proc = self.string_literal_processor(dialect) # type: ignore[attr-defined] # noqa: E501
|
||||
|
||||
def process(value: Any) -> str:
|
||||
value = self._format_value(value)
|
||||
if super_proc:
|
||||
value = super_proc(value)
|
||||
return value # type: ignore[no-any-return]
|
||||
|
||||
return process
|
||||
|
||||
|
||||
class JSONIndexType(_FormatTypeMixin, sqltypes.JSON.JSONIndexType):
|
||||
def _format_value(self, value: Any) -> str:
|
||||
if isinstance(value, int):
|
||||
formatted_value = "$[%s]" % value
|
||||
else:
|
||||
formatted_value = '$."%s"' % value
|
||||
return formatted_value
|
||||
|
||||
|
||||
class JSONPathType(_FormatTypeMixin, sqltypes.JSON.JSONPathType):
|
||||
def _format_value(self, value: Any) -> str:
|
||||
return "$%s" % (
|
||||
"".join(
|
||||
[
|
||||
"[%s]" % elem if isinstance(elem, int) else '."%s"' % elem
|
||||
for elem in value
|
||||
]
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,237 @@
|
||||
# dialects/sqlite/provision.py
|
||||
# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of SQLAlchemy and is released under
|
||||
# the MIT License: https://www.opensource.org/licenses/mit-license.php
|
||||
# mypy: ignore-errors
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from ... import event
|
||||
from ... import exc
|
||||
from ...engine import url as sa_url
|
||||
from ...testing import config
|
||||
from ...testing.provision import create_db
|
||||
from ...testing.provision import drop_db
|
||||
from ...testing.provision import follower_url_from_main
|
||||
from ...testing.provision import generate_driver_url
|
||||
from ...testing.provision import log
|
||||
from ...testing.provision import post_configure_engine
|
||||
from ...testing.provision import post_configure_testing_engine
|
||||
from ...testing.provision import profile_platform_tokens
|
||||
from ...testing.provision import run_reap_dbs
|
||||
from ...testing.provision import stop_test_class_outside_fixtures
|
||||
from ...testing.provision import temp_table_keyword_args
|
||||
from ...testing.provision import upsert
|
||||
|
||||
# TODO: I can't get this to build dynamically with pytest-xdist procs
|
||||
_drivernames = {
|
||||
"pysqlite",
|
||||
"aiosqlite",
|
||||
"pysqlcipher",
|
||||
"pysqlite_numeric",
|
||||
"pysqlite_dollar",
|
||||
}
|
||||
|
||||
|
||||
def _format_url(url, driver, ident):
|
||||
"""given a sqlite url + desired driver + ident, make a canonical
|
||||
URL out of it
|
||||
|
||||
"""
|
||||
url = sa_url.make_url(url)
|
||||
|
||||
if driver is None:
|
||||
driver = url.get_driver_name()
|
||||
|
||||
filename = url.database
|
||||
|
||||
needs_enc = driver == "pysqlcipher"
|
||||
name_token = None
|
||||
|
||||
if filename and filename != ":memory:":
|
||||
assert "test_schema" not in filename
|
||||
tokens = re.split(r"[_\.]", filename)
|
||||
|
||||
for token in tokens:
|
||||
if token in _drivernames:
|
||||
if driver is None:
|
||||
driver = token
|
||||
continue
|
||||
elif token in ("db", "enc"):
|
||||
continue
|
||||
elif name_token is None:
|
||||
name_token = token.strip("_")
|
||||
|
||||
assert name_token, f"sqlite filename has no name token: {url.database}"
|
||||
|
||||
new_filename = f"{name_token}_{driver}"
|
||||
if ident:
|
||||
new_filename += f"_{ident}"
|
||||
new_filename += ".db"
|
||||
if needs_enc:
|
||||
new_filename += ".enc"
|
||||
url = url.set(database=new_filename)
|
||||
|
||||
if needs_enc:
|
||||
url = url.set(password="test")
|
||||
|
||||
url = url.set(drivername="sqlite+%s" % (driver,))
|
||||
|
||||
return url
|
||||
|
||||
|
||||
@generate_driver_url.for_db("sqlite")
|
||||
def generate_driver_url(url, driver, query_str):
|
||||
url = _format_url(url, driver, None)
|
||||
|
||||
try:
|
||||
url.get_dialect()
|
||||
except exc.NoSuchModuleError:
|
||||
return None
|
||||
else:
|
||||
return url
|
||||
|
||||
|
||||
@follower_url_from_main.for_db("sqlite")
|
||||
def _sqlite_follower_url_from_main(url, ident):
|
||||
return _format_url(url, None, ident)
|
||||
|
||||
|
||||
@post_configure_engine.for_db("sqlite")
|
||||
def _sqlite_post_configure_engine(url, engine, follower_ident):
|
||||
from sqlalchemy import event
|
||||
|
||||
if follower_ident:
|
||||
attach_path = f"{follower_ident}_{engine.driver}_test_schema.db"
|
||||
else:
|
||||
attach_path = f"{engine.driver}_test_schema.db"
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
def connect(dbapi_connection, connection_record):
|
||||
# use file DBs in all cases, memory acts kind of strangely
|
||||
# as an attached
|
||||
|
||||
# NOTE! this has to be done *per connection*. New sqlite connection,
|
||||
# as we get with say, QueuePool, the attaches are gone.
|
||||
# so schemes to delete those attached files have to be done at the
|
||||
# filesystem level and not rely upon what attachments are in a
|
||||
# particular SQLite connection
|
||||
dbapi_connection.execute(
|
||||
f'ATTACH DATABASE "{attach_path}" AS test_schema'
|
||||
)
|
||||
|
||||
@event.listens_for(engine, "engine_disposed")
|
||||
def dispose(engine):
|
||||
"""most databases should be dropped using
|
||||
stop_test_class_outside_fixtures
|
||||
|
||||
however a few tests like AttachedDBTest might not get triggered on
|
||||
that main hook
|
||||
|
||||
"""
|
||||
|
||||
if os.path.exists(attach_path):
|
||||
os.remove(attach_path)
|
||||
|
||||
filename = engine.url.database
|
||||
|
||||
if filename and filename != ":memory:" and os.path.exists(filename):
|
||||
os.remove(filename)
|
||||
|
||||
|
||||
@post_configure_testing_engine.for_db("sqlite")
|
||||
def _sqlite_post_configure_testing_engine(url, engine, options, scope):
|
||||
|
||||
sqlite_savepoint = options.get("sqlite_savepoint", False)
|
||||
sqlite_share_pool = options.get("sqlite_share_pool", False)
|
||||
|
||||
if sqlite_savepoint and engine.name == "sqlite":
|
||||
# apply SQLite savepoint workaround
|
||||
@event.listens_for(engine, "connect")
|
||||
def do_connect(dbapi_connection, connection_record):
|
||||
dbapi_connection.isolation_level = None
|
||||
|
||||
@event.listens_for(engine, "begin")
|
||||
def do_begin(conn):
|
||||
conn.exec_driver_sql("BEGIN")
|
||||
|
||||
if sqlite_share_pool:
|
||||
# SingletonThreadPool, StaticPool both support "transfer"
|
||||
# so a new pool can share the same SQLite connection
|
||||
# (single thread only)
|
||||
if hasattr(engine.pool, "_transfer_from"):
|
||||
options["use_reaper"] = False
|
||||
engine.pool._transfer_from(config.db.pool)
|
||||
|
||||
|
||||
@create_db.for_db("sqlite")
|
||||
def _sqlite_create_db(cfg, eng, ident):
|
||||
pass
|
||||
|
||||
|
||||
@drop_db.for_db("sqlite")
|
||||
def _sqlite_drop_db(cfg, eng, ident):
|
||||
_drop_dbs_w_ident(eng.url.database, eng.driver, ident)
|
||||
|
||||
|
||||
def _drop_dbs_w_ident(databasename, driver, ident):
|
||||
for path in os.listdir("."):
|
||||
fname, ext = os.path.split(path)
|
||||
if ident in fname and ext in [".db", ".db.enc"]:
|
||||
log.info("deleting SQLite database file: %s", path)
|
||||
os.remove(path)
|
||||
|
||||
|
||||
@stop_test_class_outside_fixtures.for_db("sqlite")
|
||||
def stop_test_class_outside_fixtures(config, db, cls):
|
||||
db.dispose()
|
||||
|
||||
|
||||
@temp_table_keyword_args.for_db("sqlite")
|
||||
def _sqlite_temp_table_keyword_args(cfg, eng):
|
||||
return {"prefixes": ["TEMPORARY"]}
|
||||
|
||||
|
||||
@run_reap_dbs.for_db("sqlite")
|
||||
def _reap_sqlite_dbs(url, idents):
|
||||
log.info("db reaper connecting to %r", url)
|
||||
log.info("identifiers in file: %s", ", ".join(idents))
|
||||
url = sa_url.make_url(url)
|
||||
for ident in idents:
|
||||
for drivername in _drivernames:
|
||||
_drop_dbs_w_ident(url.database, drivername, ident)
|
||||
|
||||
|
||||
@upsert.for_db("sqlite")
|
||||
def _upsert(
|
||||
cfg,
|
||||
table,
|
||||
returning,
|
||||
*,
|
||||
set_lambda=None,
|
||||
sort_by_parameter_order=False,
|
||||
index_elements=None,
|
||||
):
|
||||
from sqlalchemy.dialects.sqlite import insert
|
||||
|
||||
stmt = insert(table)
|
||||
|
||||
if set_lambda:
|
||||
stmt = stmt.on_conflict_do_update(set_=set_lambda(stmt.excluded))
|
||||
else:
|
||||
stmt = stmt.on_conflict_do_nothing()
|
||||
|
||||
stmt = stmt.returning(
|
||||
*returning, sort_by_parameter_order=sort_by_parameter_order
|
||||
)
|
||||
return stmt
|
||||
|
||||
|
||||
@profile_platform_tokens.for_db("sqlite")
|
||||
def _sqlite_profile_platform_tokens(eng):
|
||||
# a file based database goes through the filesystem for every page it
|
||||
# reads, which a ``:memory:`` database doesn't
|
||||
return ("file",) if eng.dialect._is_url_file_db(eng.url) else ()
|
||||
@@ -0,0 +1,166 @@
|
||||
# dialects/sqlite/pysqlcipher.py
|
||||
# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of SQLAlchemy and is released under
|
||||
# the MIT License: https://www.opensource.org/licenses/mit-license.php
|
||||
# mypy: ignore-errors
|
||||
|
||||
|
||||
"""
|
||||
.. dialect:: sqlite+pysqlcipher
|
||||
:name: pysqlcipher
|
||||
:dbapi: sqlcipher 3 or pysqlcipher
|
||||
:connectstring: sqlite+pysqlcipher://:passphrase@/file_path[?kdf_iter=<iter>]
|
||||
|
||||
Dialect for support of DBAPIs that make use of the
|
||||
`SQLCipher <https://www.zetetic.net/sqlcipher>`_ backend.
|
||||
|
||||
|
||||
Driver
|
||||
------
|
||||
|
||||
Current dialect selection logic is:
|
||||
|
||||
* If the :paramref:`_sa.create_engine.module` parameter supplies a DBAPI module,
|
||||
that module is used.
|
||||
* Otherwise for Python 3, choose https://pypi.org/project/sqlcipher3/
|
||||
* If not available, fall back to https://pypi.org/project/pysqlcipher3/
|
||||
* For Python 2, https://pypi.org/project/pysqlcipher/ is used.
|
||||
|
||||
.. warning:: The ``pysqlcipher3`` and ``pysqlcipher`` DBAPI drivers are no
|
||||
longer maintained; the ``sqlcipher3`` driver as of this writing appears
|
||||
to be current. For future compatibility, any pysqlcipher-compatible DBAPI
|
||||
may be used as follows::
|
||||
|
||||
import sqlcipher_compatible_driver
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
e = create_engine(
|
||||
"sqlite+pysqlcipher://:password@/dbname.db",
|
||||
module=sqlcipher_compatible_driver,
|
||||
)
|
||||
|
||||
These drivers make use of the SQLCipher engine. This system essentially
|
||||
introduces new PRAGMA commands to SQLite which allows the setting of a
|
||||
passphrase and other encryption parameters, allowing the database file to be
|
||||
encrypted.
|
||||
|
||||
|
||||
Connect Strings
|
||||
---------------
|
||||
|
||||
The format of the connect string is in every way the same as that
|
||||
of the :mod:`~sqlalchemy.dialects.sqlite.pysqlite` driver, except that the
|
||||
"password" field is now accepted, which should contain a passphrase::
|
||||
|
||||
e = create_engine("sqlite+pysqlcipher://:testing@/foo.db")
|
||||
|
||||
For an absolute file path, two leading slashes should be used for the
|
||||
database name::
|
||||
|
||||
e = create_engine("sqlite+pysqlcipher://:testing@//path/to/foo.db")
|
||||
|
||||
A selection of additional encryption-related pragmas supported by SQLCipher
|
||||
as documented at https://www.zetetic.net/sqlcipher/sqlcipher-api/ can be passed
|
||||
in the query string, and will result in that PRAGMA being called for each
|
||||
new connection. Currently, ``cipher``, ``kdf_iter``
|
||||
``cipher_page_size`` and ``cipher_use_hmac`` are supported::
|
||||
|
||||
e = create_engine(
|
||||
"sqlite+pysqlcipher://:testing@/foo.db?cipher=aes-256-cfb&kdf_iter=64000"
|
||||
)
|
||||
|
||||
.. warning:: Previous versions of sqlalchemy did not take into consideration
|
||||
the encryption-related pragmas passed in the url string, that were silently
|
||||
ignored. This may cause errors when opening files saved by a
|
||||
previous sqlalchemy version if the encryption options do not match.
|
||||
|
||||
|
||||
Pooling Behavior
|
||||
----------------
|
||||
|
||||
The driver makes a change to the default pool behavior of pysqlite
|
||||
as described in :ref:`pysqlite_threading_pooling`. The pysqlcipher driver
|
||||
has been observed to be significantly slower on connection than the
|
||||
pysqlite driver, most likely due to the encryption overhead, so the
|
||||
dialect here defaults to using the :class:`.SingletonThreadPool`
|
||||
implementation,
|
||||
instead of the :class:`.NullPool` pool used by pysqlite. As always, the pool
|
||||
implementation is entirely configurable using the
|
||||
:paramref:`_sa.create_engine.poolclass` parameter; the :class:`.
|
||||
StaticPool` may
|
||||
be more feasible for single-threaded use, or :class:`.NullPool` may be used
|
||||
to prevent unencrypted connections from being held open for long periods of
|
||||
time, at the expense of slower startup time for new connections.
|
||||
|
||||
|
||||
""" # noqa
|
||||
|
||||
from .pysqlite import SQLiteDialect_pysqlite
|
||||
from ... import pool
|
||||
from ... import util
|
||||
|
||||
|
||||
class SQLiteDialect_pysqlcipher(SQLiteDialect_pysqlite):
|
||||
driver = "pysqlcipher"
|
||||
supports_statement_cache = True
|
||||
|
||||
pragmas = ("kdf_iter", "cipher", "cipher_page_size", "cipher_use_hmac")
|
||||
|
||||
@classmethod
|
||||
def import_dbapi(cls):
|
||||
try:
|
||||
import sqlcipher3 as sqlcipher
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
return sqlcipher
|
||||
|
||||
from pysqlcipher3 import dbapi2 as sqlcipher
|
||||
|
||||
return sqlcipher
|
||||
|
||||
def retrieve_dbapi_version(self, dbapi):
|
||||
# the version of sqlcipher3 / pysqlcipher3, rather than the Python
|
||||
# version reported by the pysqlite dialect
|
||||
return util.parse_version_string(getattr(dbapi, "version", None))
|
||||
|
||||
@classmethod
|
||||
def get_pool_class(cls, url):
|
||||
return pool.SingletonThreadPool
|
||||
|
||||
def on_connect_url(self, url):
|
||||
super_on_connect = super().on_connect_url(url)
|
||||
|
||||
# pull the info we need from the URL early. Even though URL
|
||||
# is immutable, we don't want any in-place changes to the URL
|
||||
# to affect things
|
||||
ip = self.identifier_preparer
|
||||
passphrase = ip.quote_identifier(url.password or "")
|
||||
query_pragmas = {
|
||||
prag: ip.quote_identifier(url.query[prag])
|
||||
for prag in self.pragmas
|
||||
if url.query.get(prag) is not None
|
||||
}
|
||||
|
||||
def on_connect(conn):
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(f"pragma key={passphrase}")
|
||||
for prag, value in query_pragmas.items():
|
||||
cursor.execute(f"pragma {prag}={value}")
|
||||
cursor.close()
|
||||
|
||||
if super_on_connect:
|
||||
super_on_connect(conn)
|
||||
|
||||
return on_connect
|
||||
|
||||
def create_connect_args(self, url):
|
||||
plain_url = url._replace(password=None)
|
||||
plain_url = plain_url.difference_update_query(self.pragmas)
|
||||
return super().create_connect_args(plain_url)
|
||||
|
||||
|
||||
dialect = SQLiteDialect_pysqlcipher
|
||||
@@ -0,0 +1,959 @@
|
||||
# dialects/sqlite/pysqlite.py
|
||||
# Copyright (C) 2005-2026 the SQLAlchemy authors and contributors
|
||||
# <see AUTHORS file>
|
||||
#
|
||||
# This module is part of SQLAlchemy and is released under
|
||||
# the MIT License: https://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
|
||||
r"""
|
||||
.. dialect:: sqlite+pysqlite
|
||||
:name: pysqlite
|
||||
:dbapi: sqlite3
|
||||
:connectstring: sqlite+pysqlite:///file_path
|
||||
:url: https://docs.python.org/library/sqlite3.html
|
||||
|
||||
Note that ``pysqlite`` is the same driver as the ``sqlite3``
|
||||
module included with the Python distribution.
|
||||
|
||||
Driver
|
||||
------
|
||||
|
||||
The ``sqlite3`` Python DBAPI is standard on all modern Python versions;
|
||||
for cPython and Pypy, no additional installation is necessary.
|
||||
|
||||
|
||||
Connect Strings
|
||||
---------------
|
||||
|
||||
The file specification for the SQLite database is taken as the "database"
|
||||
portion of the URL. Note that the format of a SQLAlchemy url is:
|
||||
|
||||
.. sourcecode:: text
|
||||
|
||||
driver://user:pass@host/database
|
||||
|
||||
This means that the actual filename to be used starts with the characters to
|
||||
the **right** of the third slash. So connecting to a relative filepath
|
||||
looks like::
|
||||
|
||||
# relative path
|
||||
e = create_engine("sqlite:///path/to/database.db")
|
||||
|
||||
An absolute path, which is denoted by starting with a slash, means you
|
||||
need **four** slashes::
|
||||
|
||||
# absolute path
|
||||
e = create_engine("sqlite:////path/to/database.db")
|
||||
|
||||
To use a Windows path, regular drive specifications and backslashes can be
|
||||
used. Double backslashes are probably needed::
|
||||
|
||||
# absolute path on Windows
|
||||
e = create_engine("sqlite:///C:\\path\\to\\database.db")
|
||||
|
||||
To use sqlite ``:memory:`` database specify it as the filename using
|
||||
``sqlite:///:memory:``. It's also the default if no filepath is
|
||||
present, specifying only ``sqlite://`` and nothing else::
|
||||
|
||||
# in-memory database (note three slashes)
|
||||
e = create_engine("sqlite:///:memory:")
|
||||
# also in-memory database
|
||||
e2 = create_engine("sqlite://")
|
||||
|
||||
.. _pysqlite_uri_connections:
|
||||
|
||||
URI Connections
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
Modern versions of SQLite support an alternative system of connecting using a
|
||||
`driver level URI <https://www.sqlite.org/uri.html>`_, which has the advantage
|
||||
that additional driver-level arguments can be passed including options such as
|
||||
"read only". The Python sqlite3 driver supports this mode under modern Python
|
||||
3 versions. The SQLAlchemy pysqlite driver supports this mode of use by
|
||||
specifying "uri=true" in the URL query string. The SQLite-level "URI" is kept
|
||||
as the "database" portion of the SQLAlchemy url (that is, following a slash)::
|
||||
|
||||
e = create_engine("sqlite:///file:path/to/database?mode=ro&uri=true")
|
||||
|
||||
.. note:: The "uri=true" parameter must appear in the **query string**
|
||||
of the URL. It will not currently work as expected if it is only
|
||||
present in the :paramref:`_sa.create_engine.connect_args`
|
||||
parameter dictionary.
|
||||
|
||||
The logic reconciles the simultaneous presence of SQLAlchemy's query string and
|
||||
SQLite's query string by separating out the parameters that belong to the
|
||||
Python sqlite3 driver vs. those that belong to the SQLite URI. This is
|
||||
achieved through the use of a fixed list of parameters known to be accepted by
|
||||
the Python side of the driver. For example, to include a URL that indicates
|
||||
the Python sqlite3 "timeout" and "check_same_thread" parameters, along with the
|
||||
SQLite "mode" and "nolock" parameters, they can all be passed together on the
|
||||
query string::
|
||||
|
||||
e = create_engine(
|
||||
"sqlite:///file:path/to/database?"
|
||||
"check_same_thread=true&timeout=10&mode=ro&nolock=1&uri=true"
|
||||
)
|
||||
|
||||
Above, the pysqlite / sqlite3 DBAPI would be passed arguments as::
|
||||
|
||||
sqlite3.connect(
|
||||
"file:path/to/database?mode=ro&nolock=1",
|
||||
check_same_thread=True,
|
||||
timeout=10,
|
||||
uri=True,
|
||||
)
|
||||
|
||||
Regarding future parameters added to either the Python or native drivers. new
|
||||
parameter names added to the SQLite URI scheme should be automatically
|
||||
accommodated by this scheme. New parameter names added to the Python driver
|
||||
side can be accommodated by specifying them in the
|
||||
:paramref:`_sa.create_engine.connect_args` dictionary,
|
||||
until dialect support is
|
||||
added by SQLAlchemy. For the less likely case that the native SQLite driver
|
||||
adds a new parameter name that overlaps with one of the existing, known Python
|
||||
driver parameters (such as "timeout" perhaps), SQLAlchemy's dialect would
|
||||
require adjustment for the URL scheme to continue to support this.
|
||||
|
||||
As is always the case for all SQLAlchemy dialects, the entire "URL" process
|
||||
can be bypassed in :func:`_sa.create_engine` through the use of the
|
||||
:paramref:`_sa.create_engine.creator`
|
||||
parameter which allows for a custom callable
|
||||
that creates a Python sqlite3 driver level connection directly.
|
||||
|
||||
.. seealso::
|
||||
|
||||
`Uniform Resource Identifiers <https://www.sqlite.org/uri.html>`_ - in
|
||||
the SQLite documentation
|
||||
|
||||
.. _pysqlite_regexp:
|
||||
|
||||
Regular Expression Support
|
||||
---------------------------
|
||||
|
||||
.. versionadded:: 1.4
|
||||
|
||||
Support for the :meth:`_sql.ColumnOperators.regexp_match` operator is provided
|
||||
using Python's re.search_ function. SQLite itself does not include a working
|
||||
regular expression operator; instead, it includes a non-implemented placeholder
|
||||
operator ``REGEXP`` that calls a user-defined function that must be provided.
|
||||
|
||||
SQLAlchemy's implementation makes use of the pysqlite create_function_ hook
|
||||
as follows::
|
||||
|
||||
|
||||
def regexp(a, b):
|
||||
return re.search(a, b) is not None
|
||||
|
||||
|
||||
sqlite_connection.create_function(
|
||||
"regexp",
|
||||
2,
|
||||
regexp,
|
||||
)
|
||||
|
||||
There is currently no support for regular expression flags as a separate
|
||||
argument, as these are not supported by SQLite's REGEXP operator, however these
|
||||
may be included inline within the regular expression string. See `Python regular expressions`_ for
|
||||
details.
|
||||
|
||||
.. seealso::
|
||||
|
||||
`Python regular expressions`_: Documentation for Python's regular expression syntax.
|
||||
|
||||
.. _create_function: https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.create_function
|
||||
|
||||
.. _re.search: https://docs.python.org/3/library/re.html#re.search
|
||||
|
||||
.. _Python regular expressions: https://docs.python.org/3/library/re.html#re.search
|
||||
|
||||
|
||||
|
||||
Compatibility with sqlite3 "native" date and datetime types
|
||||
-----------------------------------------------------------
|
||||
|
||||
The pysqlite driver includes the sqlite3.PARSE_DECLTYPES and
|
||||
sqlite3.PARSE_COLNAMES options, which have the effect of any column
|
||||
or expression explicitly cast as "date" or "timestamp" will be converted
|
||||
to a Python date or datetime object. The date and datetime types provided
|
||||
with the pysqlite dialect are not currently compatible with these options,
|
||||
since they render the ISO date/datetime including microseconds, which
|
||||
pysqlite's driver does not. Additionally, SQLAlchemy does not at
|
||||
this time automatically render the "cast" syntax required for the
|
||||
freestanding functions "current_timestamp" and "current_date" to return
|
||||
datetime/date types natively. Unfortunately, pysqlite
|
||||
does not provide the standard DBAPI types in ``cursor.description``,
|
||||
leaving SQLAlchemy with no way to detect these types on the fly
|
||||
without expensive per-row type checks.
|
||||
|
||||
Keeping in mind that pysqlite's parsing option is not recommended,
|
||||
nor should be necessary, for use with SQLAlchemy, usage of PARSE_DECLTYPES
|
||||
can be forced if one configures "native_datetime=True" on create_engine()::
|
||||
|
||||
engine = create_engine(
|
||||
"sqlite://",
|
||||
connect_args={
|
||||
"detect_types": sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES
|
||||
},
|
||||
native_datetime=True,
|
||||
)
|
||||
|
||||
With this flag enabled, the DATE and TIMESTAMP types (but note - not the
|
||||
DATETIME or TIME types...confused yet ?) will not perform any bind parameter
|
||||
or result processing. Execution of "func.current_date()" will return a string.
|
||||
"func.current_timestamp()" is registered as returning a DATETIME type in
|
||||
SQLAlchemy, so this function still receives SQLAlchemy-level result
|
||||
processing.
|
||||
|
||||
.. _pysqlite_threading_pooling:
|
||||
|
||||
Concurrency/Threading/Pooling Behavior
|
||||
--------------------------------------
|
||||
|
||||
The ``sqlite3`` DBAPI by default prohibits the use of a particular connection
|
||||
in a thread which is not the one in which it was created. As SQLite has
|
||||
matured, it's behavior under multiple threads has improved, and even includes
|
||||
options for memory only databases to be used in multiple threads.
|
||||
|
||||
The thread prohibition is known as "check same thread" and may be controlled
|
||||
using the ``sqlite3`` parameter ``check_same_thread``, which will disable or
|
||||
enable this check. SQLAlchemy's default behavior here is to set
|
||||
``check_same_thread`` to ``False`` automatically whenever a file-based database
|
||||
is in use, to establish compatibility with the default pool class
|
||||
:class:`.QueuePool`.
|
||||
|
||||
The SQLAlchemy ``pysqlite`` DBAPI establishes the connection pool differently
|
||||
based on the kind of SQLite database that's requested:
|
||||
|
||||
* When a ``:memory:`` SQLite database is specified, the dialect by default
|
||||
will use :class:`.SingletonThreadPool`. This pool maintains a single
|
||||
connection per thread, so that all access to the engine within the current
|
||||
thread use the same ``:memory:`` database - other threads would access a
|
||||
different ``:memory:`` database. The ``check_same_thread`` parameter
|
||||
defaults to ``True``.
|
||||
* When a file-based database is specified, the dialect will use
|
||||
:class:`.QueuePool` as the source of connections. at the same time,
|
||||
the ``check_same_thread`` flag is set to False by default unless overridden.
|
||||
|
||||
.. versionchanged:: 2.0
|
||||
|
||||
SQLite file database engines now use :class:`.QueuePool` by default.
|
||||
Previously, :class:`.NullPool` were used. The :class:`.NullPool` class
|
||||
may be used by specifying it via the
|
||||
:paramref:`_sa.create_engine.poolclass` parameter.
|
||||
|
||||
This selection is made based on the database name alone. Where a
|
||||
particular pool class is desired, it should be stated explicitly using the
|
||||
:paramref:`_sa.create_engine.poolclass` parameter, in which case no
|
||||
selection takes place at all.
|
||||
|
||||
.. deprecated:: 2.1
|
||||
|
||||
A URL that passes ``mode=memory`` in the query string is currently also
|
||||
given a single-connection pool class. This behavior is deprecated and
|
||||
will be removed in a future release, at which point such URLs will
|
||||
receive :class:`.QueuePool` like any other. This affects URLs such as
|
||||
``sqlite:///file:mydb?mode=memory&cache=shared&uri=true``, for which
|
||||
:class:`.QueuePool` is in fact the appropriate class, as a shared cache
|
||||
database supports multiple concurrent connections; see
|
||||
:ref:`pysqlite_uri_shared_cache`. Applications relying on the present
|
||||
behavior should state the pool class explicitly.
|
||||
|
||||
Disabling Connection Pooling for File Databases
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Pooling may be disabled for a file based database by specifying the
|
||||
:class:`.NullPool` implementation for the :func:`_sa.create_engine.poolclass`
|
||||
parameter::
|
||||
|
||||
from sqlalchemy import NullPool
|
||||
|
||||
engine = create_engine("sqlite:///myfile.db", poolclass=NullPool)
|
||||
|
||||
It's been observed that the :class:`.NullPool` implementation incurs an
|
||||
extremely small performance overhead for repeated checkouts due to the lack of
|
||||
connection reuse implemented by :class:`.QueuePool`. However, it still
|
||||
may be beneficial to use this class if the application is experiencing
|
||||
issues with files being locked.
|
||||
|
||||
Using a Memory Database in Multiple Threads or Coroutines
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
A ``:memory:`` SQLite database exists only within the scope of a single
|
||||
DBAPI connection. It is not possible for two separate ``sqlite3``
|
||||
connection objects to access the same ``:memory:`` database unless SQLite's
|
||||
shared-cache feature is enabled. Without shared cache, each connection
|
||||
creates its own independent in-memory database.
|
||||
|
||||
This means a ``:memory:`` database is **not suitable** for use with
|
||||
multiple concurrent threads or coroutines unless either:
|
||||
|
||||
* All workers are fully serialized (mutexed) against each other
|
||||
such that only one is using the database at a time, or
|
||||
* SQLite's shared-cache URI feature is used to allow multiple
|
||||
independent connections to access the same in-memory database.
|
||||
|
||||
The same considerations apply when using the :ref:`aiosqlite <aiosqlite>`
|
||||
dialect, which wraps ``pysqlite`` connections in an async interface —
|
||||
without shared cache, each connection still creates its own independent
|
||||
in-memory database.
|
||||
|
||||
The recommended approach for multithreaded or async in-memory
|
||||
use is the shared-cache URI feature, described below.
|
||||
|
||||
.. _pysqlite_uri_shared_cache:
|
||||
|
||||
Using a Shared-Cache Memory Database
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
SQLite's
|
||||
`URI shared-cache <https://www.sqlite.org/sharedcache.html>`_
|
||||
mode allows multiple independent DBAPI connections, each with
|
||||
their own transaction state, to access the same in-memory
|
||||
database. This is enabled by using a ``file:`` URI with
|
||||
``cache=shared`` and passing ``uri=true`` in the query
|
||||
string::
|
||||
|
||||
engine = create_engine("sqlite:///file::memory:?cache=shared&uri=true")
|
||||
|
||||
For async use with aiosqlite, use
|
||||
:func:`_asyncio.create_async_engine`::
|
||||
|
||||
engine = create_async_engine(
|
||||
"sqlite+aiosqlite:///file::memory:?cache=shared&uri=true"
|
||||
)
|
||||
|
||||
Because this URL form is treated as a file-based database by the
|
||||
dialect, :class:`.QueuePool` is used automatically and
|
||||
``check_same_thread`` defaults to ``False``, so no additional pool
|
||||
or connect_args configuration is needed. Each checkout from the
|
||||
pool is a distinct DBAPI connection with its own transaction state.
|
||||
|
||||
The shared-cache database is scoped by the filename component of
|
||||
the URI. ``file::memory:`` (empty name) is process-global — all
|
||||
engines in the process that use this URI share the same database.
|
||||
To maintain multiple independent in-memory databases within the
|
||||
same process, supply a distinct name for each. A named database of
|
||||
this kind requires ``mode=memory``, which presently causes a
|
||||
single-connection pool to be selected; :class:`.QueuePool` should
|
||||
therefore be requested explicitly::
|
||||
|
||||
from sqlalchemy.pool import QueuePool
|
||||
|
||||
engine_a = create_engine(
|
||||
"sqlite:///file:db_a?mode=memory&cache=shared&uri=true",
|
||||
poolclass=QueuePool,
|
||||
)
|
||||
engine_b = create_engine(
|
||||
"sqlite:///file:db_b?mode=memory&cache=shared&uri=true",
|
||||
poolclass=QueuePool,
|
||||
)
|
||||
|
||||
For :func:`_asyncio.create_async_engine`, use
|
||||
:class:`.AsyncAdaptedQueuePool` in the same way.
|
||||
|
||||
.. deprecated:: 2.1
|
||||
|
||||
Selection of a single-connection pool class based on ``mode=memory``
|
||||
is deprecated; a future release will use :class:`.QueuePool` for these
|
||||
URLs, at which point stating
|
||||
:paramref:`_sa.create_engine.poolclass` explicitly will no longer be
|
||||
necessary. See :ref:`pysqlite_threading_pooling`.
|
||||
|
||||
.. _pysqlite_shared_cache_lifespan:
|
||||
|
||||
Lifespan of a Shared-Cache Memory Database
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
A shared-cache in-memory database exists only for as long as at least one
|
||||
connection to it remains open; when the last connection is closed, the
|
||||
database and all of its contents are discarded. A subsequent connection
|
||||
using the same URI then opens a new, empty database, which typically
|
||||
surfaces as ``no such table`` errors.
|
||||
|
||||
:class:`.QueuePool` retains connections that have been returned to it, so
|
||||
in a default configuration the database will normally persist once the
|
||||
first connection has been established. This is a consequence of pool
|
||||
behavior rather than a guarantee, however, and the database will be
|
||||
discarded by ordinary pool operations including:
|
||||
|
||||
* :meth:`_engine.Engine.dispose`, which closes all connections currently
|
||||
in the pool
|
||||
* :paramref:`_sa.create_engine.pool_recycle`, as a recycled connection is
|
||||
closed before its replacement is opened
|
||||
* connection invalidation, including that performed by
|
||||
:paramref:`_sa.create_engine.pool_pre_ping`
|
||||
* use of :class:`.NullPool`, which closes each connection as it is
|
||||
returned
|
||||
|
||||
Where the database must survive independently of pool activity, hold a
|
||||
single connection open for as long as the database is needed::
|
||||
|
||||
engine = create_engine("sqlite:///file::memory:?cache=shared&uri=true")
|
||||
|
||||
# keep the database alive for the lifetime of the engine
|
||||
keepalive = engine.connect()
|
||||
|
||||
The same consideration applies to the :ref:`aiosqlite <aiosqlite>`
|
||||
dialect, using :meth:`_asyncio.AsyncEngine.connect`.
|
||||
|
||||
Using StaticPool for Single-Connection Memory Databases
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
An older approach for sharing a ``:memory:`` database among threads
|
||||
is to force all checkouts to return the same DBAPI connection using
|
||||
:class:`.StaticPool`. **This approach does not support any form of
|
||||
concurrency** and is only useful when all access to the engine is
|
||||
fully serialized, such as in single-threaded test suites::
|
||||
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
engine = create_engine(
|
||||
"sqlite://",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
|
||||
.. warning::
|
||||
|
||||
Because :class:`.StaticPool` maintains a single DBAPI connection,
|
||||
all :class:`.Session` or :class:`.Connection` objects that use
|
||||
this engine share that one underlying connection and its single
|
||||
SQLite transaction state. A ``ROLLBACK`` issued by one session
|
||||
(e.g. during error handling) will also roll back uncommitted work
|
||||
from any other session, and concurrent ``COMMIT`` / ``ROLLBACK``
|
||||
calls can interfere with each other unpredictably. This approach
|
||||
is only appropriate when access to the engine is fully serialized,
|
||||
such as in single-threaded test suites. For concurrent workloads,
|
||||
use the shared-cache URI approach described above.
|
||||
|
||||
Using Temporary Tables with SQLite
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Due to the way SQLite deals with temporary tables, if you wish to use a
|
||||
temporary table in a file-based SQLite database across multiple checkouts
|
||||
from the connection pool, such as when using an ORM :class:`.Session` where
|
||||
the temporary table should continue to remain after :meth:`.Session.commit` or
|
||||
:meth:`.Session.rollback` is called, a pool which maintains a single
|
||||
connection must be used. Use :class:`.SingletonThreadPool` if the scope is
|
||||
only needed within the current thread, or :class:`.StaticPool` is scope is
|
||||
needed within multiple threads for this case::
|
||||
|
||||
# maintain the same connection per thread
|
||||
from sqlalchemy.pool import SingletonThreadPool
|
||||
|
||||
engine = create_engine("sqlite:///mydb.db", poolclass=SingletonThreadPool)
|
||||
|
||||
|
||||
# maintain the same connection across all threads
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
engine = create_engine("sqlite:///mydb.db", poolclass=StaticPool)
|
||||
|
||||
Note that :class:`.SingletonThreadPool` should be configured for the number
|
||||
of threads that are to be used; beyond that number, connections will be
|
||||
closed out in a non deterministic way.
|
||||
|
||||
|
||||
Dealing with Mixed String / Binary Columns
|
||||
------------------------------------------------------
|
||||
|
||||
The SQLite database is weakly typed, and as such it is possible when using
|
||||
binary values, which in Python are represented as ``b'some string'``, that a
|
||||
particular SQLite database can have data values within different rows where
|
||||
some of them will be returned as a ``b''`` value by the Pysqlite driver, and
|
||||
others will be returned as Python strings, e.g. ``''`` values. This situation
|
||||
is not known to occur if the SQLAlchemy :class:`.LargeBinary` datatype is used
|
||||
consistently, however if a particular SQLite database has data that was
|
||||
inserted using the Pysqlite driver directly, or when using the SQLAlchemy
|
||||
:class:`.String` type which was later changed to :class:`.LargeBinary`, the
|
||||
table will not be consistently readable because SQLAlchemy's
|
||||
:class:`.LargeBinary` datatype does not handle strings so it has no way of
|
||||
"encoding" a value that is in string format.
|
||||
|
||||
To deal with a SQLite table that has mixed string / binary data in the
|
||||
same column, use a custom type that will check each row individually::
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy import TypeDecorator
|
||||
|
||||
|
||||
class MixedBinary(TypeDecorator):
|
||||
impl = String
|
||||
cache_ok = True
|
||||
|
||||
def process_result_value(self, value, dialect):
|
||||
if isinstance(value, str):
|
||||
value = bytes(value, "utf-8")
|
||||
elif value is not None:
|
||||
value = bytes(value)
|
||||
|
||||
return value
|
||||
|
||||
Then use the above ``MixedBinary`` datatype in the place where
|
||||
:class:`.LargeBinary` would normally be used.
|
||||
|
||||
.. _pysqlite_serializable:
|
||||
|
||||
Serializable isolation / Savepoints / Transactional DDL
|
||||
-------------------------------------------------------
|
||||
|
||||
A newly revised version of this important section is now available
|
||||
at the top level of the SQLAlchemy SQLite documentation, in the section
|
||||
:ref:`sqlite_transactions`.
|
||||
|
||||
|
||||
.. _pysqlite_udfs:
|
||||
|
||||
User-Defined Functions
|
||||
----------------------
|
||||
|
||||
pysqlite supports a `create_function() <https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.create_function>`_
|
||||
method that allows us to create our own user-defined functions (UDFs) in Python and use them directly in SQLite queries.
|
||||
These functions are registered with a specific DBAPI Connection.
|
||||
|
||||
SQLAlchemy uses connection pooling with file-based SQLite databases, so we need to ensure that the UDF is attached to the
|
||||
connection when it is created. That is accomplished with an event listener::
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
def udf():
|
||||
return "udf-ok"
|
||||
|
||||
|
||||
engine = create_engine("sqlite:///./db_file")
|
||||
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
def connect(conn, rec):
|
||||
conn.create_function("udf", 0, udf)
|
||||
|
||||
|
||||
for i in range(5):
|
||||
with engine.connect() as conn:
|
||||
print(conn.scalar(text("SELECT UDF()")))
|
||||
|
||||
""" # noqa
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
from typing import Callable
|
||||
from typing import cast
|
||||
from typing import Optional
|
||||
from typing import Pattern
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TypeVar
|
||||
from typing import Union
|
||||
|
||||
from .base import DATE
|
||||
from .base import DATETIME
|
||||
from .base import SQLiteDialect
|
||||
from ... import exc
|
||||
from ... import pool
|
||||
from ... import types as sqltypes
|
||||
from ... import util
|
||||
from ...util.typing import Self
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...engine.interfaces import ConnectArgsType
|
||||
from ...engine.interfaces import DBAPIConnection
|
||||
from ...engine.interfaces import DBAPICursor
|
||||
from ...engine.interfaces import DBAPIModule
|
||||
from ...engine.interfaces import IsolationLevel
|
||||
from ...engine.interfaces import ServerVersionInfoType
|
||||
from ...engine.url import URL
|
||||
from ...pool.base import PoolProxiedConnection
|
||||
from ...sql.type_api import _BindProcessorType
|
||||
from ...sql.type_api import _ResultProcessorType
|
||||
|
||||
|
||||
class _SQLite_pysqliteTimeStamp(DATETIME):
|
||||
def bind_processor( # type: ignore[override]
|
||||
self, dialect: SQLiteDialect
|
||||
) -> Optional[_BindProcessorType[Any]]:
|
||||
if dialect.native_datetime:
|
||||
return None
|
||||
else:
|
||||
return DATETIME.bind_processor(self, dialect)
|
||||
|
||||
def result_processor( # type: ignore[override]
|
||||
self, dialect: SQLiteDialect, coltype: object
|
||||
) -> Optional[_ResultProcessorType[Any]]:
|
||||
if dialect.native_datetime:
|
||||
return None
|
||||
else:
|
||||
return DATETIME.result_processor(self, dialect, coltype)
|
||||
|
||||
|
||||
class _SQLite_pysqliteDate(DATE):
|
||||
def bind_processor( # type: ignore[override]
|
||||
self, dialect: SQLiteDialect
|
||||
) -> Optional[_BindProcessorType[Any]]:
|
||||
if dialect.native_datetime:
|
||||
return None
|
||||
else:
|
||||
return DATE.bind_processor(self, dialect)
|
||||
|
||||
def result_processor( # type: ignore[override]
|
||||
self, dialect: SQLiteDialect, coltype: object
|
||||
) -> Optional[_ResultProcessorType[Any]]:
|
||||
if dialect.native_datetime:
|
||||
return None
|
||||
else:
|
||||
return DATE.result_processor(self, dialect, coltype)
|
||||
|
||||
|
||||
class SQLiteDialect_pysqlite(SQLiteDialect):
|
||||
default_paramstyle = "qmark"
|
||||
supports_statement_cache = True
|
||||
returns_native_bytes = True
|
||||
|
||||
colspecs = util.update_copy(
|
||||
SQLiteDialect.colspecs,
|
||||
{
|
||||
sqltypes.Date: _SQLite_pysqliteDate,
|
||||
sqltypes.TIMESTAMP: _SQLite_pysqliteTimeStamp,
|
||||
},
|
||||
)
|
||||
|
||||
description_encoding = None
|
||||
|
||||
driver = "pysqlite"
|
||||
|
||||
@classmethod
|
||||
def import_dbapi(cls) -> DBAPIModule:
|
||||
from sqlite3 import dbapi2 as sqlite
|
||||
|
||||
return cast("DBAPIModule", sqlite)
|
||||
|
||||
@classmethod
|
||||
def _is_url_file_db(cls, url: URL) -> bool:
|
||||
if (url.database and url.database != ":memory:") and (
|
||||
url.query.get("mode", None) != "memory"
|
||||
):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _warn_memory_mode_pool_selection(
|
||||
cls,
|
||||
url: URL,
|
||||
current_pool: type[pool.Pool],
|
||||
future_pool: type[pool.Pool],
|
||||
) -> None:
|
||||
"""Warn when the ``mode=memory`` query string argument is what
|
||||
caused a single-connection pool class to be selected.
|
||||
|
||||
See :ticket:`13433`.
|
||||
|
||||
"""
|
||||
if url.query.get("mode", None) != "memory":
|
||||
return
|
||||
|
||||
util.warn_deprecated(
|
||||
"Selection of the %s pool class based on the 'mode=memory' "
|
||||
"query string argument is deprecated; a future release will "
|
||||
"use %s for this URL. Indicate the intended pool class "
|
||||
"using the create_engine.poolclass parameter."
|
||||
% (current_pool.__name__, future_pool.__name__),
|
||||
"2.1",
|
||||
code="sqmp",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_pool_class(cls, url: URL) -> type[pool.Pool]:
|
||||
if cls._is_url_file_db(url):
|
||||
return pool.QueuePool
|
||||
else:
|
||||
cls._warn_memory_mode_pool_selection(
|
||||
url, pool.SingletonThreadPool, pool.QueuePool
|
||||
)
|
||||
return pool.SingletonThreadPool
|
||||
|
||||
def _get_server_version_info(
|
||||
self, connection: Any
|
||||
) -> ServerVersionInfoType:
|
||||
return self.dbapi.sqlite_version_info # type: ignore[no-any-return, union-attr] # noqa: E501
|
||||
|
||||
def retrieve_dbapi_version(self, dbapi: DBAPIModule) -> util.VersionInfo:
|
||||
# the ``sqlite3`` module ships with CPython and has no version of
|
||||
# its own (the legacy ``sqlite3.version`` attribute was frozen at
|
||||
# 2.6.0 and removed in Python 3.14), so the Python version is used
|
||||
# here, which is what its feature set actually tracks. The version
|
||||
# of the SQLite library itself is available as
|
||||
# :attr:`.Dialect.server_version_info`.
|
||||
return util.VersionInfo(sys.version_info[:3])
|
||||
|
||||
_isolation_lookup = SQLiteDialect._isolation_lookup.union(
|
||||
{
|
||||
"AUTOCOMMIT": None,
|
||||
}
|
||||
)
|
||||
|
||||
def set_isolation_level(
|
||||
self, dbapi_connection: DBAPIConnection, level: IsolationLevel
|
||||
) -> None:
|
||||
if level == "AUTOCOMMIT":
|
||||
dbapi_connection.isolation_level = None
|
||||
else:
|
||||
dbapi_connection.isolation_level = ""
|
||||
return super().set_isolation_level(dbapi_connection, level)
|
||||
|
||||
def detect_autocommit_setting(self, dbapi_conn: DBAPIConnection) -> bool:
|
||||
return dbapi_conn.isolation_level is None
|
||||
|
||||
def on_connect(self) -> Callable[[DBAPIConnection], None]:
|
||||
def regexp(a: str, b: Optional[str]) -> Optional[bool]:
|
||||
if b is None:
|
||||
return None
|
||||
return re.search(a, b) is not None
|
||||
|
||||
if self._get_server_version_info(None) >= (3, 9):
|
||||
# sqlite must be greater than 3.8.3 for deterministic=True
|
||||
# https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.create_function
|
||||
# the check is more conservative since there were still issues
|
||||
# with following 3.8 sqlite versions
|
||||
create_func_kw = {"deterministic": True}
|
||||
else:
|
||||
create_func_kw = {}
|
||||
|
||||
def set_regexp(dbapi_connection: DBAPIConnection) -> None:
|
||||
dbapi_connection.create_function(
|
||||
"regexp", 2, regexp, **create_func_kw
|
||||
)
|
||||
|
||||
def floor_func(dbapi_connection: DBAPIConnection) -> None:
|
||||
# NOTE: floor is optionally present in sqlite 3.35+ , however
|
||||
# as it is normally non-present we deliver floor() unconditionally
|
||||
# for now.
|
||||
# https://www.sqlite.org/lang_mathfunc.html
|
||||
dbapi_connection.create_function(
|
||||
"floor", 1, math.floor, **create_func_kw
|
||||
)
|
||||
|
||||
fns = [set_regexp, floor_func]
|
||||
|
||||
def connect(conn: DBAPIConnection) -> None:
|
||||
for fn in fns:
|
||||
fn(conn)
|
||||
|
||||
return connect
|
||||
|
||||
def create_connect_args(self, url: URL) -> ConnectArgsType:
|
||||
if url.username or url.password or url.host or url.port:
|
||||
raise exc.ArgumentError(
|
||||
"Invalid SQLite URL: %s\n"
|
||||
"Valid SQLite URL forms are:\n"
|
||||
" sqlite:///:memory: (or, sqlite://)\n"
|
||||
" sqlite:///relative/path/to/file.db\n"
|
||||
" sqlite:////absolute/path/to/file.db" % (url,)
|
||||
)
|
||||
|
||||
# theoretically, this list can be augmented, at least as far as
|
||||
# parameter names accepted by sqlite3/pysqlite, using
|
||||
# inspect.getfullargspec(). for the moment this seems like overkill
|
||||
# as these parameters don't change very often, and as always,
|
||||
# parameters passed to connect_args will always go to the
|
||||
# sqlite3/pysqlite driver.
|
||||
pysqlite_args = [
|
||||
("uri", bool),
|
||||
("timeout", float),
|
||||
("isolation_level", str),
|
||||
("detect_types", int),
|
||||
("check_same_thread", bool),
|
||||
("cached_statements", int),
|
||||
]
|
||||
opts = url.query
|
||||
pysqlite_opts: dict[str, Any] = {}
|
||||
for key, type_ in pysqlite_args:
|
||||
util.coerce_kw_type(opts, key, type_, dest=pysqlite_opts)
|
||||
|
||||
if pysqlite_opts.get("uri", False):
|
||||
uri_opts = dict(opts)
|
||||
# here, we are actually separating the parameters that go to
|
||||
# sqlite3/pysqlite vs. those that go the SQLite URI. What if
|
||||
# two names conflict? again, this seems to be not the case right
|
||||
# now, and in the case that new names are added to
|
||||
# either side which overlap, again the sqlite3/pysqlite parameters
|
||||
# can be passed through connect_args instead of in the URL.
|
||||
# If SQLite native URIs add a parameter like "timeout" that
|
||||
# we already have listed here for the python driver, then we need
|
||||
# to adjust for that here.
|
||||
for key, type_ in pysqlite_args:
|
||||
uri_opts.pop(key, None)
|
||||
filename: str = url.database # type: ignore[assignment]
|
||||
if uri_opts:
|
||||
# sorting of keys is for unit test support
|
||||
filename += "?" + (
|
||||
"&".join(
|
||||
"%s=%s" % (key, uri_opts[key])
|
||||
for key in sorted(uri_opts)
|
||||
)
|
||||
)
|
||||
else:
|
||||
# without uri=True, the SQLite URI query string is not in
|
||||
# play at all, so anything left over here is silently
|
||||
# discarded; warn rather than have it appear to take effect
|
||||
ignored = sorted(
|
||||
set(opts).difference(key for key, _ in pysqlite_args)
|
||||
)
|
||||
if ignored:
|
||||
util.warn(
|
||||
"Query string argument(s) %s are not accepted by the "
|
||||
"pysqlite driver and are being ignored; SQLite URI "
|
||||
"arguments require that 'uri=true' also be present "
|
||||
"in the URL."
|
||||
% (", ".join("'%s'" % key for key in ignored),),
|
||||
code="squa",
|
||||
)
|
||||
|
||||
filename = url.database or ":memory:"
|
||||
if filename != ":memory:":
|
||||
filename = os.path.abspath(filename)
|
||||
|
||||
pysqlite_opts.setdefault(
|
||||
"check_same_thread", not self._is_url_file_db(url)
|
||||
)
|
||||
|
||||
return ([filename], pysqlite_opts)
|
||||
|
||||
def is_disconnect(
|
||||
self,
|
||||
e: DBAPIModule.Error,
|
||||
connection: Optional[Union[PoolProxiedConnection, DBAPIConnection]],
|
||||
cursor: Optional[DBAPICursor],
|
||||
) -> bool:
|
||||
self.dbapi = cast("DBAPIModule", self.dbapi)
|
||||
return isinstance(
|
||||
e, self.dbapi.ProgrammingError
|
||||
) and "Cannot operate on a closed database." in str(e)
|
||||
|
||||
|
||||
dialect = SQLiteDialect_pysqlite
|
||||
|
||||
|
||||
class _SQLiteDialect_pysqlite_numeric(SQLiteDialect_pysqlite):
|
||||
"""numeric dialect for testing only
|
||||
|
||||
internal use only. This dialect is **NOT** supported by SQLAlchemy
|
||||
and may change at any time.
|
||||
|
||||
"""
|
||||
|
||||
supports_statement_cache = True
|
||||
default_paramstyle = "numeric"
|
||||
driver = "pysqlite_numeric"
|
||||
|
||||
_first_bind = ":1"
|
||||
_not_in_statement_regexp: Optional[Pattern[str]] = None
|
||||
|
||||
def __init__(self, *arg: Any, **kw: Any) -> None:
|
||||
kw.setdefault("paramstyle", "numeric")
|
||||
super().__init__(*arg, **kw)
|
||||
|
||||
def create_connect_args(self, url: URL) -> ConnectArgsType:
|
||||
arg, opts = super().create_connect_args(url)
|
||||
opts["factory"] = self._fix_sqlite_issue_99953()
|
||||
return arg, opts
|
||||
|
||||
def _fix_sqlite_issue_99953(self) -> Any:
|
||||
import sqlite3
|
||||
|
||||
first_bind = self._first_bind
|
||||
if self._not_in_statement_regexp:
|
||||
nis = self._not_in_statement_regexp
|
||||
|
||||
def _test_sql(sql: str) -> None:
|
||||
m = nis.search(sql)
|
||||
assert not m, f"Found {nis.pattern!r} in {sql!r}"
|
||||
|
||||
else:
|
||||
|
||||
def _test_sql(sql: str) -> None:
|
||||
pass
|
||||
|
||||
def _numeric_param_as_dict(
|
||||
parameters: Any,
|
||||
) -> Union[dict[str, Any], tuple[Any, ...]]:
|
||||
if parameters:
|
||||
assert isinstance(parameters, tuple)
|
||||
return {
|
||||
str(idx): value for idx, value in enumerate(parameters, 1)
|
||||
}
|
||||
else:
|
||||
return ()
|
||||
|
||||
class SQLiteFix99953Cursor(sqlite3.Cursor):
|
||||
def execute(self, sql: str, parameters: Any = ()) -> Self:
|
||||
_test_sql(sql)
|
||||
if first_bind in sql:
|
||||
parameters = _numeric_param_as_dict(parameters)
|
||||
return super().execute(sql, parameters)
|
||||
|
||||
def executemany(self, sql: str, parameters: Any) -> Self:
|
||||
_test_sql(sql)
|
||||
if first_bind in sql:
|
||||
parameters = [
|
||||
_numeric_param_as_dict(p) for p in parameters
|
||||
]
|
||||
return super().executemany(sql, parameters)
|
||||
|
||||
class SQLiteFix99953Connection(sqlite3.Connection):
|
||||
_CursorT = TypeVar("_CursorT", bound=sqlite3.Cursor)
|
||||
|
||||
def cursor(
|
||||
self,
|
||||
factory: Optional[
|
||||
Callable[[sqlite3.Connection], _CursorT]
|
||||
] = None,
|
||||
) -> _CursorT:
|
||||
if factory is None:
|
||||
factory = SQLiteFix99953Cursor # type: ignore[assignment]
|
||||
return super().cursor(factory=factory) # type: ignore[return-value] # noqa[E501]
|
||||
|
||||
def execute(
|
||||
self, sql: str, parameters: Any = ()
|
||||
) -> sqlite3.Cursor:
|
||||
_test_sql(sql)
|
||||
if first_bind in sql:
|
||||
parameters = _numeric_param_as_dict(parameters)
|
||||
return super().execute(sql, parameters)
|
||||
|
||||
def executemany(self, sql: str, parameters: Any) -> sqlite3.Cursor:
|
||||
_test_sql(sql)
|
||||
if first_bind in sql:
|
||||
parameters = [
|
||||
_numeric_param_as_dict(p) for p in parameters
|
||||
]
|
||||
return super().executemany(sql, parameters)
|
||||
|
||||
return SQLiteFix99953Connection
|
||||
|
||||
|
||||
class _SQLiteDialect_pysqlite_dollar(_SQLiteDialect_pysqlite_numeric):
|
||||
"""numeric dialect that uses $ for testing only
|
||||
|
||||
internal use only. This dialect is **NOT** supported by SQLAlchemy
|
||||
and may change at any time.
|
||||
|
||||
"""
|
||||
|
||||
supports_statement_cache = True
|
||||
default_paramstyle = "numeric_dollar"
|
||||
driver = "pysqlite_dollar"
|
||||
|
||||
_first_bind = "$1"
|
||||
_not_in_statement_regexp = re.compile(r"[^\d]:\d+")
|
||||
|
||||
def __init__(self, *arg: Any, **kw: Any) -> None:
|
||||
kw.setdefault("paramstyle", "numeric_dollar")
|
||||
super().__init__(*arg, **kw)
|
||||
Reference in New Issue
Block a user