diff --git a/docs/_static/demos/migration_workflow.gif b/docs/_static/demos/migration_workflow.gif deleted file mode 100644 index 5fb0ddbf0..000000000 Binary files a/docs/_static/demos/migration_workflow.gif and /dev/null differ diff --git a/docs/_tapes/migration_workflow.tape b/docs/_tapes/migration_workflow.tape deleted file mode 100644 index d08a78408..000000000 --- a/docs/_tapes/migration_workflow.tape +++ /dev/null @@ -1,61 +0,0 @@ -# SQLSpec Migration Workflow Demo -# Demonstrates: CLI migration commands with SQLite - -Output docs/_static/demos/migration_workflow.gif - -Set Shell "bash" -Set FontSize 14 -Set Width 1000 -Set Height 600 -Set Theme "Catppuccin Mocha" -Set Padding 20 -Set TypingSpeed 50ms - -Type "# SQLSpec Migrations - lightweight, code-first workflow" -Enter -Sleep 1s - -Hide -Type "source .venv/bin/activate" -Enter -Sleep 500ms -Type "cd $(mktemp -d)" -Enter -Sleep 500ms -Show - -Type "# Initialize the migration environment" -Enter -Sleep 500ms - -Type "sqlspec init" -Enter -Sleep 3s - -Type "# Create a new migration" -Enter -Sleep 500ms - -Type 'sqlspec create-migration -m "add users table"' -Enter -Sleep 3s - -Type "# Apply the migration" -Enter -Sleep 500ms - -Type "sqlspec upgrade" -Enter -Sleep 3s - -Type "# Check current revision" -Enter -Sleep 500ms - -Type "sqlspec show-current-revision" -Enter -Sleep 3s - -Type "# Clean, simple, no ORM required!" -Enter -Sleep 3s diff --git a/docs/changelog.rst b/docs/changelog.rst index 3955d9a97..ac2ddf188 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,32 @@ important operational fixes. Recent Updates ============== +Unreleased +------------------------------------------------------------------------------ + +**Changed:** + +* ``migration_config`` now rejects keys SQLSpec does not read, raising + :class:`~sqlspec.exceptions.ImproperConfigurationError` with the closest + valid key. A misspelling such as ``version_table`` instead of + ``version_table_name`` was previously accepted and silently ignored, leaving + the setting at its default. Remove or correct unrecognized keys to upgrade. + +**Fixed:** + +* Pointing ``--config`` at a module rather than a configuration object now + reports the ``module:attribute`` references that module exports, instead of + failing later with ``AttributeError: module has no attribute 'bind_key'``. +* Configuration references that misuse ``:`` report the accepted syntax rather + than an import failure. +* Errors raised while importing a configuration module keep their original type + and message instead of being rewrapped as an import failure. +* The "No SQLSpec config found" help text shows the ``[tool.sqlspec]`` section + name, which console markup previously consumed, and no longer mangles config + paths that contain colons. +* ``author`` is declared on :class:`~sqlspec.config.MigrationConfig`. The + migration generator already read it, but type checkers rejected it. + v0.57.0 ------------------------------------------------------------------------------ diff --git a/docs/examples/migration_quickstart_config.py b/docs/examples/migration_quickstart_config.py new file mode 100644 index 000000000..df008a91e --- /dev/null +++ b/docs/examples/migration_quickstart_config.py @@ -0,0 +1,7 @@ +from sqlspec.adapters.sqlite import SqliteConfig + +database_config = SqliteConfig( + bind_key="app", + connection_config={"database": "app.db"}, + migration_config={"script_location": "migrations", "version_table_name": "schema_versions"}, +) diff --git a/docs/examples/quickstart_migrations.py b/docs/examples/quickstart_migrations.py new file mode 100644 index 000000000..a9aaaf927 --- /dev/null +++ b/docs/examples/quickstart_migrations.py @@ -0,0 +1,52 @@ +import sqlite3 +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from sqlspec.cli import add_migration_commands + +__all__ = ("test_sqlite_migration_quickstart",) + + +def test_sqlite_migration_quickstart(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Exercise the documented config discovery and migration workflow.""" + config_example = Path(__file__).with_name("migration_quickstart_config.py") + (tmp_path / "database.py").write_text(config_example.read_text()) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("SQLSPEC_CONFIG", raising=False) + runner = CliRunner() + config_path = "database:database_config" + + cli_result = runner.invoke(add_migration_commands(), ["--config", config_path, "show-config"]) + assert cli_result.exit_code == 0, cli_result.output + assert "app" in cli_result.output + + env_result = runner.invoke( + add_migration_commands(), ["show-config"], env={"SQLSPEC_CONFIG": "database.database_config"} + ) + assert env_result.exit_code == 0, env_result.output + assert "app" in env_result.output + + (tmp_path / "pyproject.toml").write_text('[tool.sqlspec]\nconfig = "database:database_config"\n') + pyproject_result = runner.invoke(add_migration_commands(), ["show-config"]) + assert pyproject_result.exit_code == 0, pyproject_result.output + assert "Using config from pyproject.toml" in pyproject_result.output + + commands = ( + ["init", "--no-prompt"], + ["create-migration", "-m", "create users table", "--no-prompt"], + ["upgrade", "--no-prompt"], + ["show-current-revision"], + ) + for command in commands: + result = runner.invoke(add_migration_commands(), ["--config", config_path, *command]) + assert result.exit_code == 0, result.output + + assert (tmp_path / "app.db").is_file() + assert len(list((tmp_path / "migrations").glob("*.sql"))) == 1 + with sqlite3.connect(tmp_path / "app.db") as connection: + tracker_name = connection.execute( + "SELECT name FROM sqlite_master WHERE type = ? AND name = ?", ("table", "schema_versions") + ).fetchone() + assert tracker_name == ("schema_versions",) diff --git a/docs/usage/cli.rst b/docs/usage/cli.rst index 1bbdfe740..e74bb0f37 100644 --- a/docs/usage/cli.rst +++ b/docs/usage/cli.rst @@ -4,18 +4,21 @@ Command Line Interface SQLSpec includes a CLI for managing migrations and inspecting configuration. Use it when you want a fast, explicit workflow without additional tooling. -Configuration can come from ``--config``, ``SQLSPEC_CONFIG``, or -``[tool.sqlspec]`` in ``pyproject.toml``. +Every command needs a configuration reference. Pass it with ``--config``, set +``SQLSPEC_CONFIG``, or record it in ``[tool.sqlspec]`` -- see +:ref:`pointing-the-cli-at-your-configuration` for the details. Core Commands ------------- .. code-block:: console - sqlspec init - sqlspec create-migration -m "add users" - sqlspec upgrade - sqlspec downgrade + sqlspec --config database:database_config show-config + sqlspec --config database:database_config init --no-prompt + sqlspec --config database:database_config create-migration -m "add users" --no-prompt + sqlspec --config database:database_config upgrade --no-prompt + sqlspec --config database:database_config downgrade --no-prompt + sqlspec --config database:database_config show-current-revision Common Options -------------- @@ -23,6 +26,7 @@ Common Options - ``--bind-key`` targets a specific database configuration. - ``--no-prompt`` skips confirmation prompts. - ``--format`` selects SQL vs Python migration files. +- ``--validate-config`` reports each configuration and whether it is async-capable. - ``--use-logger`` emits migration output via structured logger. - ``--no-echo`` disables console output for migration commands. - ``--summary`` emits a single summary log entry when logger output is enabled. @@ -30,6 +34,8 @@ Common Options Tips ---- +- ``show-config`` verifies the CLI resolved the configuration you expected + before you run anything that touches the database. - Run ``sqlspec --help`` to see global options. - Run ``sqlspec upgrade --help`` to see command-specific migration options. diff --git a/docs/usage/migrations.rst b/docs/usage/migrations.rst index 99dc22c99..1497afe20 100644 --- a/docs/usage/migrations.rst +++ b/docs/usage/migrations.rst @@ -1,39 +1,94 @@ Migrations ========== -.. image:: /_static/demos/migration_workflow.gif - :alt: SQLSpec migration workflow demo - :class: demo-gif - SQLSpec ships with a built-in migration system backed by the SQL file loader. Use it when you want a lightweight, code-first workflow without pulling in Alembic or a full ORM stack. -Core Concepts -------------- - - Migrations are SQL or Python files stored in a migrations directory. -- Each database configuration can include its own migration settings. +- Each database configuration carries its own migration settings. - Extension migrations (ADK, events, Litestar sessions) are opt-in and versioned. - Any installed package can ship migrations, not just those under ``sqlspec.extensions``. -Common Commands ---------------- +Quickstart +---------- + +Export a configuration from a module the CLI can import. Importing this module +only defines the configuration -- migrations run when you invoke the CLI, not at +application import time. + +.. literalinclude:: /examples/migration_quickstart_config.py + :language: python + :caption: database.py + +Point the CLI at that object with ``module:attribute`` and run the workflow: + +.. code-block:: console + + sqlspec --config database:database_config show-config + sqlspec --config database:database_config init --no-prompt + sqlspec --config database:database_config create-migration -m "create users table" --no-prompt + sqlspec --config database:database_config upgrade --no-prompt + sqlspec --config database:database_config show-current-revision + +``show-config`` is the fastest way to confirm the CLI found what you expected +before running anything that touches the database. + +This example and the full command sequence are exercised by the documentation +test suite. + +.. _pointing-the-cli-at-your-configuration: + +Pointing the CLI at your configuration +-------------------------------------- + +The reference names a module and the attribute holding your configuration. Both +separators work, so ``database:database_config`` and +``database.database_config`` are equivalent: .. code-block:: console - sqlspec init - sqlspec create-migration -m "add users" - sqlspec upgrade + sqlspec --config database:database_config show-config + +The attribute may be a single configuration, a list of configurations, or a +factory function returning either. Naming the module alone is not enough -- +SQLSpec reports the references the module exports so you can correct the +command. + +To avoid repeating ``--config``, set an environment variable: + +.. code-block:: console + + export SQLSPEC_CONFIG=database:database_config + sqlspec show-config + +Or record it once in ``pyproject.toml``: + +.. code-block:: toml + + [tool.sqlspec] + config = "database:database_config" + +``--config`` wins over ``SQLSPEC_CONFIG``, which wins over ``pyproject.toml``. + +To manage several databases at once, separate references with commas. +Configurations are deduplicated by ``bind_key``, so give each one a distinct +key: + +.. code-block:: console + + sqlspec --config database:primary_config,database:replica_config upgrade + +Modules are imported from the current working directory, so a ``database.py`` +beside your ``pyproject.toml`` is importable without installing your project. Configuration ------------- -Set ``migration_config`` on your database configuration to customize script -locations, version table names, and extension migration behavior. - -The migration CLI resolves config from ``--config``, ``SQLSPEC_CONFIG``, or -``[tool.sqlspec]`` in ``pyproject.toml``. +``migration_config`` customizes script locations, the version table, and +extension behavior. Unrecognized keys raise +:class:`~sqlspec.exceptions.ImproperConfigurationError` at construction rather +than being silently ignored, so a misspelling surfaces immediately. .. code-block:: python @@ -43,20 +98,19 @@ The migration CLI resolves config from ``--config``, ``SQLSPEC_CONFIG``, or connection_config={"database": "/tmp/analytics.db"}, migration_config={ "script_location": "migrations/duckdb", - "version_table": "_schema_versions", + "version_table_name": "_schema_versions", }, ) - # Apply all pending migrations - config.migrate_up() +Migrations can also be driven in process: - # Apply up to a specific revision - config.migrate_up(revision="003") +.. code-block:: python - # Dry run to see what would happen + config.migrate_up() + config.migrate_up(revision="003") config.migrate_up(dry_run=True) -For async configs, ``migrate_up()`` returns an awaitable: +For async configurations, ``migrate_up()`` returns an awaitable: .. code-block:: python @@ -69,18 +123,138 @@ For async configs, ``migrate_up()`` returns an awaitable: await config.migrate_up() -Extension migrations are auto-included when the corresponding entry exists in -``extension_config``. Use ``migration_config["exclude_extensions"]`` to skip a -specific extension, ``migration_config["include_extensions"]`` to opt in -explicitly by extension name, or ``migration_config["enabled"] = False`` to -disable migrations entirely for a database config. +Common keys +~~~~~~~~~~~ + +.. list-table:: + :header-rows: 1 + :widths: 26 74 + + * - Key + - Purpose + * - ``script_location`` + - Migrations directory. Defaults to ``migrations``. + * - ``version_table_name`` + - Tracking table name. Defaults to ``sqlspec_migrations``. + * - ``enabled`` + - Set ``False`` to exclude this configuration from CLI operations. + * - ``strict_ordering`` + - Reject out-of-order migrations. Defaults to ``False``. + * - ``transactional`` + - Wrap each migration in a transaction where the adapter supports it. + * - ``include_extensions`` / ``exclude_extensions`` + - Opt extensions into or out of migration discovery by name. + +See :class:`~sqlspec.config.MigrationConfig` for the complete set. + +Running Against an Existing Schema +---------------------------------- + +Set ``default_schema`` when migration SQL should run against a pre-existing +schema without qualifying every table in every migration file. SQLSpec +validates the schema before creating the tracker table or applying DDL, then +configures the session before each migration runs. + +Set ``version_table_schema`` when the tracker table belongs somewhere other +than the objects being migrated. It falls back to ``default_schema``; if +neither is set, the tracker table is unqualified and uses the adapter's normal +default namespace. + +.. code-block:: python + + from sqlspec.adapters.asyncpg import AsyncpgConfig + + config = AsyncpgConfig( + connection_config={"dsn": "postgresql://localhost/app"}, + migration_config={ + "script_location": "migrations/postgres", + "version_table_name": "schema_versions", + "default_schema": "app_schema", + "version_table_schema": "admin_schema", + }, + ) + +Create the target schema before running migrations. The migration role needs +the database-specific privileges to create objects there -- for PostgreSQL, +usually ``USAGE`` and ``CREATE`` on the target schema plus permission to create +or update the tracker table. + +Example with unqualified DDL: + +.. literalinclude:: /examples/patterns/migrations_with_schema.py + :language: python + :start-after: # start-example + :end-before: # end-example + +Adapter support +~~~~~~~~~~~~~~~ + +Support is opt-in per adapter via the ``supports_migration_schemas`` class +flag. Configuring ``default_schema`` against an adapter that does not opt in +raises ``MigrationError`` before any DDL is issued. -Third-Party Extension Migrations --------------------------------- +.. list-table:: Supported + :header-rows: 1 + :widths: 34 66 + + * - Adapter + - Mechanism + * - ``asyncpg``, ``psycopg``, ``psqlpy`` + - ``SET LOCAL search_path`` when transactional, otherwise ``SET + search_path`` followed by ``RESET``. Validates against + ``information_schema.schemata``. + * - ``cockroach_asyncpg``, ``cockroach_psycopg`` + - Inherit the PostgreSQL driver behavior above; CockroachDB accepts ``SET + search_path`` over the PostgreSQL wire protocol. + * - ``adbc`` (PostgreSQL dialect) + - Same as ``asyncpg``. Detection is dialect-based on the configured ADBC + URI, so ``supports_migration_schemas`` is ``True`` only when the + resolved dialect is PostgreSQL-compatible. + * - ``oracledb`` + - ``ALTER SESSION SET CURRENT_SCHEMA``, validated against ``ALL_USERS``. + Names follow Oracle's stored identifier rules: unquoted lowercase names + are uppercased, mixed-case and quoted names are preserved. + * - ``duckdb`` + - ``SET search_path``. Validates against ``information_schema.schemata``. -Extension names resolve against the ``sqlspec.extensions.`` namespace by -default. A package distributed separately from SQLSpec points at its own -migrations directory with ``migrations_path``: +.. list-table:: Not supported + :header-rows: 1 + :widths: 34 66 + + * - Adapter + - Use instead + * - ``sqlite``, ``aiosqlite`` + - SQLite has no schema namespace. Layer additional databases with + ``ATTACH DATABASE``. + * - ``asyncmy``, ``aiomysql``, ``mysqlconnector``, ``pymysql`` + - MySQL conflates schema and database. Select the target database in the + connection URL, or issue ``USE`` inside the migration. + * - ``adbc`` (non-PostgreSQL dialects, including SQL Server) + - No portable per-session schema setter. Configure the default schema at + the user or login level in the database. + * - ``mssql_python`` + - SQL Server resolves the default schema from the login. Set it with + ``ALTER USER ... WITH DEFAULT_SCHEMA = ...``. + * - ``bigquery`` + - Cross-dataset DDL requires fully qualified + ``project.dataset.table`` references; there is no session-scoped default + dataset. + * - ``spanner`` + - Objects are tied to a single schema per database, with no session-scoped + switch. + * - ``arrow_odbc`` + - ODBC connection-string semantics vary per driver. Configure the default + schema through the DSN. + +Extension Migrations +-------------------- + +Extensions are auto-included when a matching entry exists in +``extension_config``. Names resolve against the ``sqlspec.extensions.`` +namespace by default. + +A package distributed separately from SQLSpec points at its own migrations +directory with ``migrations_path``: .. code-block:: python @@ -96,15 +270,12 @@ migrations directory with ``migrations_path``: Declaring ``migrations_path`` auto-includes the extension, so it does not also need to appear in ``include_extensions``. ``exclude_extensions`` still opts it -back out. +back out. The value takes either form: -``migrations_path`` accepts either form: - -- A ``':'`` specification, resolved against the installed - package. Portable across machines, so this is the form to use in - ``[tool.sqlspec]`` configuration. -- A filesystem path, absolute or relative to the working directory, matching how - ``script_location`` resolves. +- A ``':'`` specification resolved against the installed + package. Portable across machines, so prefer it in ``pyproject.toml``. +- A filesystem path, absolute or relative to the working directory, matching + how ``script_location`` resolves. Packages that register migrations at runtime can call ``add_extension_migrations`` instead of declaring the key: @@ -121,142 +292,40 @@ Packages that register migrations at runtime can call Both forms record the extension under ``extension_config`` and opt it into ``include_extensions``. Call ``add_extension_migrations`` before -``get_migration_commands()``; mutating ``extension_config`` directly after the +``get_migration_commands()`` -- mutating ``extension_config`` directly after the configuration is built does not re-run discovery. .. note:: - Migrations are versioned under an ``ext_{name}_`` prefix, and that prefix is - written to the migration tracking table. Keep the extension name stable once - migrations have been applied — renaming it orphans the applied records. + Extension migrations are versioned under an ``ext_{name}_`` prefix, and that + prefix is written to the tracking table. Keep the extension name stable once + migrations have been applied -- renaming it orphans the applied records. A package shipping migrations must include the directory as package data. If it compiles its own modules, the migration sources must remain on disk, since Python migrations are read and compiled at runtime. -Configuring a Default Schema ----------------------------- - -Use ``migration_config["default_schema"]`` when migration SQL should run -against a pre-existing schema without qualifying every table in each migration -file. SQLSpec validates the schema before creating the tracker table or applying -DDL, then configures the migration session before each migration is executed. - -Use ``migration_config["version_table_schema"]`` when the migration tracker -table should live somewhere different from the objects managed by migrations. -If ``version_table_schema`` is not set, the tracker schema resolves to -``default_schema``. If neither field is set, the tracker table is unqualified and -uses the adapter's normal default namespace. - -.. code-block:: python - - from sqlspec.adapters.asyncpg import AsyncpgConfig - - config = AsyncpgConfig( - connection_config={"dsn": "postgresql://localhost/app"}, - migration_config={ - "script_location": "migrations/postgres", - "version_table_name": "schema_versions", - "default_schema": "app_schema", - "version_table_schema": "admin_schema", - }, - ) - -The operator must create the target schema before running migrations. The -migration role also needs the database-specific privileges to create objects -there. For PostgreSQL, that usually means ``USAGE`` and -``CREATE`` on the target schema, plus permission to create or update the -tracker table. +Output and Logging +------------------ -Adapter support is opt-in via the ``supports_migration_schemas`` class flag on -each config. Configuring ``default_schema`` against an adapter that does not -opt in raises ``MigrationError`` before any DDL is issued. +Control output with ``migration_config`` keys or their CLI equivalents: .. list-table:: :header-rows: 1 - :widths: 28 18 54 - - * - Adapter - - Default schema - - Mechanism - * - ``asyncpg`` - - Supported - - ``SET LOCAL search_path`` (transactional) / ``SET search_path`` + ``RESET`` (non-transactional); - validates ``information_schema.schemata``. - * - ``psycopg`` (sync and async) - - Supported - - Same as ``asyncpg``. - * - ``psqlpy`` - - Supported - - Same as ``asyncpg``. - * - ``cockroach_asyncpg`` - - Supported - - Inherits ``asyncpg`` behavior. CockroachDB exposes the PostgreSQL - wire protocol and accepts ``SET search_path``. - * - ``cockroach_psycopg`` (sync and async) - - Supported - - Inherits ``psycopg`` behavior. - * - ``adbc`` (PostgreSQL dialect) - - Supported - - Same as ``asyncpg``. Detection is dialect-based on the configured - ADBC URI; ``supports_migration_schemas`` becomes ``True`` only when - the resolved dialect is PostgreSQL-compatible. - * - ``oracledb`` (sync and async) - - Supported - - ``ALTER SESSION SET CURRENT_SCHEMA``; validates ``ALL_USERS``. The - schema is normalized to Oracle's stored identifier rules: lowercase - unquoted names are uppercased, while mixed-case and explicitly quoted - names are preserved. - * - ``duckdb`` - - Supported - - ``SET search_path``; validates ``information_schema.schemata``. - * - ``sqlite``, ``aiosqlite`` - - Not supported - - SQLite has no schema namespace; use ``ATTACH DATABASE`` to layer - additional databases instead. - * - ``asyncmy``, ``aiomysql``, ``mysqlconnector``, ``pymysql`` - - Not supported - - MySQL conflates schema and database. Select the target database in - the connection URL or via ``USE`` inside the migration. - * - ``adbc`` (non-PostgreSQL dialects, including SQL Server) - - Not supported - - ADBC does not expose a portable per-session schema setter for these - dialects. Configure the default schema at the user or login level in - the underlying database. - * - ``mssql_python`` - - Not supported - - SQL Server resolves the default schema from the login. Set it with - ``ALTER USER ... WITH DEFAULT_SCHEMA = ...`` in your database. - * - ``bigquery`` - - Not supported - - BigQuery requires fully qualified ``project.dataset.table`` references - for cross-dataset DDL; there is no session-scoped default dataset. - * - ``spanner`` - - Not supported - - Cloud Spanner ties objects to a single schema per database; there is - no session-scoped switch. - * - ``arrow_odbc`` - - Not supported - - ODBC connection-string semantics vary per driver. Configure the - default schema through the underlying DSN. - -Example with unqualified DDL: - -.. literalinclude:: /examples/patterns/migrations_with_schema.py - :language: python - :start-after: # start-example - :end-before: # end-example - -Logging and Echo Controls -------------------------- - -Configure output behavior with ``migration_config`` or CLI flags: - -- ``use_logger`` to emit structured logs instead of console output. -- ``echo`` to control console output when not using the logger. -- ``summary_only`` to emit a single summary log entry when logger output is enabled. - -The CLI equivalents are ``--use-logger``, ``--no-echo``, and ``--summary``. + :widths: 24 20 56 + + * - Key + - CLI flag + - Effect + * - ``use_logger`` + - ``--use-logger`` + - Emit structured logs instead of console output. + * - ``echo`` + - ``--no-echo`` + - Control console output when not using the logger. + * - ``summary_only`` + - ``--summary`` + - Emit a single summary log entry when logger output is enabled. Related Guides -------------- diff --git a/sqlspec/cli.py b/sqlspec/cli.py index 204a1993c..304490c26 100644 --- a/sqlspec/cli.py +++ b/sqlspec/cli.py @@ -7,11 +7,12 @@ import rich_click as click from click.core import ParameterSource from rich import get_console +from rich.markup import escape from rich.prompt import Confirm, Prompt from rich.table import Table from sqlspec.config import AsyncDatabaseConfig, SyncDatabaseConfig -from sqlspec.exceptions import ConfigResolverError +from sqlspec.exceptions import SQLSpecError from sqlspec.utils.config_tools import discover_config_from_pyproject, resolve_config_sync from sqlspec.utils.module_loader import import_string from sqlspec.utils.sync_tools import run_ @@ -67,7 +68,7 @@ def sqlspec_group(ctx: "click.Context", config: str | None, validate_config: boo console.print("\nSpecify config using one of:") console.print(" 1. CLI flag: sqlspec --config myapp.config:get_configs ") console.print(" 2. Environment var: export SQLSPEC_CONFIG=myapp.config:get_configs") - console.print(" 3. pyproject.toml: [tool.sqlspec]") + console.print(f" 3. pyproject.toml: {escape('[tool.sqlspec]')}") console.print(' config = "myapp.config:get_configs"') ctx.exit(1) @@ -111,8 +112,8 @@ def sqlspec_group(ctx: "click.Context", config: str | None, validate_config: boo execution_hint = "[dim cyan](async-capable)[/]" if is_async else "[dim](sync)[/]" console.print(f" [dim]•[/] {config_name}: {config_type} {execution_hint}") - except (ImportError, ConfigResolverError) as e: - console.print(f"[red]Error loading config: {e}[/]") + except (ImportError, SQLSpecError) as e: + console.print(f"[red]Error loading config: {escape(str(e))}[/]", emoji=False) ctx.exit(1) finally: if cwd_added and cwd in sys.path and sys.path[0] == cwd: diff --git a/sqlspec/config.py b/sqlspec/config.py index f726e55c9..688c271f1 100644 --- a/sqlspec/config.py +++ b/sqlspec/config.py @@ -12,6 +12,7 @@ import threading from abc import ABC, abstractmethod from collections.abc import Callable, Mapping +from difflib import get_close_matches from inspect import Signature, signature from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar, cast @@ -26,7 +27,7 @@ create_sync_pool, seed_runtime_driver_features, ) -from sqlspec.exceptions import MissingDependencyError +from sqlspec.exceptions import ImproperConfigurationError, MissingDependencyError from sqlspec.extensions.events import EventRuntimeHints from sqlspec.loader import SQLFileLoader from sqlspec.migrations import AsyncMigrationTracker, SyncMigrationTracker, create_migration_commands @@ -46,6 +47,7 @@ __all__ = ( + "MIGRATION_CONFIG_KEYS", "ADKConfig", "AsyncConfigT", "AsyncDatabaseConfig", @@ -69,6 +71,7 @@ "StarletteConfig", "SyncConfigT", "SyncDatabaseConfig", + "validate_migration_config_keys", ) AsyncConfigT = TypeVar("AsyncConfigT", bound="AsyncDatabaseConfig[Any, Any, Any] | NoPoolAsyncConfig[Any, Any]") @@ -132,6 +135,9 @@ class MigrationConfig(TypedDict): project_root: NotRequired[str] """Path to the project root directory. Used for relative path resolution.""" + author: NotRequired[str] + """Author recorded on generated migration files. Defaults to the detected git user.""" + enabled: NotRequired[bool] """Whether this configuration should be included in CLI operations. Defaults to True.""" @@ -193,6 +199,31 @@ class MigrationConfig(TypedDict): """ +MIGRATION_CONFIG_KEYS: "frozenset[str]" = MigrationConfig.__required_keys__ | MigrationConfig.__optional_keys__ + + +def validate_migration_config_keys(migration_config: "Mapping[str, Any]") -> None: + """Reject migration configuration keys that SQLSpec does not read. + + Args: + migration_config: Migration configuration mapping to check. + + Raises: + ImproperConfigurationError: If the mapping contains an unrecognized key. + """ + unknown = sorted(key for key in migration_config if key not in MIGRATION_CONFIG_KEYS) + if not unknown: + return + + lines = [] + for key in unknown: + suggestions = get_close_matches(key, MIGRATION_CONFIG_KEYS, n=1, cutoff=0.6) + hint = f" Did you mean {suggestions[0]!r}?" if suggestions else "" + lines.append(f"Unknown migration_config key {key!r}.{hint}") + lines.append(f"Valid keys: {', '.join(sorted(MIGRATION_CONFIG_KEYS))}.") + raise ImproperConfigurationError(" ".join(lines)) + + class FlaskConfig(TypedDict): """Configuration options for Flask SQLSpec extension. @@ -831,7 +862,9 @@ def migration_config(self) -> "dict[str, Any] | MigrationConfig": @migration_config.setter def migration_config(self, value: "dict[str, Any] | MigrationConfig | None") -> None: """Store migration configuration and refresh derived migration helpers.""" - object.__setattr__(self, "_migration_config", dict(cast("dict[str, Any]", value) or {})) + resolved = dict(cast("dict[str, Any]", value) or {}) + validate_migration_config_keys(resolved) + object.__setattr__(self, "_migration_config", resolved) if self._has_initialized_attribute("extension_config"): self._ensure_extension_migrations() if self._migration_components_ready(): diff --git a/sqlspec/utils/config_tools.py b/sqlspec/utils/config_tools.py index 19c3c9055..aa12feb98 100644 --- a/sqlspec/utils/config_tools.py +++ b/sqlspec/utils/config_tools.py @@ -10,6 +10,7 @@ import sys from collections.abc import Sequence from pathlib import Path +from types import ModuleType from typing import TYPE_CHECKING, Any, cast from sqlspec.exceptions import ConfigResolverError, ImproperConfigurationError @@ -144,6 +145,30 @@ def parse_pyproject_config(pyproject_path: "Path") -> str | None: # ============================================================================= +def _normalize_config_path(config_path: str) -> str: + """Normalize supported config resolver path syntax to a dotted path. + + Args: + config_path: Dotted ``module.attribute`` or ``module:attribute`` path. + + Returns: + A dotted path accepted by :func:`import_string`. + + Raises: + ConfigResolverError: If the path uses ``:`` but is not ``module:attribute``. + """ + module_path, separator, attribute_path = config_path.partition(":") + if not separator: + return config_path + if not module_path or not attribute_path or ":" in attribute_path: + msg = ( + f"Config path '{config_path}' is not a valid reference. " + "Use 'module:attribute' with a single ':', or dotted 'module.attribute'." + ) + raise ConfigResolverError(msg) + return f"{module_path}.{attribute_path}" + + async def resolve_config_async( config_path: str, ) -> "list[AsyncDatabaseConfig[Any, Any, Any] | SyncDatabaseConfig[Any, Any, Any]] | AsyncDatabaseConfig[Any, Any, Any] | SyncDatabaseConfig[Any, Any, Any]": @@ -161,7 +186,7 @@ async def resolve_config_async( ConfigResolverError: If config resolution fails. """ try: - config_obj = import_string(config_path) + config_obj = import_string(_normalize_config_path(config_path)) except ImportError as e: msg = f"Failed to import config from path '{config_path}': {e}" raise ConfigResolverError(msg) from e @@ -193,7 +218,7 @@ def resolve_config_sync( Resolved config instance or list of config instances. """ try: - config_obj = import_string(config_path) + config_obj = import_string(_normalize_config_path(config_path)) except ImportError as e: msg = f"Failed to import config from path '{config_path}': {e}" raise ConfigResolverError(msg) from e @@ -228,6 +253,9 @@ def _validate_config_result( Raises: ConfigResolverError: If config result is invalid. """ + if isinstance(config_result, ModuleType): + raise ConfigResolverError(_describe_module_reference(config_result, config_path)) + if config_result is None: msg = f"Config '{config_path}' resolved to None. Expected config instance or list of configs." raise ConfigResolverError(msg) @@ -242,13 +270,85 @@ def _validate_config_result( msg = f"Config '{config_path}' returned invalid config at index {i}. Expected database config instance." raise ConfigResolverError(msg) - return cast("list[AsyncDatabaseConfig[Any, Any, Any] | SyncDatabaseConfig[Any, Any, Any]]", list(config_result)) # pyright: ignore + return cast( + "list[AsyncDatabaseConfig[Any, Any, Any] | SyncDatabaseConfig[Any, Any, Any]]", + [_unwrap_nested_config(config) for config in config_result], # pyright: ignore + ) if not _is_valid_config(config_result): msg = f"Config '{config_path}' returned invalid type '{type(config_result).__name__}'. Expected database config instance or list." raise ConfigResolverError(msg) - return cast("AsyncDatabaseConfig[Any, Any, Any] | SyncDatabaseConfig[Any, Any, Any]", config_result) + return cast( + "AsyncDatabaseConfig[Any, Any, Any] | SyncDatabaseConfig[Any, Any, Any]", _unwrap_nested_config(config_result) + ) + + +def _is_direct_config(config: Any) -> bool: + """Check whether an object is itself a database config rather than a wrapper. + + Args: + config: Object to inspect. + + Returns: + True if the object carries migration and connection configuration itself. + """ + if isinstance(config, type) or not has_migration_config(config) or config.migration_config is None: + return False + return has_connection_config(config) or has_database_url_and_bind_key(config) + + +def _unwrap_nested_config(config: Any) -> Any: + """Return the database config held by a wrapper object. + + Args: + config: Resolved object, either a config or a wrapper exposing ``.config``. + + Returns: + The nested config when the object only wraps one, otherwise the object itself. + """ + if _is_direct_config(config): + return config + if has_config_attribute(config) and has_migration_config(config.config): + return config.config + return config + + +def _describe_module_reference(module: "ModuleType", config_path: str) -> str: + """Build an actionable error message for a config path that names a module. + + Args: + module: Module the config path resolved to. + config_path: Original config path supplied by the user. + + Returns: + Error message naming the configurations the module exports, when it has any. + """ + candidates = sorted( + name + for name, value in vars(module).items() + if not name.startswith("_") + and ( + _is_valid_config(value) + or ( + isinstance(value, Sequence) + and not isinstance(value, str) + and bool(value) + and all(_is_valid_config(item) for item in value) + ) + ) + ) + if candidates: + examples = ", ".join(f"'{config_path}:{name}'" for name in candidates) + return ( + f"Config '{config_path}' names a module, not a database configuration. " + f"Point at the configuration itself, for example {examples}." + ) + return ( + f"Config '{config_path}' names a module that exports no database configuration. " + "Point at a config instance, a list of configs, or a factory returning them, " + "using 'module:attribute' or 'module.attribute'." + ) def _is_valid_config(config: Any) -> bool: diff --git a/sqlspec/utils/module_loader.py b/sqlspec/utils/module_loader.py index dbcbfcabe..105f40f59 100644 --- a/sqlspec/utils/module_loader.py +++ b/sqlspec/utils/module_loader.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, TypeVar, cast -from sqlspec.exceptions import MissingDependencyError +from sqlspec.exceptions import MissingDependencyError, SQLSpecError if TYPE_CHECKING: from types import ModuleType @@ -274,6 +274,11 @@ def module_to_os_path(dotted_path: str = "app") -> "Path": def import_string(dotted_path: str) -> "Any": """Import a module or attribute from a dotted path string. + Failures are reported as ``ImportError``, except for SQLSpec errors raised by + the imported module itself, which propagate unchanged so their own message + survives. ``MissingDependencyError`` is reported as ``ImportError`` because it + describes an import failure. + Args: dotted_path: The path of the module to import. @@ -314,6 +319,10 @@ def import_string(dotted_path: str) -> "Any": for attr in attrs: obj = _resolve_import_attr(obj, attr, module, dotted_path) + except MissingDependencyError as e: + _raise_import_error(f"Could not import '{dotted_path}': {e}", e) + except SQLSpecError: + raise except Exception as e: # pylint: disable=broad-exception-caught _raise_import_error(f"Could not import '{dotted_path}': {e}", e) return obj diff --git a/tests/unit/cli/test_config_loading.py b/tests/unit/cli/test_config_loading.py index a6ab12b85..5f9e8a626 100644 --- a/tests/unit/cli/test_config_loading.py +++ b/tests/unit/cli/test_config_loading.py @@ -151,3 +151,64 @@ def test_show_config_with_path_object( assert "path_test" in result.output assert "custom_migrations" in result.output assert "Migration Enabled" in result.output or "migrations enabled" in result.output + + +def test_module_reference_reports_its_config_attributes( + tmp_path: Path, cleanup_test_modules: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """Pointing --config at a module names the module:attribute reference to use.""" + runner = CliRunner() + + config_module = """ +from sqlspec.adapters.sqlite.config import SqliteConfig + +database_config = SqliteConfig( + bind_key="app", + connection_config={"database": ":memory:"}, + migration_config={"script_location": "migrations"}, +) +""" + module_name = _create_module(tmp_path, config_module) + monkeypatch.chdir(tmp_path) + + result = runner.invoke(add_migration_commands(), ["--config", module_name, "show-config"]) + + assert result.exit_code == 1 + normalized = " ".join(result.output.split()) + assert "names a module, not a database configuration" in normalized + assert f"{module_name}:database_config" in normalized + + +def test_unknown_migration_config_key_is_reported_without_a_traceback( + tmp_path: Path, cleanup_test_modules: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """A misspelled migration_config key surfaces as a CLI error, not a stack trace.""" + runner = CliRunner() + + config_module = """ +from sqlspec.adapters.sqlite.config import SqliteConfig + +database_config = SqliteConfig( + bind_key="app", + connection_config={"database": ":memory:"}, + migration_config={"script_location": "migrations", "version_table": "_schema_versions"}, +) +""" + module_name = _create_module(tmp_path, config_module) + monkeypatch.chdir(tmp_path) + + result = runner.invoke(add_migration_commands(), ["--config", f"{module_name}:database_config", "show-config"]) + + assert result.exit_code == 1 + assert "Did you mean 'version_table_name'?" in " ".join(result.output.split()) + assert "Traceback" not in result.output + + +def test_missing_config_help_shows_the_pyproject_section_name() -> None: + """The pyproject hint renders literally instead of being consumed as console markup.""" + runner = CliRunner() + + result = runner.invoke(add_migration_commands(), ["show-config"], env={"SQLSPEC_CONFIG": ""}) + + assert result.exit_code == 1 + assert "[tool.sqlspec]" in result.output diff --git a/tests/unit/config/test_migration_config_validation.py b/tests/unit/config/test_migration_config_validation.py new file mode 100644 index 000000000..5014afbdd --- /dev/null +++ b/tests/unit/config/test_migration_config_validation.py @@ -0,0 +1,80 @@ +"""Regression tests for ``migration_config`` key validation. + +``migration_config`` is a ``TypedDict``, so a misspelled key was accepted at +runtime and silently ignored, leaving the corresponding setting at its default. +These tests pin the validate-and-raise behavior and the suggestion text. +""" + +import pytest + +from sqlspec.adapters.sqlite import SqliteConfig +from sqlspec.config import MIGRATION_CONFIG_KEYS, MigrationConfig, validate_migration_config_keys +from sqlspec.exceptions import ImproperConfigurationError + + +def test_known_keys_cover_every_typed_dict_field() -> None: + """Every declared MigrationConfig field is accepted by the validator.""" + assert MIGRATION_CONFIG_KEYS == MigrationConfig.__required_keys__ | MigrationConfig.__optional_keys__ + validate_migration_config_keys(dict.fromkeys(MIGRATION_CONFIG_KEYS, None)) + + +def test_author_key_is_declared() -> None: + """The author key is read when generating migrations, so it must be declared.""" + assert "author" in MIGRATION_CONFIG_KEYS + + +def test_unknown_key_raises_with_suggestion() -> None: + """A near-miss key reports the intended field name.""" + with pytest.raises(ImproperConfigurationError) as exc_info: + validate_migration_config_keys({"version_table": "_schema_versions"}) + + message = str(exc_info.value) + assert "Unknown migration_config key 'version_table'" in message + assert "Did you mean 'version_table_name'?" in message + + +def test_unrecognizable_key_lists_valid_keys_without_a_suggestion() -> None: + """A key with no close match still reports the accepted set.""" + with pytest.raises(ImproperConfigurationError) as exc_info: + validate_migration_config_keys({"totally_unrelated": 1}) + + message = str(exc_info.value) + assert "Did you mean" not in message + assert "script_location" in message + + +def test_every_unknown_key_is_reported() -> None: + """Reporting covers all offending keys, not just the first.""" + with pytest.raises(ImproperConfigurationError) as exc_info: + validate_migration_config_keys({"version_table": "a", "scriptlocation": "b"}) + + message = str(exc_info.value) + assert "'version_table'" in message + assert "'scriptlocation'" in message + + +def test_config_construction_rejects_unknown_key() -> None: + """The reporter's original config fails at construction instead of silently ignoring the key.""" + with pytest.raises(ImproperConfigurationError, match="version_table_name"): + SqliteConfig( + connection_config={"database": ":memory:"}, + migration_config={"script_location": "migrations", "version_table": "_schema_versions"}, + ) + + +def test_post_construction_assignment_is_validated() -> None: + """Assigning migration_config after construction runs the same check.""" + config = SqliteConfig(connection_config={"database": ":memory:"}) + + with pytest.raises(ImproperConfigurationError, match="version_table_name"): + config.set_migration_config({"version_table": "_schema_versions"}) + + +def test_valid_config_is_unchanged() -> None: + """A correctly spelled configuration is stored as provided.""" + config = SqliteConfig( + connection_config={"database": ":memory:"}, + migration_config={"script_location": "migrations", "version_table_name": "_schema_versions"}, + ) + + assert config.migration_config["version_table_name"] == "_schema_versions" diff --git a/tests/unit/config/test_resolver.py b/tests/unit/config/test_resolver.py index 93d33bb44..ebfd8b61b 100644 --- a/tests/unit/config/test_resolver.py +++ b/tests/unit/config/test_resolver.py @@ -1,13 +1,15 @@ """Tests for configuration resolver functionality.""" +import uuid from pathlib import Path +from types import ModuleType from typing import Any from unittest.mock import Mock, NonCallableMock, patch import pytest from sqlspec.adapters.sqlite.config import SqliteConfig -from sqlspec.exceptions import ConfigResolverError +from sqlspec.exceptions import ConfigResolverError, ImproperConfigurationError from sqlspec.migrations.commands import SyncMigrationCommands from sqlspec.utils.config_tools import _is_valid_config, resolve_config_async, resolve_config_sync @@ -39,6 +41,73 @@ async def test_resolve_direct_config_instance() -> None: assert hasattr(result, "migration_config") +async def test_resolve_config_async_accepts_colon_path() -> None: + """Test resolving a config from a module:attribute path.""" + mock_config = _create_mock_config() + with patch("sqlspec.utils.config_tools.import_string", return_value=mock_config) as import_mock: + result = await resolve_config_async("myapp.config:database_config") + + assert result is mock_config + import_mock.assert_called_once_with("myapp.config.database_config") + + +def test_resolve_config_sync_accepts_colon_path() -> None: + """Test resolving a config from a module:attribute path.""" + mock_config = _create_mock_config() + with patch("sqlspec.utils.config_tools.import_string", return_value=mock_config) as import_mock: + result = resolve_config_sync("myapp.config:database_config") + + assert result is mock_config + import_mock.assert_called_once_with("myapp.config.database_config") + + +@pytest.mark.parametrize("config_path", ["myapp:config:extra", ":database_config", "myapp.config:"]) +def test_resolve_config_rejects_malformed_reference(config_path: str) -> None: + """Test that a reference using ':' incorrectly reports the accepted syntax.""" + with pytest.raises(ConfigResolverError, match="is not a valid reference"): + resolve_config_sync(config_path) + + +def test_resolve_config_rejects_module_and_names_its_configs() -> None: + """Test that pointing at a module reports the module:attribute references it exports.""" + module = ModuleType("myapp.database") + module.database_config = _create_mock_config() # type: ignore[attr-defined] + module.other_config = _create_mock_config(bind_key="other") # type: ignore[attr-defined] + module.not_a_config = "sqlite:///test.db" # type: ignore[attr-defined] + + with patch("sqlspec.utils.config_tools.import_string", return_value=module): + with pytest.raises(ConfigResolverError) as exc_info: + resolve_config_sync("myapp.database") + + message = str(exc_info.value) + assert "names a module, not a database configuration" in message + assert "'myapp.database:database_config'" in message + assert "'myapp.database:other_config'" in message + assert "not_a_config" not in message + + +def test_resolve_config_rejects_module_without_configs() -> None: + """Test that a module exporting no config explains what to point at instead.""" + module = ModuleType("myapp.empty") + + with patch("sqlspec.utils.config_tools.import_string", return_value=module): + with pytest.raises(ConfigResolverError, match="exports no database configuration"): + resolve_config_sync("myapp.empty") + + +def test_resolve_config_unwraps_nested_config_holder() -> None: + """Test that a wrapper exposing .config resolves to the config it holds.""" + nested = _create_mock_config() + + class _Holder: + config = nested + + with patch("sqlspec.utils.config_tools.import_string", return_value=_Holder()): + result = resolve_config_sync("myapp.config.plugin") + + assert result is nested + + async def test_resolve_config_list() -> None: """Test resolving a list of config instances.""" mock_config1 = _create_mock_config(database_url="sqlite:///test1.db", bind_key="test1") @@ -285,3 +354,35 @@ def test_assert_guards_default_serializer_raises_runtime_error_if_fallback_does_ monkeypatch.setattr(json_module, "StandardLibSerializer", lambda: None) with pytest.raises(RuntimeError, match="No JSON serializer available"): json_module.get_default_serializer() + + +def test_import_string_preserves_sqlspec_errors_from_the_imported_module( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A SQLSpec error raised while importing a config module keeps its type and message.""" + from sqlspec.utils.module_loader import import_string + + module_name = f"resolver_error_module_{uuid.uuid4().hex}" + (tmp_path / f"{module_name}.py").write_text( + "from sqlspec.exceptions import ImproperConfigurationError\n\nraise ImproperConfigurationError('bad key')\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + + with pytest.raises(ImproperConfigurationError, match="bad key"): + import_string(f"{module_name}.database_config") + + +def test_import_string_reports_missing_dependency_as_import_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A missing dependency stays an ImportError so existing handlers keep working.""" + from sqlspec.utils.module_loader import import_string + + module_name = f"resolver_missing_dep_{uuid.uuid4().hex}" + (tmp_path / f"{module_name}.py").write_text( + "from sqlspec.exceptions import MissingDependencyError\n\nraise MissingDependencyError('somepkg')\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + + with pytest.raises(ImportError): + import_string(f"{module_name}.database_config")