Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

datasette.yaml plugin support #2183

Merged
merged 8 commits into from
Sep 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 38 additions & 10 deletions datasette/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ def __init__(
for key in config_settings:
if key not in DEFAULT_SETTINGS:
raise StartupError("Invalid setting '{}' in datasette.json".format(key))

self.config = config
# CLI settings should overwrite datasette.json settings
self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {}))
self.renderers = {} # File extension -> (renderer, can_render) functions
Expand Down Expand Up @@ -674,15 +674,43 @@ def get_internal_database(self):

def plugin_config(self, plugin_name, database=None, table=None, fallback=True):
"""Return config for plugin, falling back from specified database/table"""
plugins = self.metadata(
"plugins", database=database, table=table, fallback=fallback
)
if plugins is None:
return None
plugin_config = plugins.get(plugin_name)
# Resolve any $file and $env keys
plugin_config = resolve_env_secrets(plugin_config, os.environ)
return plugin_config
if database is None and table is None:
config = self._plugin_config_top(plugin_name)
else:
config = self._plugin_config_nested(plugin_name, database, table, fallback)

return resolve_env_secrets(config, os.environ)

def _plugin_config_top(self, plugin_name):
"""Returns any top-level plugin configuration for the specified plugin."""
return ((self.config or {}).get("plugins") or {}).get(plugin_name)

def _plugin_config_nested(self, plugin_name, database, table=None, fallback=True):
"""Returns any database or table-level plugin configuration for the specified plugin."""
db_config = ((self.config or {}).get("databases") or {}).get(database)

# if there's no db-level configuration, then return early, falling back to top-level if needed
if not db_config:
return self._plugin_config_top(plugin_name) if fallback else None

db_plugin_config = (db_config.get("plugins") or {}).get(plugin_name)

if table:
table_plugin_config = (
((db_config.get("tables") or {}).get(table) or {}).get("plugins") or {}
).get(plugin_name)

# fallback to db_config or top-level config, in that order, if needed
if table_plugin_config is None and fallback:
return db_plugin_config or self._plugin_config_top(plugin_name)

return table_plugin_config

# fallback to top-level if needed
if db_plugin_config is None and fallback:
self._plugin_config_top(plugin_name)

return db_plugin_config

def app_css_hash(self):
if not hasattr(self, "_app_css_hash"):
Expand Down
97 changes: 94 additions & 3 deletions docs/configuration.rst
Original file line number Diff line number Diff line change
@@ -1,10 +1,101 @@
.. _configuration:

Configuration
========
=============

Datasette offers many way to configure your Datasette instances: server settings, plugin configuration, authentication, and more.
Datasette offers several ways to configure your Datasette instances: server settings, plugin configuration, authentication, and more.

To facilitate this, You can provide a `datasette.yaml` configuration file to datasette with the ``--config``/ ``-c`` flag:
To facilitate this, You can provide a ``datasette.yaml`` configuration file to datasette with the ``--config``/ ``-c`` flag:

.. code-block:: bash

datasette mydatabase.db --config datasette.yaml

.. _configuration_reference:

``datasette.yaml`` reference
----------------------------

Here's a full example of all the valid configuration options that can exist inside ``datasette.yaml``.

.. tab:: YAML

.. code-block:: yaml

# Datasette settings block
settings:
default_page_size: 50
sql_time_limit_ms: 3500
max_returned_rows: 2000

# top-level plugin configuration
plugins:
datasette-my-plugin:
key: valueA

# Database and table-level configuration
databases:
your_db_name:
# plugin configuration for the your_db_name database
plugins:
datasette-my-plugin:
key: valueA
tables:
your_table_name:
# plugin configuration for the your_table_name table
# inside your_db_name database
plugins:
datasette-my-plugin:
key: valueB

.. _configuration_reference_settings:
Settings configuration
~~~~~~~~~~~~~~~~~~~~~~

:ref:`settings` can be configured in ``datasette.yaml`` with the ``settings`` key.

.. tab:: YAML

.. code-block:: yaml

# inside datasette.yaml
settings:
default_allow_sql: off
default_page_size: 50


.. _configuration_reference_plugins:
Plugin configuration
~~~~~~~~~~~~~~~~~~~~

Configuration for plugins can be defined inside ``datasette.yaml``. For top-level plugin configuration, use the ``plugins`` key.

.. tab:: YAML

.. code-block:: yaml

# inside datasette.yaml
plugins:
datasette-my-plugin:
key: my_value

For database level or table level plugin configuration, nest it under the appropriate place under ``databases``.

.. tab:: YAML

.. code-block:: yaml

# inside datasette.yaml
databases:
my_database:
# plugin configuration for the my_database database
plugins:
datasette-my-plugin:
key: my_value
my_other_database:
tables:
my_table:
# plugin configuration for the my_table table inside the my_other_database database
plugins:
datasette-my-plugin:
key: my_value
1 change: 1 addition & 0 deletions docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ Contents

getting_started
installation
configuration
ecosystem
cli-reference
pages
Expand Down
2 changes: 1 addition & 1 deletion docs/internals.rst
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ The dictionary keys are the permission names - e.g. ``view-instance`` - and the
``table`` - None or string
The table the user is interacting with.

This method lets you read plugin configuration values that were set in ``metadata.json``. See :ref:`writing_plugins_configuration` for full details of how this method should be used.
This method lets you read plugin configuration values that were set in ``datasette.yaml``. See :ref:`writing_plugins_configuration` for full details of how this method should be used.

The return value will be the value from the configuration file - usually a dictionary.

Expand Down
2 changes: 1 addition & 1 deletion docs/plugin_hooks.rst
Original file line number Diff line number Diff line change
Expand Up @@ -909,7 +909,7 @@ Potential use-cases:

* Run some initialization code for the plugin
* Create database tables that a plugin needs on startup
* Validate the metadata configuration for a plugin on startup, and raise an error if it is invalid
* Validate the configuration for a plugin on startup, and raise an error if it is invalid

.. note::

Expand Down
7 changes: 2 additions & 5 deletions docs/writing_plugins.rst
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ This will return the ``{"latitude_column": "lat", "longitude_column": "lng"}`` i

If there is no configuration for that plugin, the method will return ``None``.

If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option like so:
If it cannot find the requested configuration at the table layer, it will fall back to the database layer and then the root layer. For example, a user may have set the plugin configuration option inside ``datasette.yaml`` like so:

.. [[[cog
from metadata_doc import metadata_example
Expand Down Expand Up @@ -234,11 +234,10 @@ If it cannot find the requested configuration at the table layer, it will fall b

In this case, the above code would return that configuration for ANY table within the ``sf-trees`` database.

The plugin configuration could also be set at the top level of ``metadata.yaml``:
The plugin configuration could also be set at the top level of ``datasette.yaml``:

.. [[[cog
metadata_example(cog, {
"title": "This is the top-level title in metadata.json",
"plugins": {
"datasette-cluster-map": {
"latitude_column": "xlat",
Expand All @@ -252,7 +251,6 @@ The plugin configuration could also be set at the top level of ``metadata.yaml``

.. code-block:: yaml

title: This is the top-level title in metadata.json
plugins:
datasette-cluster-map:
latitude_column: xlat
Expand All @@ -264,7 +262,6 @@ The plugin configuration could also be set at the top level of ``metadata.yaml``
.. code-block:: json

{
"title": "This is the top-level title in metadata.json",
"plugins": {
"datasette-cluster-map": {
"latitude_column": "xlat",
Expand Down
3 changes: 2 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,15 @@ def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs):
@pytest_asyncio.fixture
async def ds_client():
from datasette.app import Datasette
from .fixtures import METADATA, PLUGINS_DIR
from .fixtures import CONFIG, METADATA, PLUGINS_DIR

global _ds_client
if _ds_client is not None:
return _ds_client

ds = Datasette(
metadata=METADATA,
config=CONFIG,
plugins_dir=PLUGINS_DIR,
settings={
"default_page_size": 50,
Expand Down
50 changes: 35 additions & 15 deletions tests/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ def make_app_client(
inspect_data=None,
static_mounts=None,
template_dir=None,
config=None,
metadata=None,
crossdb=False,
):
Expand Down Expand Up @@ -158,6 +159,7 @@ def make_app_client(
memory=memory,
cors=cors,
metadata=metadata or METADATA,
config=config or CONFIG,
plugins_dir=PLUGINS_DIR,
settings=settings,
inspect_data=inspect_data,
Expand Down Expand Up @@ -296,16 +298,7 @@ def generate_sortable_rows(num):
}


METADATA = {
"title": "Datasette Fixtures",
"description_html": 'An example SQLite database demonstrating Datasette. <a href="/login-as-root">Sign in as root user</a>',
"license": "Apache License 2.0",
"license_url": "https://github.com/simonw/datasette/blob/main/LICENSE",
"source": "tests/fixtures.py",
"source_url": "https://github.com/simonw/datasette/blob/main/tests/fixtures.py",
"about": "About Datasette",
"about_url": "https://github.com/simonw/datasette",
"extra_css_urls": ["/static/extra-css-urls.css"],
CONFIG = {
"plugins": {
"name-of-plugin": {"depth": "root"},
"env-plugin": {"foo": {"$env": "FOO_ENV"}},
Expand All @@ -314,27 +307,49 @@ def generate_sortable_rows(num):
},
"databases": {
"fixtures": {
"description": "Test tables description",
"plugins": {"name-of-plugin": {"depth": "database"}},
"tables": {
"simple_primary_key": {
"description_html": "Simple <em>primary</em> key",
"title": "This <em>HTML</em> is escaped",
"plugins": {
"name-of-plugin": {
"depth": "table",
"special": "this-is-simple_primary_key",
}
},
},
"sortable": {
"plugins": {"name-of-plugin": {"depth": "table"}},
},
},
}
},
}

METADATA = {
"title": "Datasette Fixtures",
"description_html": 'An example SQLite database demonstrating Datasette. <a href="/login-as-root">Sign in as root user</a>',
"license": "Apache License 2.0",
"license_url": "https://github.com/simonw/datasette/blob/main/LICENSE",
"source": "tests/fixtures.py",
"source_url": "https://github.com/simonw/datasette/blob/main/tests/fixtures.py",
"about": "About Datasette",
"about_url": "https://github.com/simonw/datasette",
"extra_css_urls": ["/static/extra-css-urls.css"],
"databases": {
"fixtures": {
"description": "Test tables description",
"tables": {
"simple_primary_key": {
"description_html": "Simple <em>primary</em> key",
"title": "This <em>HTML</em> is escaped",
},
"sortable": {
"sortable_columns": [
"sortable",
"sortable_with_nulls",
"sortable_with_nulls_2",
"text",
],
"plugins": {"name-of-plugin": {"depth": "table"}},
},
"no_primary_key": {"sortable_columns": [], "hidden": True},
"units": {"units": {"distance": "m", "frequency": "Hz"}},
Expand Down Expand Up @@ -768,6 +783,7 @@ def assert_permissions_checked(datasette, actions):
type=click.Path(file_okay=True, dir_okay=False),
)
@click.argument("metadata", required=False)
@click.argument("config", required=False)
@click.argument(
"plugins_path", type=click.Path(file_okay=False, dir_okay=True), required=False
)
Expand All @@ -782,7 +798,7 @@ def assert_permissions_checked(datasette, actions):
type=click.Path(file_okay=True, dir_okay=False),
help="Write out second test DB to this file",
)
def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename):
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like that you updated this too. Trying that myself:

python tests/fixtures.py --help
Usage: fixtures.py [OPTIONS] [DB_FILENAME] [METADATA] [CONFIG] [PLUGINS_PATH]

  Write out the fixtures database used by Datasette's test suite

Options:
  --recreate                Delete and recreate database if it exists
  --extra-db-filename FILE  Write out second test DB to this file
  --help                    Show this message and exit.
python tests/fixtures.py /tmp/demo/fixtures.db /tmp/demo/metadata.json /tmp/demo/config.json /tmp/demo/plugins
Test tables written to /tmp/demo/fixtures.db
- metadata written to /tmp/demo/metadata.json
- config written to /tmp/demo/config.json
  Wrote plugin: /tmp/demo/plugins/register_output_renderer.py
  Wrote plugin: /tmp/demo/plugins/view_name.py
  Wrote plugin: /tmp/demo/plugins/my_plugin.py
  Wrote plugin: /tmp/demo/plugins/messages_output_renderer.py
  Wrote plugin: /tmp/demo/plugins/sleep_sql_function.py
  Wrote plugin: /tmp/demo/plugins/my_plugin_2.py
find /tmp/demo
/tmp/demo
/tmp/demo/plugins
/tmp/demo/plugins/register_output_renderer.py
/tmp/demo/plugins/view_name.py
/tmp/demo/plugins/my_plugin.py
/tmp/demo/plugins/messages_output_renderer.py
/tmp/demo/plugins/sleep_sql_function.py
/tmp/demo/plugins/my_plugin_2.py
/tmp/demo/config.json
/tmp/demo/fixtures.db
/tmp/demo/metadata.json
cat /tmp/demo/config.json
{
    "plugins": {
        "name-of-plugin": {
            "depth": "root"
        },
        "env-plugin": {
            "foo": {
                "$env": "FOO_ENV"
            }
        },
        "env-plugin-list": [
            {
                "in_a_list": {
                    "$env": "FOO_ENV"
                }
            }
        ],
        "file-plugin": {
            "foo": {
                "$file": "/var/folders/x6/31xf1vxj0nn9mxqq8z0mmcfw0000gn/T/plugin-secret"
            }
        }
    },
    "databases": {
        "fixtures": {
            "plugins": {
                "name-of-plugin": {
                    "depth": "database"
                }
            },
            "tables": {
                "simple_primary_key": {
                    "plugins": {
                        "name-of-plugin": {
                            "depth": "table",
                            "special": "this-is-simple_primary_key"
                        }
                    }
                },
                "sortable": {
                    "plugins": {
                        "name-of-plugin": {
                            "depth": "table"
                        }
                    }
                }
            }
        }
    }
}

def cli(db_filename, config, metadata, plugins_path, recreate, extra_db_filename):
"""Write out the fixtures database used by Datasette's test suite"""
if metadata and not metadata.endswith(".json"):
raise click.ClickException("Metadata should end with .json")
Expand All @@ -805,6 +821,10 @@ def cli(db_filename, metadata, plugins_path, recreate, extra_db_filename):
with open(metadata, "w") as fp:
fp.write(json.dumps(METADATA, indent=4))
print(f"- metadata written to {metadata}")
if config:
with open(config, "w") as fp:
fp.write(json.dumps(CONFIG, indent=4))
print(f"- config written to {config}")
if plugins_path:
path = pathlib.Path(plugins_path)
if not path.exists():
Expand Down
Loading
Loading