From 4fdbeb49245d4d314e4e599030284265d3ca8980 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 22 Jul 2019 18:00:07 -0700 Subject: [PATCH 1/6] Handle databases with spaces in their names --- datasette/views/base.py | 2 ++ tests/fixtures.py | 5 +++++ tests/test_api.py | 6 ++++++ 3 files changed, 13 insertions(+) diff --git a/datasette/views/base.py b/datasette/views/base.py index 2c14be574d..4d783a0799 100644 --- a/datasette/views/base.py +++ b/datasette/views/base.py @@ -203,6 +203,8 @@ async def resolve_db_name(self, request, db_name, **kwargs): hash = None else: name = db_name + if "%" in name: + name = urllib.parse.unquote_plus(name) # Verify the hash try: db = self.ds.databases[name] diff --git a/tests/fixtures.py b/tests/fixtures.py index dac28dc04d..4ba816e6d8 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -215,6 +215,11 @@ def app_client_with_dot(): yield from make_app_client(filename="fixtures.dot.db") +@pytest.fixture(scope="session") +def app_client_with_space(): + yield from make_app_client(filename="fixtures with space.db") + + @pytest.fixture(scope="session") def app_client_with_cors(): yield from make_app_client(cors=True) diff --git a/tests/test_api.py b/tests/test_api.py index cc00b780fa..163e2ec746 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -9,6 +9,7 @@ app_client_two_attached_databases_one_immutable, app_client_with_cors, app_client_with_dot, + app_client_with_space, generate_compound_rows, generate_sortable_rows, make_app_client, @@ -544,6 +545,11 @@ def test_database_page_for_database_with_dot_in_name(app_client_with_dot): assert 200 == response.status +def test_database_page_for_database_with_space_in_name(app_client_with_space): + response = app_client_with_space.get("/fixtures%20with%20space.json") + assert 200 == response.status + + def test_custom_sql(app_client): response = app_client.get( "/fixtures.json?sql=select+content+from+simple_primary_key&_shape=objects" From 9ef0cf6d693c50ed417c3b2535f7102ecafc0e0e Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Jul 2019 16:07:44 +0300 Subject: [PATCH 2/6] Refactored connection logic to database.connect() --- datasette/app.py | 15 +-------------- datasette/database.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 1a41c1c6f4..501d14674e 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -470,20 +470,7 @@ async def execute_against_connection_in_thread(self, db_name, fn): def in_thread(): conn = getattr(connections, db_name, None) if not conn: - db = self.databases[db_name] - if db.is_memory: - conn = sqlite3.connect(":memory:") - else: - # mode=ro or immutable=1? - if db.is_mutable: - qs = "mode=ro" - else: - qs = "immutable=1" - conn = sqlite3.connect( - "file:{}?{}".format(db.path, qs), - uri=True, - check_same_thread=False, - ) + conn = self.databases[db_name].connect() self.prepare_connection(conn) setattr(connections, db_name, conn) return fn(conn) diff --git a/datasette/database.py b/datasette/database.py index e491577093..7e6f7245c7 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -33,6 +33,18 @@ def __init__(self, ds, path=None, is_mutable=False, is_memory=False): for key, value in self.ds.inspect_data[self.name]["tables"].items() } + def connect(self): + if self.is_memory: + return sqlite3.connect(":memory:") + # mode=ro or immutable=1? + if self.is_mutable: + qs = "mode=ro" + else: + qs = "immutable=1" + return sqlite3.connect( + "file:{}?{}".format(self.path, qs), uri=True, check_same_thread=False + ) + @property def size(self): if self.is_memory: From 894c424b90f03963dc09f1c820f93add2b45eec5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Jul 2019 17:09:13 +0300 Subject: [PATCH 3/6] New plugin hook: extra_serve_options() --- datasette/app.py | 2 ++ datasette/cli.py | 12 +++++++++++- datasette/hookspecs.py | 5 +++++ docs/plugins.rst | 42 ++++++++++++++++++++++++++++++++++++++++++ tests/fixtures.py | 5 ++++- tests/test_plugins.py | 14 ++++++++++++++ 6 files changed, 78 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 501d14674e..032c244a0d 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -151,6 +151,7 @@ def __init__( memory=False, config=None, version_note=None, + plugin_extra_options=None, ): immutables = immutables or [] self.files = tuple(files) + tuple(immutables) @@ -159,6 +160,7 @@ def __init__( self.files = [MEMORY] elif memory: self.files = (MEMORY,) + self.files + self.plugin_extra_options = plugin_extra_options or {} self.databases = {} self.inspect_data = inspect_data for file in self.files: diff --git a/datasette/cli.py b/datasette/cli.py index 181b281c7c..68104aee3f 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -186,7 +186,7 @@ def package( install, spatialite, version_note, - **extra_metadata + **extra_metadata, ): "Package specified SQLite files into a new datasette Docker container" if not shutil.which("docker"): @@ -220,6 +220,13 @@ def package( call(args) +def extra_serve_options(serve): + for options in pm.hook.extra_serve_options(): + for option in reversed(options): + serve = option(serve) + return serve + + @cli.command() @click.argument("files", type=click.Path(exists=True), nargs=-1) @click.option( @@ -286,6 +293,7 @@ def package( ) @click.option("--version-note", help="Additional note to show on /-/versions") @click.option("--help-config", is_flag=True, help="Show available config options") +@extra_serve_options def serve( files, immutable, @@ -304,6 +312,7 @@ def serve( config, version_note, help_config, + **plugin_extra_options, ): """Serve up specified SQLite database files with a web UI""" if help_config: @@ -350,6 +359,7 @@ def serve( config=dict(config), memory=memory, version_note=version_note, + plugin_extra_options=plugin_extra_options, ) # Run async sanity checks - but only if we're not under pytest asyncio.get_event_loop().run_until_complete(ds.run_sanity_checks()) diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 3c6726b7f5..bca4799027 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -58,3 +58,8 @@ def register_output_renderer(datasette): @hookspec def register_facet_classes(): "Register Facet subclasses" + + +@hookspec +def extra_serve_options(): + "Return list of extra click.option decorators to be applied to 'datasette serve'" diff --git a/docs/plugins.rst b/docs/plugins.rst index 1d4f1e1af9..ead0c56980 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -812,3 +812,45 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att await app(scope, recieve, wrapped_send) return add_x_databases_header return wrap_with_databases_header + +.. _plugin_hook_extra_serve_options: + +extra_serve_options() +~~~~~~~~~~~~~~~~~~~~~ + +Add extra Click options to the ``datasette serve`` command. Options you add here will be displayed in ``datasette serve --help`` and their values will be available to your plugin anywhere it can access the ``datasette`` object by reading from ``datasette.plugin_extra_options``. + +.. code-block:: python + + from datasette import hookimpl + import click + + @hookimpl + def extra_serve_options(): + return [ + click.option( + "--my-plugin-paths", + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Directories to use with my-plugin", + multiple=True, + ), + click.option( + "--my-plugin-enable", + is_flag=True, + help="Enable functionality from my-plugin", + ), + ] + +Your other plugin hooks can then access these settings like so: + +.. code-block:: python + + from datasette import hookimpl + + @hookimpl + def extra_template_vars(datasette): + return { + "my_plugin_paths": datasette.plugin_extra_options.get("my_plugin_paths") or [] + } + +Be careful not to define an option which clashes with a Datasette default option, or with options provided by another plugin. For this reason we recommend using a common prefix for your plugin, as shown above. diff --git a/tests/fixtures.py b/tests/fixtures.py index dac28dc04d..5a272e12e8 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -108,6 +108,7 @@ def make_app_client( inspect_data=None, static_mounts=None, template_dir=None, + plugin_extra_options=None, ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, filename) @@ -151,6 +152,7 @@ def make_app_client( inspect_data=inspect_data, static_mounts=static_mounts, template_dir=template_dir, + plugin_extra_options=plugin_extra_options, ) ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n)))) client = TestClient(ds.app()) @@ -386,7 +388,8 @@ def extra_template_vars(template, database, table, view_name, request, datasette return { "extra_template_vars": json.dumps({ "template": template, - "scope_path": request.scope["path"] + "scope_path": request.scope["path"], + "plugin_extra_options": datasette.plugin_extra_options, }, default=lambda b: b.decode("utf8")) } """ diff --git a/tests/test_plugins.py b/tests/test_plugins.py index b1c7fd9a43..939b462715 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -203,6 +203,7 @@ def test_plugins_extra_template_vars(restore_working_directory): assert { "template": "show_json.html", "scope_path": "/-/metadata", + "plugin_extra_options": {}, } == extra_template_vars extra_template_vars_from_awaitable = json.loads( Soup(response.body, "html.parser") @@ -214,3 +215,16 @@ def test_plugins_extra_template_vars(restore_working_directory): "awaitable": True, "scope_path": "/-/metadata", } == extra_template_vars_from_awaitable + + +def test_plugin_extra_options_available_on_datasette(restore_working_directory): + for client in make_app_client( + template_dir=str(pathlib.Path(__file__).parent / "test_templates"), + plugin_extra_options={"foo": "bar"}, + ): + response = client.get("/-/metadata") + assert response.status == 200 + extra_template_vars = json.loads( + Soup(response.body, "html.parser").select("pre.extra_template_vars")[0].text + ) + assert {"foo": "bar"} == extra_template_vars["plugin_extra_options"] From a0fd07adc1fe35091291059356790af5df288e62 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Jul 2019 17:09:37 +0300 Subject: [PATCH 4/6] Fixed broken link in changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d04ae2cae6..08d3b34728 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -51,7 +51,7 @@ Two new plugins take advantage of this hook: New plugin hook: extra_template_vars ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :ref:`plugin_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github `__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 `__). +The :ref:`plugin_hook_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github `__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 `__). Secret plugin configuration options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From f4b0bc64dc5edd560153ce8eca46e8eeeede5c11 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Thu, 25 Jul 2019 17:15:51 +0300 Subject: [PATCH 5/6] Renamed plugin_extra_options to extra_serve_options --- datasette/app.py | 4 ++-- datasette/cli.py | 4 ++-- docs/plugins.rst | 4 ++-- tests/fixtures.py | 6 +++--- tests/test_plugins.py | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 032c244a0d..d80b209449 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -151,7 +151,7 @@ def __init__( memory=False, config=None, version_note=None, - plugin_extra_options=None, + extra_serve_options=None, ): immutables = immutables or [] self.files = tuple(files) + tuple(immutables) @@ -160,7 +160,7 @@ def __init__( self.files = [MEMORY] elif memory: self.files = (MEMORY,) + self.files - self.plugin_extra_options = plugin_extra_options or {} + self.extra_serve_options = extra_serve_options or {} self.databases = {} self.inspect_data = inspect_data for file in self.files: diff --git a/datasette/cli.py b/datasette/cli.py index 68104aee3f..f478d496b6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -312,7 +312,7 @@ def serve( config, version_note, help_config, - **plugin_extra_options, + **extra_serve_options, ): """Serve up specified SQLite database files with a web UI""" if help_config: @@ -359,7 +359,7 @@ def serve( config=dict(config), memory=memory, version_note=version_note, - plugin_extra_options=plugin_extra_options, + extra_serve_options=extra_serve_options, ) # Run async sanity checks - but only if we're not under pytest asyncio.get_event_loop().run_until_complete(ds.run_sanity_checks()) diff --git a/docs/plugins.rst b/docs/plugins.rst index ead0c56980..72473a39cf 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -818,7 +818,7 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att extra_serve_options() ~~~~~~~~~~~~~~~~~~~~~ -Add extra Click options to the ``datasette serve`` command. Options you add here will be displayed in ``datasette serve --help`` and their values will be available to your plugin anywhere it can access the ``datasette`` object by reading from ``datasette.plugin_extra_options``. +Add extra Click options to the ``datasette serve`` command. Options you add here will be displayed in ``datasette serve --help`` and their values will be available to your plugin anywhere it can access the ``datasette`` object by reading from ``datasette.extra_serve_options``. .. code-block:: python @@ -850,7 +850,7 @@ Your other plugin hooks can then access these settings like so: @hookimpl def extra_template_vars(datasette): return { - "my_plugin_paths": datasette.plugin_extra_options.get("my_plugin_paths") or [] + "my_plugin_paths": datasette.extra_serve_options.get("my_plugin_paths") or [] } Be careful not to define an option which clashes with a Datasette default option, or with options provided by another plugin. For this reason we recommend using a common prefix for your plugin, as shown above. diff --git a/tests/fixtures.py b/tests/fixtures.py index 5a272e12e8..272458fe4f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -108,7 +108,7 @@ def make_app_client( inspect_data=None, static_mounts=None, template_dir=None, - plugin_extra_options=None, + extra_serve_options=None, ): with tempfile.TemporaryDirectory() as tmpdir: filepath = os.path.join(tmpdir, filename) @@ -152,7 +152,7 @@ def make_app_client( inspect_data=inspect_data, static_mounts=static_mounts, template_dir=template_dir, - plugin_extra_options=plugin_extra_options, + extra_serve_options=extra_serve_options, ) ds.sqlite_functions.append(("sleep", 1, lambda n: time.sleep(float(n)))) client = TestClient(ds.app()) @@ -389,7 +389,7 @@ def extra_template_vars(template, database, table, view_name, request, datasette "extra_template_vars": json.dumps({ "template": template, "scope_path": request.scope["path"], - "plugin_extra_options": datasette.plugin_extra_options, + "extra_serve_options": datasette.extra_serve_options, }, default=lambda b: b.decode("utf8")) } """ diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 939b462715..b306963c16 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -203,7 +203,7 @@ def test_plugins_extra_template_vars(restore_working_directory): assert { "template": "show_json.html", "scope_path": "/-/metadata", - "plugin_extra_options": {}, + "extra_serve_options": {}, } == extra_template_vars extra_template_vars_from_awaitable = json.loads( Soup(response.body, "html.parser") @@ -217,14 +217,14 @@ def test_plugins_extra_template_vars(restore_working_directory): } == extra_template_vars_from_awaitable -def test_plugin_extra_options_available_on_datasette(restore_working_directory): +def test_extra_serve_options_available_on_datasette(restore_working_directory): for client in make_app_client( template_dir=str(pathlib.Path(__file__).parent / "test_templates"), - plugin_extra_options={"foo": "bar"}, + extra_serve_options={"foo": "bar"}, ): response = client.get("/-/metadata") assert response.status == 200 extra_template_vars = json.loads( Soup(response.body, "html.parser").select("pre.extra_template_vars")[0].text ) - assert {"foo": "bar"} == extra_template_vars["plugin_extra_options"] + assert {"foo": "bar"} == extra_template_vars["extra_serve_options"] From 947645d84710677ea50762016081a9fbc6b014a8 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 26 Jul 2019 13:18:19 +0300 Subject: [PATCH 6/6] First working -d based Datasette Library Refs #417 First proof-of-concept for Datasette Library. Run like this: datasette -d ~/Library Uses a new plugin hook - available_databases() BUT... I don't think this is quite the way I want to go. --- datasette/app.py | 12 +++++- datasette/database.py | 12 ++++-- datasette/hookspecs.py | 5 +++ datasette/plugins.py | 1 + datasette/serve_dir.py | 76 ++++++++++++++++++++++++++++++++++ datasette/templates/index.html | 1 + datasette/views/database.py | 1 + datasette/views/index.py | 1 + docs/datasette-serve-help.txt | 2 + docs/plugins.rst | 22 ++++++++++ tests/fixtures.py | 16 +++++++ tests/test_api.py | 48 ++++++++++++++++----- tests/test_cli.py | 3 +- tests/test_html.py | 1 + tests/test_plugins.py | 18 +++++++- 15 files changed, 201 insertions(+), 18 deletions(-) create mode 100644 datasette/serve_dir.py diff --git a/datasette/app.py b/datasette/app.py index d80b209449..2ee32dc8d5 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -161,7 +161,7 @@ def __init__( elif memory: self.files = (MEMORY,) + self.files self.extra_serve_options = extra_serve_options or {} - self.databases = {} + self._databases = {} self.inspect_data = inspect_data for file in self.files: path = file @@ -173,7 +173,7 @@ def __init__( db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory) if db.name in self.databases: raise Exception("Multiple files with same stem: {}".format(db.name)) - self.databases[db.name] = db + self._databases[db.name] = db self.cache_headers = cache_headers self.cors = cors self._metadata = metadata or {} @@ -203,6 +203,14 @@ def __init__( # Plugin already registered pass + @property + def databases(self): + databases = dict(self._databases) + # pylint: disable=no-member + for pairs in pm.hook.available_databases(datasette=self): + databases.update(pairs) + return databases + async def run_sanity_checks(self): # Only one check right now, for Spatialite for database_name, database in self.databases.items(): diff --git a/datasette/database.py b/datasette/database.py index 7e6f7245c7..06d1c2ada9 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -14,15 +14,19 @@ class Database: - def __init__(self, ds, path=None, is_mutable=False, is_memory=False): + def __init__( + self, ds, path=None, name=None, is_mutable=False, is_memory=False, comment=None + ): self.ds = ds + self._name = name self.path = path self.is_mutable = is_mutable self.is_memory = is_memory self.hash = None self.cached_size = None self.cached_table_counts = None - if not self.is_mutable: + self.comment = comment + if not self.is_mutable and path is not None: p = Path(path) self.hash = inspect_hash(p) self.cached_size = p.stat().st_size @@ -47,7 +51,7 @@ def connect(self): @property def size(self): - if self.is_memory: + if self.is_memory or self.path is None: return 0 if self.cached_size is not None: return self.cached_size @@ -83,6 +87,8 @@ def mtime_ns(self): @property def name(self): + if self._name: + return self._name if self.is_memory: return ":memory:" else: diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index bca4799027..780b7732b0 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -63,3 +63,8 @@ def register_facet_classes(): @hookspec def extra_serve_options(): "Return list of extra click.option decorators to be applied to 'datasette serve'" + + +@hookspec +def available_databases(datasette): + "Return list of (name, database) pairs to be added to the available databases" diff --git a/datasette/plugins.py b/datasette/plugins.py index bf3735dcb7..7e7556593a 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -8,6 +8,7 @@ "datasette.publish.now", "datasette.publish.cloudrun", "datasette.facets", + "datasette.serve_dir", ) pm = pluggy.PluginManager("datasette") diff --git a/datasette/serve_dir.py b/datasette/serve_dir.py new file mode 100644 index 0000000000..addeb33ad8 --- /dev/null +++ b/datasette/serve_dir.py @@ -0,0 +1,76 @@ +from datasette import hookimpl +from pathlib import Path +from .database import Database +from .utils import escape_sqlite +import click + + +@hookimpl +def extra_serve_options(): + return [ + click.option( + "-d", + "--dir", + type=click.Path(exists=True, file_okay=False, dir_okay=True), + help="Directories to scan for SQLite databases", + multiple=True, + ), + click.option( + "--scan", + is_flag=True, + help="Continually scan directories for new database files", + ), + ] + + +cached_results = None + + +@hookimpl +def available_databases(datasette): + global cached_results + if cached_results is not None: + return cached_results + i = 0 + counts = {name: 0 for name in datasette._databases} + results = [] + for directory in datasette.extra_serve_options.get("dir") or []: + for filepath in Path(directory).glob("**/*"): + if is_sqlite(filepath): + name = filepath.stem + if name in counts: + new_name = "{}_{}".format(name, counts[name] + 1) + counts[name] += 1 + name = new_name + try: + database = Database(datasette, str(filepath), comment=str(filepath)) + conn = database.connect() + result = conn.execute( + "select name from sqlite_master where type = 'table'" + ) + table_names = [r[0] for r in result] + for table_name in table_names: + conn.execute( + "PRAGMA table_info({});".format(escape_sqlite(table_name)) + ) + except Exception as e: + print("Could not open {}".format(filepath)) + print(" " + str(e)) + else: + results.append((name, database)) + + cached_results = results + return results + + +magic = b"SQLite format 3\x00" + + +def is_sqlite(path): + if not path.is_file(): + return False + try: + with open(path, "rb") as fp: + return fp.read(len(magic)) == magic + except PermissionError: + return False diff --git a/datasette/templates/index.html b/datasette/templates/index.html index b394564a84..69a3480817 100644 --- a/datasette/templates/index.html +++ b/datasette/templates/index.html @@ -11,6 +11,7 @@

{{ metadata.title or "Datasette" }}

{% for database in databases %}

{{ database.name }}

+ {% if database.comment %}

{{ database.comment }}

{% endif %}

{% if database.show_table_row_counts %}{{ "{:,}".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.tables_count and database.hidden_tables_count %}, {% endif -%} {% if database.hidden_tables_count -%} diff --git a/datasette/views/database.py b/datasette/views/database.py index 78af19c5c3..ce9498c577 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -47,6 +47,7 @@ async def data(self, request, database, hash, default_labels=False, _size=None): { "database": database, "size": db.size, + "comment": db.comment, "tables": tables, "hidden_count": len([t for t in tables if t["hidden"]]), "views": views, diff --git a/datasette/views/index.py b/datasette/views/index.py index fddb04d9d7..64877f2b4a 100644 --- a/datasette/views/index.py +++ b/datasette/views/index.py @@ -79,6 +79,7 @@ async def get(self, request, as_format): { "name": name, "hash": db.hash, + "comment": db.comment, "color": db.hash[:6] if db.hash else hashlib.md5(name.encode("utf8")).hexdigest()[:6], diff --git a/docs/datasette-serve-help.txt b/docs/datasette-serve-help.txt index 7b7c3b09e8..d2b96f1ff0 100644 --- a/docs/datasette-serve-help.txt +++ b/docs/datasette-serve-help.txt @@ -23,4 +23,6 @@ Options: datasette.readthedocs.io/en/latest/config.html --version-note TEXT Additional note to show on /-/versions --help-config Show available config options + -d, --dir DIRECTORY Directories to scan for SQLite databases + --scan Continually scan directories for new database files --help Show this message and exit. diff --git a/docs/plugins.rst b/docs/plugins.rst index 72473a39cf..4937e32ff1 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -854,3 +854,25 @@ Your other plugin hooks can then access these settings like so: } Be careful not to define an option which clashes with a Datasette default option, or with options provided by another plugin. For this reason we recommend using a common prefix for your plugin, as shown above. + +.. _plugin_hook_available_databases: + +available_databases(datasette) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Return a list of ``(name, database)`` pairs to be added to the available databases. + +``name`` should be a string. ``database`` should be a ``datasette.database.Database`` instance. + +This allows plugins to make databases available from new sources. + +.. code-block:: python + + from datasette import hookimpl + from datasette.database import Database + + @hookimpl + def available_databases(datasette): + return [ + ("hardcoded_database", Database(datasette, "/mnt/hard_coded.db")) + ] diff --git a/tests/fixtures.py b/tests/fixtures.py index 801abc43a3..d9ea947a3d 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -321,6 +321,8 @@ def generate_sortable_rows(num): PLUGIN1 = """ from datasette import hookimpl +from datasette.database import Database +from datasette.utils import sqlite3 import base64 import pint import json @@ -397,6 +399,20 @@ def extra_template_vars(template, database, table, view_name, request, datasette "extra_serve_options": datasette.extra_serve_options, }, default=lambda b: b.decode("utf8")) } + + +class SpecialDatabase(Database): + def connect(self): + db = sqlite3.connect(":memory:") + db.executescript("CREATE TABLE foo (id integer primary key, bar text)") + db.executescript("INSERT INTO foo (id, bar) VALUES (1, 'hello')") + return db + +@hookimpl +def available_databases(datasette): + return [ + ("special", SpecialDatabase(datasette, name="special")), + ] """ PLUGIN2 = """ diff --git a/tests/test_api.py b/tests/test_api.py index 163e2ec746..661ca75a2e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -24,7 +24,7 @@ def test_homepage(app_client): response = app_client.get("/.json") assert response.status == 200 assert "application/json; charset=utf-8" == response.headers["content-type"] - assert response.json.keys() == {"fixtures": 0}.keys() + assert {"fixtures", "special"} == set(response.json.keys()) d = response.json["fixtures"] assert d["name"] == "fixtures" assert d["tables_count"] == 24 @@ -518,19 +518,45 @@ def test_no_files_uses_memory_database(app_client_no_files): assert response.status == 200 assert { ":memory:": { + "name": ":memory:", "hash": None, + "comment": None, "color": "f7935d", - "hidden_table_rows_sum": 0, - "hidden_tables_count": 0, - "name": ":memory:", - "show_table_row_counts": False, "path": "/:memory:", - "table_rows_sum": 0, + "tables_and_views_truncated": [], + "tables_and_views_more": False, "tables_count": 0, + "table_rows_sum": 0, + "show_table_row_counts": False, + "hidden_table_rows_sum": 0, + "hidden_tables_count": 0, + "views_count": 0, + }, + "special": { + "name": "special", + "hash": None, + "comment": None, + "color": "0bd650", + "path": "/special", + "tables_and_views_truncated": [ + { + "name": "foo", + "columns": ["id", "bar"], + "primary_keys": ["id"], + "count": 1, + "hidden": False, + "fts_table": None, + "num_relationships_for_sorting": 0, + } + ], "tables_and_views_more": False, - "tables_and_views_truncated": [], + "tables_count": 1, + "table_rows_sum": 1, + "show_table_row_counts": True, + "hidden_table_rows_sum": 0, + "hidden_tables_count": 0, "views_count": 0, - } + }, } == response.json # Try that SQL query response = app_client_no_files.get( @@ -1170,8 +1196,10 @@ def test_unit_filters(app_client): def test_databases_json(app_client_two_attached_databases_one_immutable): response = app_client_two_attached_databases_one_immutable.get("/-/databases.json") databases = response.json - assert 2 == len(databases) - extra_database, fixtures_database = databases + assert 3 == len(databases) + by_name = {database["name"]: database for database in databases} + extra_database = by_name["extra_database"] + fixtures_database = by_name["fixtures"] assert "extra_database" == extra_database["name"] assert None == extra_database["hash"] assert True == extra_database["is_mutable"] diff --git a/tests/test_cli.py b/tests/test_cli.py index d1ab6522f0..1dab4d1f77 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,7 +9,6 @@ def test_inspect_cli(app_client): runner = CliRunner() result = runner.invoke(cli, ["inspect", "fixtures.db"]) data = json.loads(result.output) - assert ["fixtures"] == list(data.keys()) database = data["fixtures"] assert "fixtures.db" == database["file"] assert isinstance(database["hash"], str) @@ -28,7 +27,7 @@ def test_inspect_cli_writes_to_file(app_client): ) assert 0 == result.exit_code, result.output data = json.load(open("foo.json")) - assert ["fixtures"] == list(data.keys()) + assert {"fixtures", "special"} == set(data.keys()) def test_serve_with_inspect_file_prepopulates_table_counts_cache(): diff --git a/tests/test_html.py b/tests/test_html.py index 0a6df984de..4a49551f65 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -28,6 +28,7 @@ def test_homepage(app_client_two_attached_databases): assert [ {"href": "/extra_database", "text": "extra_database"}, {"href": "/fixtures", "text": "fixtures"}, + {"href": "/special", "text": "special"}, ] == [{"href": a["href"], "text": a.text.strip()} for a in soup.select("h2 a")] # The first attached database should show count text and attached tables h2 = soup.select("h2")[0] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index b306963c16..c89a466d28 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -188,7 +188,7 @@ def test_plugins_extra_body_script(app_client, path, expected_extra_body_script) def test_plugins_asgi_wrapper(app_client): response = app_client.get("/fixtures") - assert "fixtures" == response.headers["x-databases"] + assert "fixtures, special" == response.headers["x-databases"] def test_plugins_extra_template_vars(restore_working_directory): @@ -228,3 +228,19 @@ def test_extra_serve_options_available_on_datasette(restore_working_directory): Soup(response.body, "html.parser").select("pre.extra_template_vars")[0].text ) assert {"foo": "bar"} == extra_template_vars["extra_serve_options"] + + +def test_plugins_available_databases(app_client): + response = app_client.get("/-/databases.json") + assert 200 == response.status + assert { + "name": "special", + "path": None, + "size": 0, + "is_mutable": False, + "is_memory": False, + "hash": None, + } in response.json + assert [{"id": 1, "bar": "hello"}] == app_client.get( + "/special/foo.json?_shape=array" + ).json