First proof-of-concept of Datasette Library

datasette/app.py

@@ -151,6 +151,7 @@ def __init__(
         memory=False,
         config=None,
         version_note=None,
+        extra_serve_options=None,
     ):
         immutables = immutables or []
         self.files = tuple(files) + tuple(immutables)
@@ -159,7 +160,8 @@ def __init__(
             self.files = [MEMORY]
         elif memory:
             self.files = (MEMORY,) + self.files
-        self.databases = {}
+        self.extra_serve_options = extra_serve_options or {}
+        self._databases = {}
         self.inspect_data = inspect_data
         for file in self.files:
             path = file
@@ -171,7 +173,7 @@ def __init__(
             db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory)
             if db.name in self.databases:
                 raise Exception("Multiple files with same stem: {}".format(db.name))
-            self.databases[db.name] = db
+            self._databases[db.name] = db
         self.cache_headers = cache_headers
         self.cors = cors
         self._metadata = metadata or {}
@@ -201,6 +203,14 @@ def __init__(
                     # Plugin already registered
                     pass
 
+    @property
+    def databases(self):
+        databases = dict(self._databases)
+        # pylint: disable=no-member
+        for pairs in pm.hook.available_databases(datasette=self):
+            databases.update(pairs)
+        return databases
+
     async def run_sanity_checks(self):
         # Only one check right now, for Spatialite
         for database_name, database in self.databases.items():
@@ -470,20 +480,7 @@ async def execute_against_connection_in_thread(self, db_name, fn):
         def in_thread():
             conn = getattr(connections, db_name, None)
             if not conn:
-                db = self.databases[db_name]
-                if db.is_memory:
-                    conn = sqlite3.connect(":memory:")
-                else:
-                    # mode=ro or immutable=1?
-                    if db.is_mutable:
-                        qs = "mode=ro"
-                    else:
-                        qs = "immutable=1"
-                    conn = sqlite3.connect(
-                        "file:{}?{}".format(db.path, qs),
-                        uri=True,
-                        check_same_thread=False,
-                    )
+                conn = self.databases[db_name].connect()
                 self.prepare_connection(conn)
                 setattr(connections, db_name, conn)
             return fn(conn)

datasette/cli.py

@@ -186,7 +186,7 @@ def package(
     install,
     spatialite,
     version_note,
-    **extra_metadata
+    **extra_metadata,
 ):
     "Package specified SQLite files into a new datasette Docker container"
     if not shutil.which("docker"):
@@ -220,6 +220,13 @@ def package(
         call(args)
 
 
+def extra_serve_options(serve):
+    for options in pm.hook.extra_serve_options():
+        for option in reversed(options):
+            serve = option(serve)
+    return serve
+
+
 @cli.command()
 @click.argument("files", type=click.Path(exists=True), nargs=-1)
 @click.option(
@@ -286,6 +293,7 @@ def package(
 )
 @click.option("--version-note", help="Additional note to show on /-/versions")
 @click.option("--help-config", is_flag=True, help="Show available config options")
+@extra_serve_options
 def serve(
     files,
     immutable,
@@ -304,6 +312,7 @@ def serve(
     config,
     version_note,
     help_config,
+    **extra_serve_options,
 ):
     """Serve up specified SQLite database files with a web UI"""
     if help_config:
@@ -350,6 +359,7 @@ def serve(
         config=dict(config),
         memory=memory,
         version_note=version_note,
+        extra_serve_options=extra_serve_options,
     )
     # Run async sanity checks - but only if we're not under pytest
     asyncio.get_event_loop().run_until_complete(ds.run_sanity_checks())

datasette/database.py

@@ -14,15 +14,19 @@
 
 
 class Database:
-    def __init__(self, ds, path=None, is_mutable=False, is_memory=False):
+    def __init__(
+        self, ds, path=None, name=None, is_mutable=False, is_memory=False, comment=None
+    ):
         self.ds = ds
+        self._name = name
         self.path = path
         self.is_mutable = is_mutable
         self.is_memory = is_memory
         self.hash = None
         self.cached_size = None
         self.cached_table_counts = None
-        if not self.is_mutable:
+        self.comment = comment
+        if not self.is_mutable and path is not None:
             p = Path(path)
             self.hash = inspect_hash(p)
             self.cached_size = p.stat().st_size
@@ -33,9 +37,21 @@ def __init__(self, ds, path=None, is_mutable=False, is_memory=False):
                     for key, value in self.ds.inspect_data[self.name]["tables"].items()
                 }
 
+    def connect(self):
+        if self.is_memory:
+            return sqlite3.connect(":memory:")
+        # mode=ro or immutable=1?
+        if self.is_mutable:
+            qs = "mode=ro"
+        else:
+            qs = "immutable=1"
+        return sqlite3.connect(
+            "file:{}?{}".format(self.path, qs), uri=True, check_same_thread=False
+        )
+
     @property
     def size(self):
-        if self.is_memory:
+        if self.is_memory or self.path is None:
             return 0
         if self.cached_size is not None:
             return self.cached_size
@@ -71,6 +87,8 @@ def mtime_ns(self):
 
     @property
     def name(self):
+        if self._name:
+            return self._name
         if self.is_memory:
             return ":memory:"
         else:

datasette/hookspecs.py

@@ -58,3 +58,13 @@ def register_output_renderer(datasette):
 @hookspec
 def register_facet_classes():
     "Register Facet subclasses"
+
+
+@hookspec
+def extra_serve_options():
+    "Return list of extra click.option decorators to be applied to 'datasette serve'"
+
+
+@hookspec
+def available_databases(datasette):
+    "Return list of (name, database) pairs to be added to the available databases"

datasette/plugins.py

@@ -8,6 +8,7 @@
     "datasette.publish.now",
     "datasette.publish.cloudrun",
     "datasette.facets",
+    "datasette.serve_dir",
 )
 
 pm = pluggy.PluginManager("datasette")

datasette/serve_dir.py

@@ -0,0 +1,76 @@
+from datasette import hookimpl
+from pathlib import Path
+from .database import Database
+from .utils import escape_sqlite
+import click
+
+
+@hookimpl
+def extra_serve_options():
+    return [
+        click.option(
+            "-d",
+            "--dir",
+            type=click.Path(exists=True, file_okay=False, dir_okay=True),
+            help="Directories to scan for SQLite databases",
+            multiple=True,
+        ),
+        click.option(
+            "--scan",
+            is_flag=True,
+            help="Continually scan directories for new database files",
+        ),
+    ]
+
+
+cached_results = None
+
+
+@hookimpl
+def available_databases(datasette):
+    global cached_results
+    if cached_results is not None:
+        return cached_results
+    i = 0
+    counts = {name: 0 for name in datasette._databases}
+    results = []
+    for directory in datasette.extra_serve_options.get("dir") or []:
+        for filepath in Path(directory).glob("**/*"):
+            if is_sqlite(filepath):
+                name = filepath.stem
+                if name in counts:
+                    new_name = "{}_{}".format(name, counts[name] + 1)
+                    counts[name] += 1
+                    name = new_name
+                try:
+                    database = Database(datasette, str(filepath), comment=str(filepath))
+                    conn = database.connect()
+                    result = conn.execute(
+                        "select name from sqlite_master where type = 'table'"
+                    )
+                    table_names = [r[0] for r in result]
+                    for table_name in table_names:
+                        conn.execute(
+                            "PRAGMA table_info({});".format(escape_sqlite(table_name))
+                        )
+                except Exception as e:
+                    print("Could not open {}".format(filepath))
+                    print("  " + str(e))
+                else:
+                    results.append((name, database))
+
+    cached_results = results
+    return results
+
+
+magic = b"SQLite format 3\x00"
+
+
+def is_sqlite(path):
+    if not path.is_file():
+        return False
+    try:
+        with open(path, "rb") as fp:
+            return fp.read(len(magic)) == magic
+    except PermissionError:
+        return False

datasette/templates/index.html

@@ -11,6 +11,7 @@ <h1>{{ metadata.title or "Datasette" }}</h1>
 
 {% for database in databases %}
     <h2 style="padding-left: 10px; border-left: 10px solid #{{ database.color }}"><a href="{{ database.path }}">{{ database.name }}</a></h2>
+    {% if database.comment %}<p style="color: #aaa">{{ database.comment }}</p>{% endif %}
     <p>
         {% if database.show_table_row_counts %}{{ "{:,}".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.tables_count and database.hidden_tables_count %}, {% endif -%}
         {% if database.hidden_tables_count -%}

datasette/views/base.py

@@ -203,6 +203,8 @@ async def resolve_db_name(self, request, db_name, **kwargs):
                 hash = None
         else:
             name = db_name
+        if "%" in name:
+            name = urllib.parse.unquote_plus(name)
         # Verify the hash
         try:
             db = self.ds.databases[name]

datasette/views/database.py

@@ -47,6 +47,7 @@ async def data(self, request, database, hash, default_labels=False, _size=None):
             {
                 "database": database,
                 "size": db.size,
+                "comment": db.comment,
                 "tables": tables,
                 "hidden_count": len([t for t in tables if t["hidden"]]),
                 "views": views,

datasette/views/index.py

@@ -79,6 +79,7 @@ async def get(self, request, as_format):
                 {
                     "name": name,
                     "hash": db.hash,
+                    "comment": db.comment,
                     "color": db.hash[:6]
                     if db.hash
                     else hashlib.md5(name.encode("utf8")).hexdigest()[:6],

docs/changelog.rst

@@ -51,7 +51,7 @@ Two new plugins take advantage of this hook:
 New plugin hook: extra_template_vars
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The :ref:`plugin_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github <https://github.com/simonw/datasette-auth-github>`__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 <https://github.com/simonw/datasette/issues/540>`__).
+The :ref:`plugin_hook_extra_template_vars` plugin hook allows plugins to inject their own additional variables into the Datasette template context. This can be used in conjunction with custom templates to customize the Datasette interface. `datasette-auth-github <https://github.com/simonw/datasette-auth-github>`__ uses this hook to add custom HTML to the new top navigation bar (which is designed to be modified by plugins, see `#540 <https://github.com/simonw/datasette/issues/540>`__).
 
 Secret plugin configuration options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/datasette-serve-help.txt

@@ -23,4 +23,6 @@ Options:
                             datasette.readthedocs.io/en/latest/config.html
   --version-note TEXT       Additional note to show on /-/versions
   --help-config             Show available config options
+  -d, --dir DIRECTORY       Directories to scan for SQLite databases
+  --scan                    Continually scan directories for new database files
   --help                    Show this message and exit.

docs/plugins.rst

@@ -812,3 +812,67 @@ This example plugin adds a ``x-databases`` HTTP header listing the currently att
                 await app(scope, recieve, wrapped_send)
             return add_x_databases_header
         return wrap_with_databases_header
+
+.. _plugin_hook_extra_serve_options:
+
+extra_serve_options()
+~~~~~~~~~~~~~~~~~~~~~
+
+Add extra Click options to the ``datasette serve`` command. Options you add here will be displayed in ``datasette serve --help`` and their values will be available to your plugin anywhere it can access the ``datasette`` object by reading from ``datasette.extra_serve_options``.
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    import click
+
+    @hookimpl
+    def extra_serve_options():
+        return [
+            click.option(
+                "--my-plugin-paths",
+                type=click.Path(exists=True, file_okay=False, dir_okay=True),
+                help="Directories to use with my-plugin",
+                multiple=True,
+            ),
+            click.option(
+                "--my-plugin-enable",
+                is_flag=True,
+                help="Enable functionality from my-plugin",
+            ),
+        ]
+
+Your other plugin hooks can then access these settings like so:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+    @hookimpl
+    def extra_template_vars(datasette):
+        return {
+            "my_plugin_paths": datasette.extra_serve_options.get("my_plugin_paths") or []
+        }
+
+Be careful not to define an option which clashes with a Datasette default option, or with options provided by another plugin. For this reason we recommend using a common prefix for your plugin, as shown above.
+
+.. _plugin_hook_available_databases:
+
+available_databases(datasette)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Return a list of ``(name, database)`` pairs to be added to the available databases.
+
+``name`` should be a string. ``database`` should be a ``datasette.database.Database`` instance.
+
+This allows plugins to make databases available from new sources.
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    from datasette.database import Database
+
+    @hookimpl
+    def available_databases(datasette):
+        return [
+            ("hardcoded_database", Database(datasette, "/mnt/hard_coded.db"))
+        ]