Implement full text search on all models

Author: 3j14

alembic/versions/b6f593818541_search.py

@@ -0,0 +1,32 @@
+"""empty message
+
+Revision ID: b6f593818541
+Revises: c2d7aa36d0e7
+Create Date: 2022-06-17 18:35:58.043728
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from doku.models import TSVector
+
+
+# revision identifiers, used by Alembic.
+
+revision = 'b6f593818541'
+down_revision = 'c2d7aa36d0e7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('doku_document', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True))
+    op.create_index('doku_document___ts_vector__', 'doku_document', ['__ts_vector__'], unique=False, postgresql_using='gin')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('doku_document___ts_vector__', table_name='doku_document', postgresql_using='gin')
+    op.drop_column('doku_document', '__ts_vector__')
+    # ### end Alembic commands ###

alembic/versions/c7d3e7e86fe0_all_models_search.py

@@ -0,0 +1,43 @@
+"""empty message
+
+Revision ID: c7d3e7e86fe0
+Revises: b6f593818541
+Create Date: 2022-06-17 18:58:45.565407
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from doku.models import TSVector
+
+
+# revision identifiers, used by Alembic.
+revision = 'c7d3e7e86fe0'
+down_revision = 'b6f593818541'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('doku_resource', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True))
+    op.create_index('doku_resource___ts_vector__', 'doku_resource', ['__ts_vector__'], unique=False, postgresql_using='gin')
+    op.add_column('doku_snippet', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True))
+    op.create_index('doku_snippet___ts_vector__', 'doku_snippet', ['__ts_vector__'], unique=False, postgresql_using='gin')
+    op.add_column('doku_stylesheet', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True))
+    op.create_index('doku_stylesheet___ts_vector__', 'doku_stylesheet', ['__ts_vector__'], unique=False, postgresql_using='gin')
+    op.add_column('doku_template', sa.Column('__ts_vector__', TSVector(), sa.Computed("to_tsvector('english', name)", persisted=True), nullable=True))
+    op.create_index('doku_template___ts_vector__', 'doku_template', ['__ts_vector__'], unique=False, postgresql_using='gin')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('doku_template___ts_vector__', table_name='doku_template', postgresql_using='gin')
+    op.drop_column('doku_template', '__ts_vector__')
+    op.drop_index('doku_stylesheet___ts_vector__', table_name='doku_stylesheet', postgresql_using='gin')
+    op.drop_column('doku_stylesheet', '__ts_vector__')
+    op.drop_index('doku_snippet___ts_vector__', table_name='doku_snippet', postgresql_using='gin')
+    op.drop_column('doku_snippet', '__ts_vector__')
+    op.drop_index('doku_resource___ts_vector__', table_name='doku_resource', postgresql_using='gin')
+    op.drop_column('doku_resource', '__ts_vector__')
+    # ### end Alembic commands ###

doku/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 __author__ = "Jonas Drotleff <j.drotleff@desk-lab.de>"
 __all__ = ["create_app", "cli"]
 

doku/blueprints/base.py

@@ -1,4 +1,4 @@
-from flask import Blueprint, render_template
+from flask import Blueprint, render_template, request
 
 from doku.models import db
 from doku.models.document import Document
@@ -15,9 +15,15 @@ def index():
     ordering, order, direction = get_ordering(
         Document, default_order="last_updated", default_dir="desc"
     )
-    documents = (
-        db.session.query(Document).order_by(ordering).paginate(page=page, per_page=10)
-    )
+    query: str = request.args.get("query", "", type=str)
+    documents = db.session.query(Document)
+    if query != "":
+        documents = documents.filter(Document.__ts_vector__.match(query))
+    documents = documents.order_by(ordering).paginate(page=page, per_page=10)
     return render_template(
-        "sites/index.html", documents=documents, order=order, direction=direction
+        "sites/index.html",
+        documents=documents,
+        order=order,
+        direction=direction,
+        query=query,
     )

doku/blueprints/resources.py

@@ -43,13 +43,18 @@ def index():
                 db.session.add(resource)
                 db.session.commit()
     page = get_pagination_page()
-    resources = db.session.query(Resource).paginate(page=page, per_page=10)
+    query: str = request.args.get("query", "", type=str)
+    resources = db.session.query(Resource)
+    if query != "":
+        resources = resources.filter(Resource.__ts_vector__.match(query))
+    resources = resources.paginate(page=page, per_page=10)
 
     resource_schemas = ResourceSchema(session=db.session, many=True)
     return render_template(
         "sites/resources.html",
         resources_json=resource_schemas.dumps(resources.items),
         resources=resources,
+        query=query,
     )
 
 

doku/blueprints/snippet.py

@@ -15,8 +15,16 @@
 @login_required
 def index():
     page = get_pagination_page()
-    snippets = db.session.query(Snippet).paginate(page=page, per_page=10)
-    return render_template("sites/snippets.html", snippets=snippets)
+    query: str = request.args.get("query", "", type=str)
+    snippets = db.session.query(Snippet)
+    if query != "":
+        snippets = snippets.filter(Snippet.__ts_vector__.match(query))
+    snippets = snippets.paginate(page=page, per_page=10)
+    return render_template(
+        "sites/snippets.html",
+        snippets=snippets,
+        query=query,
+    )
 
 
 @bp.route("/<int:snippet_id>", methods=["GET", "POST"])

doku/blueprints/stylesheets.py

@@ -46,13 +46,17 @@ def index():
             db.session.commit()
 
     page = get_pagination_page()
-    stylesheets = db.session.query(Stylesheet).paginate(page=page, per_page=10)
+    query: str = request.args.get("query", "", type=str)
+    stylesheets = db.session.query(Stylesheet)
+    if query != "":
+        stylesheets = stylesheets.filter(Stylesheet.__ts_vector__.match(query))
+    stylesheets = stylesheets.paginate(page=page, per_page=10)
 
     stylesheet_schemas = StylesheetSchema(
         session=db.session, many=True, include=("source",)
     )
     return render_template(
         "sites/stylesheets.html",
         stylesheets_json=stylesheet_schemas.dumps(stylesheets.items),
-        stylesheets=stylesheets,
+        stylesheets=stylesheets, query=query
     )

doku/blueprints/template.py

@@ -1,4 +1,4 @@
-from flask import Blueprint, render_template
+from flask import Blueprint, render_template, request
 
 from doku import db
 from doku.models.schemas import TemplateSchema, StylesheetSchema
@@ -13,19 +13,17 @@
 @login_required
 def index_all():
     page = get_pagination_page()
-    ordering, order, direction = get_ordering(
-        Template, default_order="name", default_dir="asc"
-    )
-    templates = (
-        db.session.query(Template).order_by(ordering).paginate(page=page, per_page=10)
-    )
+    query: str = request.args.get("query", "", type=str)
+    templates = db.session.query(Template)
+    if query != "":
+        templates = templates.filter(Template.__ts_vector__.match(query))
+    templates = templates.paginate(page=page, per_page=10)
     template_schema = TemplateSchema(session=db.session, many=True)
     return render_template(
         "sites/templates.html",
         templates_json=template_schema.dumps(templates.items),
         templates=templates,
-        order=order,
-        direction=direction,
+        query=query,
     )
 
 

doku/models/__init__.py

@@ -3,10 +3,16 @@
 from flask_babel import format_timedelta, format_datetime
 from flask_sqlalchemy import SQLAlchemy
 from marshmallow_sqlalchemy import auto_field
+from sqlalchemy import TypeDecorator
+from sqlalchemy.dialects.postgresql import TSVECTOR
 
 db = SQLAlchemy(session_options={"autoflush": False})
 
 
+class TSVector(TypeDecorator):
+    impl = TSVECTOR
+
+
 class DateMixin:
     created_on = db.Column(db.DateTime(timezone=True), default=datetime.now)
     last_updated = db.Column(

doku/models/document.py

@@ -1,6 +1,7 @@
 from flask import session
+from sqlalchemy import Index
 
-from doku.models import db, DateMixin
+from doku.models import db, DateMixin, TSVector
 
 
 class Document(db.Model, DateMixin):
@@ -34,6 +35,18 @@ class Document(db.Model, DateMixin):
         overlaps="variables, document",
     )
 
+    __ts_vector__ = db.Column(
+        TSVector(),
+        db.Computed(
+            "to_tsvector('english', name)",
+            persisted=True
+        )
+    )
+
+    __table_args__ = (
+        Index('doku_document___ts_vector__', __ts_vector__, postgresql_using='gin'),
+    )
+
     def __init__(self, *args, author_id=None, author=None, **kwargs):
         if author_id is None and author is None:
             author_id = session.get("id", None)

doku/models/resource.py

@@ -2,9 +2,10 @@
 import string
 
 from flask import url_for
+from sqlalchemy import Index
 from werkzeug.utils import secure_filename
 
-from doku.models import db, DateMixin
+from doku.models import db, DateMixin, TSVector
 
 
 class Resource(db.Model, DateMixin):
@@ -16,6 +17,18 @@ class Resource(db.Model, DateMixin):
     name = db.Column(db.String(255), unique=False, nullable=False)
     filename = db.Column(db.String(255), unique=True, nullable=False)
 
+    __ts_vector__ = db.Column(
+        TSVector(),
+        db.Computed(
+            "to_tsvector('english', name)",
+            persisted=True
+        )
+    )
+
+    __table_args__ = (
+        Index('doku_resource___ts_vector__', __ts_vector__, postgresql_using='gin'),
+    )
+
     @property
     def url(self):
         return url_for("resources.view", resource_id=self.id)

doku/models/snippet.py

@@ -1,6 +1,6 @@
-from sqlalchemy import event
+from sqlalchemy import event, Index
 
-from doku.models import db, DateMixin
+from doku.models import db, DateMixin, TSVector
 from doku.utils.markdown import compile_content
 
 
@@ -23,6 +23,18 @@ class Snippet(db.Model, DateMixin):
 
     used_by = db.relationship("Variable")
 
+    __ts_vector__ = db.Column(
+        TSVector(),
+        db.Computed(
+            "to_tsvector('english', name)",
+            persisted=True
+        )
+    )
+
+    __table_args__ = (
+        Index('doku_snippet___ts_vector__', __ts_vector__, postgresql_using='gin'),
+    )
+
     def __str__(self):
         return f"Snippet {self.name}"
 

doku/models/template.py

@@ -4,9 +4,10 @@
 from jinja2 import Environment, meta
 from jinja2 import Template as Jinja2Template
 from pygments.formatters.html import HtmlFormatter
+from sqlalchemy import Index
 from weasyprint import HTML, CSS
 
-from doku.models import db, DateMixin
+from doku.models import db, DateMixin, TSVector
 from doku.utils.weasyfetch import url_fetcher
 
 DEFAULT_TEMPLATE = """<!doctype html>
@@ -49,6 +50,18 @@ class Template(db.Model, DateMixin):
         "Stylesheet", secondary=template_stylesheet_relation, back_populates="templates"
     )
 
+    __ts_vector__ = db.Column(
+        TSVector(),
+        db.Computed(
+            "to_tsvector('english', name)",
+            persisted=True
+        )
+    )
+
+    __table_args__ = (
+        Index('doku_template___ts_vector__', __ts_vector__, postgresql_using='gin'),
+    )
+
     def __str__(self):
         return self.name
 
@@ -110,6 +123,18 @@ class Stylesheet(db.Model, DateMixin):
 
     MAX_CONTENT_LENGTH = 125000
 
+    __ts_vector__ = db.Column(
+        TSVector(),
+        db.Computed(
+            "to_tsvector('english', name)",
+            persisted=True
+        )
+    )
+
+    __table_args__ = (
+        Index('doku_stylesheet___ts_vector__', __ts_vector__, postgresql_using='gin'),
+    )
+
     def __str__(self):
         return self.name
 

doku/static/src/sites/templates.vue

@@ -9,6 +9,15 @@
         Create new template
       </button>
     </div>
+    <form action="" class="mb-2" method="get">
+      <input type="hidden" name="page" :value="formPage"/>
+      <div class="form-inline">
+          <input class="form-input input-sm" type="text" name="query" :value="formQuery" placeholder="Search">
+      </div>
+      <button class="btn btn-sm btn-primary" type="submit">
+          Search
+      </button>
+    </form>
 
     <template-item
       v-for="template in templates"
@@ -61,6 +70,12 @@ export default {
   components: {
     TemplateItem, Modal, PlusIcon
   },
+  data() {
+    return {
+      formPage: window._formPage,
+      formQuery: window._formQuery,
+    };
+  },
   computed: mapState({
     templates: state => state.template.templates,
   }),