Merge pull request #264 from joyofrails/feat/search-query-parser

rossta · web-flow · commit 6b4ba5a2d8bd · 2024-10-31T13:21:23.000-04:00
Add search query parser
diff --git a/Gemfile b/Gemfile
@@ -31,6 +31,7 @@ gem "scout_apm" # Scout APM Ruby Agent [https://scoutapm.com]
 gem "rails_admin" # RailsAdmin is a Rails engine that provides an easy-to-use interface for managing your data [https://github.com/railsadminteam/rails_admin]
 gem "addressable" # Addressable is an alternative implementation to URI [https://github.com/sporkmonger/addressable]
 gem "ostruct" # OpenStruct is a data structure, similar to a Hash, that allows the definition of arbitrary attributes with their accompanying values
+gem "parslet" # Parslet is a small Ruby library for constructing parsers [https://github.com/kschiess/parslet]
 
 # Rendering
 gem "image_processing" # Use Active Storage variants [https://guides.rubyonrails.org/active_storage_overview.html#transforming-images]
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -337,6 +337,7 @@ GEM
     parser (3.3.5.0)
       ast (~> 2.4.1)
       racc
+    parslet (2.0.0)
     phlex (2.0.0.beta2)
     phlex-rails (2.0.0.beta2)
       phlex (= 2.0.0.beta2)
@@ -621,6 +622,7 @@ DEPENDENCIES
   meta-tags
   mission_control-jobs
   ostruct
+  parslet
   phlex (= 2.0.0.beta2)
   phlex-rails (= 2.0.0.beta2)
   postmark-rails
diff --git a/app/controllers/searches_controller.rb b/app/controllers/searches_controller.rb
@@ -1,12 +1,20 @@
 class SearchesController < ApplicationController
+  rescue_from Searches::ParseFailed do |error|
+    respond_to do |format|
+      format.html { render Searches::ShowView.new(pages: [], query: @raw_query) }
+      format.turbo_stream { render turbo_stream: turbo_stream.replace("search-results", plain: "No results") }
+    end
+  end
+
   def show
-    query = params[:query] || ""
-    pages = if query.present?
-      Page.search(query).limit(3)
+    @raw_query = params[:query] || ""
+    pages = if @raw_query.present?
+      query = Searches::Query.parse!(@raw_query)
+      Page.search("#{query}*").with_snippets.ranked.limit(3)
     else
-      Page.limit(3)
+      []
     end
 
-    render Searches::ShowView.new(pages:, query:)
+    render Searches::ShowView.new(pages:, query: @raw_query)
   end
 end
diff --git a/app/javascript/css/baseline.css b/app/javascript/css/baseline.css
@@ -621,6 +621,10 @@ figcaption {
   padding: 0;
 }
 
+mark {
+  background-color: aqua;
+}
+
 .dark {
   p {
     font-weight: 300;
diff --git a/app/javascript/css/components/combobox.css b/app/javascript/css/components/combobox.css
@@ -7,6 +7,6 @@
 
   & a:hover,
   & .selected {
-    background-color: var(--joy-button-secondary);
+    background-color: var(--joy-block-selected);
   }
 }
diff --git a/app/javascript/css/components/dialog.scss b/app/javascript/css/components/dialog.scss
@@ -1,6 +1,7 @@
 @import '../config/variables.scss';
 
 dialog {
+  color: var(--joy-text);
   width: 100%;
   border: none;
   animation: dialog-popup 0.25s ease-in-out;
@@ -24,7 +25,22 @@ dialog {
   }
 
   input[type='search'] {
-    border: 0;
+    border: none;
+    border-left: 1px solid var(--joy-border-quiet);
+    color: var(--joy-text);
+  }
+}
+
+.dark {
+  & dialog {
+    border: 1px solid var(--joy-border-quiet);
+    background-color: var(--color-gray-950);
+
+    & input[type='search'] {
+      margin-left: 1rem;
+      background-color: var(--color-gray-800);
+      border: 1px solid var(--joy-border);
+    }
   }
 }
 
diff --git a/app/javascript/css/config/theme.css b/app/javascript/css/config/theme.css
@@ -76,6 +76,8 @@
   --joy-link-decoration: var(--joy-link-1);
   --joy-link-active: var(--joy-link-5);
 
+  --joy-block-selected: var(--joy-color-200);
+
   --joy-button-primary: var(--joy-color-600);
   --joy-button-primary-hover: var(--joy-color-700);
   --joy-button-primary-active: var(--joy-color-800);
@@ -127,6 +129,8 @@
   --joy-link-decoration: var(--joy-link-8);
   --joy-link-active: var(--joy-link-5);
 
+  --joy-block-selected: var(--joy-color-900);
+
   --joy-button-transparent: none;
   --joy-button-transparent-hover: var(--joy-color-950);
   --joy-button-transparent-active: var(--joy-color-900);
diff --git a/app/javascript/css/layout.scss b/app/javascript/css/layout.scss
@@ -96,6 +96,10 @@ main > *,
   column-gap: var(--space-m);
 }
 
+.col-gap-xs {
+  column-gap: var(--space-xs);
+}
+
 .col-gap-3xs {
   column-gap: var(--space-3xs);
 }
diff --git a/app/javascript/css/utilities/tailwind.css b/app/javascript/css/utilities/tailwind.css
@@ -699,6 +699,10 @@
   padding-top: 4rem;
 }
 
+.pt-2 {
+  padding-top: 0.5rem;
+}
+
 .pt-4 {
   padding-top: 1rem;
 }
diff --git a/app/models/page.rb b/app/models/page.rb
@@ -1,16 +1,37 @@
+# frozen_string_literal: true
+
 # ActiveRecord model to represent a static page in the database.
 class Page < ApplicationRecord
   after_create_commit :create_in_search_index
   after_update_commit :update_in_search_index
   after_destroy_commit :remove_from_search_index
 
-  def self.rebuild_search_index
-    find_each(&:update_in_search_index)
+  # The pages_search_index table is a full-text search index for the pages
+  # table. FTS5 tables contain a hidden 'rank' column that is (hand waving) a
+  # relevancy score for sorting search results.
+  # https://www.sqlite.org/fts5.html#sorting_by_auxiliary_function_results
+  #
+  scope :ranked, -> { order(:rank) }
+
+  # snippet() is a SQLite function that returns a snippet of text containing the search term.
+  # It’s used to highlight search results. The snippet() function is not
+  # available in all databases, so this scope is only available for SQLite as
+  # part of the FTS5 extension.
+  # https://www.sqlite.org/fts5.html#the_snippet_function
+  #
+  scope :with_snippets, ->(**options) do
+    select("#{table_name}.*")
+      .select("snippet(pages_search_index, 0, '<mark>', '</mark>', '…', 32) AS title_snippet")
+      .select("snippet(pages_search_index, 1, '<mark>', '</mark>', '…', 32) AS body_snippet")
   end
 
   def self.search(query)
     joins("JOIN pages_search_index ON pages.id = pages_search_index.page_id")
-      .where("pages_search_index MATCH ?", query.to_s.gsub(/\W/, " ") + "*")
+      .where("pages_search_index MATCH ?", query)
+  end
+
+  def self.rebuild_search_index
+    find_each(&:update_in_search_index)
   end
 
   def title
diff --git a/app/models/searches.rb b/app/models/searches.rb
@@ -0,0 +1,4 @@
+module Searches
+  Error = Class.new(StandardError)
+  ParseFailed = Class.new(Error)
+end
diff --git a/app/models/searches/condition.rb b/app/models/searches/condition.rb
@@ -0,0 +1,15 @@
+module Searches
+  class Condition
+    attr_accessor :left, :operator, :right
+
+    def initialize(left, operator, right)
+      @left = left
+      @operator = operator
+      @right = right
+    end
+
+    def to_s
+      "#{left} #{operator} #{right}"
+    end
+  end
+end
diff --git a/app/models/searches/operator.rb b/app/models/searches/operator.rb
@@ -0,0 +1,28 @@
+module Searches
+  class Operator
+    def self.symbol(str)
+      case str
+      when "+"
+        :and
+      when "-"
+        :not
+      when "|"
+        :or
+      else
+        raise "Unknown operator: #{str}"
+      end
+    end
+
+    def initialize(operator)
+      @operator = self.class.symbol(operator)
+    end
+
+    def to_s
+      @operator.to_s.upcase
+    end
+
+    def to_sym
+      @operator.to_sym
+    end
+  end
+end
diff --git a/app/models/searches/phrase.rb b/app/models/searches/phrase.rb
@@ -0,0 +1,13 @@
+module Searches
+  class Phrase
+    attr_accessor :phrase # Array of strings
+
+    def initialize(phrase)
+      @phrase = phrase
+    end
+
+    def to_s
+      %("#{phrase.join(" ")}")
+    end
+  end
+end
diff --git a/app/models/searches/query.rb b/app/models/searches/query.rb
@@ -0,0 +1,27 @@
+module Searches
+  class Query
+    attr_accessor :expressions
+
+    def self.parse!(query)
+      ast = Searches::QueryParser.new.parse(query)
+
+      Searches::QueryTransformer.new.apply(ast)
+    rescue Parslet::ParseFailed => error
+      raise Searches::ParseFailed, error.message
+    end
+
+    def self.parse(query)
+      parse!(query)
+    rescue Searches::ParseFailed
+      nil
+    end
+
+    def initialize(expressions)
+      @expressions = expressions
+    end
+
+    def to_s
+      expressions.map(&:to_s).join(" ")
+    end
+  end
+end
diff --git a/app/models/searches/query_parser.rb b/app/models/searches/query_parser.rb
@@ -0,0 +1,29 @@
+module Searches
+  class QueryParser < Parslet::Parser
+    rule(:dquote) { str(%(")) }
+    rule(:squote) { str(%(')) }
+    rule(:lparen) { str("(") }
+    rule(:rparen) { str(")") }
+
+    rule(:space) { match('\s').repeat(1) }
+    rule(:space?) { space.maybe }
+
+    rule(:operator) { (str("+") | str("-") | str("|")).as(:operator) >> space? }
+    rule(:term) { match[%(a-zA-Z0-9_)].repeat(1).as(:term) >> space? }
+    rule(:phrase) do
+      (
+        (squote >> (term >> space?).repeat >> squote) |
+        (dquote >> (term >> space?).repeat >> dquote)
+      ).as(:phrase) >> space?
+    end
+
+    rule(:condition) { (token.as(:left) >> operator >> expression.as(:right)).as(:condition) }
+    rule(:token) { phrase | term }
+    rule(:expression) { (condition | token) }
+
+    rule(:subexpression) { (lparen >> expression.as(:subexpression) >> rparen) }
+
+    rule(:query) { (expression | subexpression).repeat.as(:query) }
+    root(:query)
+  end
+end
diff --git a/app/models/searches/query_transformer.rb b/app/models/searches/query_transformer.rb
@@ -0,0 +1,11 @@
+module Searches
+  class QueryTransformer < Parslet::Transform
+    rule(term: simple(:term)) { Term.new(term.to_s) }
+    rule(phrase: sequence(:phrase)) { Phrase.new(phrase) }
+    rule(condition: {left: simple(:left), operator: simple(:operator), right: subtree(:right)}) do
+      Condition.new(left, Operator.new(operator), right)
+    end
+    rule(subexpression: simple(:expression)) { Subexpression.new(expression) }
+    rule(query: sequence(:expressions)) { Searches::Query.new(expressions) }
+  end
+end
diff --git a/app/models/searches/subexpression.rb b/app/models/searches/subexpression.rb
@@ -0,0 +1,13 @@
+module Searches
+  class Subexpression
+    attr_reader :expression
+
+    def initialize(expression)
+      @expression = expression
+    end
+
+    def to_s
+      "(#{expression})"
+    end
+  end
+end
diff --git a/app/models/searches/term.rb b/app/models/searches/term.rb
@@ -0,0 +1,13 @@
+module Searches
+  class Term
+    attr_accessor :term
+
+    def initialize(term)
+      @term = term
+    end
+
+    def to_s
+      @term.to_s
+    end
+  end
+end
diff --git a/app/views/searches/_help.html.mdrb b/app/views/searches/_help.html.mdrb
@@ -0,0 +1,41 @@
+Search the Joy of Rails archives.
+
+### Usage
+
+Type search text into the Search box above.
+
+On all other pages on the site, click the Search icon in the top nav or type `cmd+k` (macos) / `Windows+k` (windows) to open the Search dialog.
+
+![Screenshot of search dialog](https://github.com/user-attachments/assets/7ee482bf-3f56-4750-bf38-31832923bb65)
+
+### Syntax
+
+The Search input supports a bare text search:
+
+```sh:{"header":false}
+Ruby on Rails  # => Search for any of the three tokens
+```
+
+Quotes help group terms into phrases:
+
+```sh:{"header":false}
+"Ruby Rails" "Laravel PHP"  # => Search for any of the two phrases
+```
+
+The symbols `+`, `-`, and `|` are used to perform `AND`, `NOT`, and `OR` operations respectively with left-to-right precedence.
+
+```sh:{"header":false}
+Phlex + ERB # => Search must return both terms
+
+"Active Record" - "Action View" # => Search must only return the first of the two phrases
+
+Ruby + Rails | Hanami # => Search for "Ruby AND Rails" OR "Hanami"
+```
+
+Use parens to group conditions and change precedence:
+
+```sh:{"header":false}
+Ruby + (Rails | Hanami) # => Search for "Ruby AND Rails" OR "Ruby AND Hanami"
+```
+
+The search parsing and transformation logic is built on top of [Parslet](https://kschiess.github.io/parslet/get-started.html). The Parslet docs and [this blog post on query parsing](http://recursion.org/query-parser) provided helpful guidance.
diff --git a/app/views/searches/button.rb b/app/views/searches/button.rb
diff --git a/app/views/searches/combobox.rb b/app/views/searches/combobox.rb
diff --git a/app/views/searches/show_view.rb b/app/views/searches/show_view.rb
diff --git a/app/views/users/header_navigations/show_view.rb b/app/views/users/header_navigations/show_view.rb
diff --git a/config/application.rb b/config/application.rb
diff --git a/config/initializers/new_framework_defaults_8_0.rb b/config/initializers/new_framework_defaults_8_0.rb
diff --git a/spec/models/page_spec.rb b/spec/models/page_spec.rb
diff --git a/spec/models/searches/query_parser_spec.rb b/spec/models/searches/query_parser_spec.rb
diff --git a/spec/models/searches/query_transformer_spec.rb b/spec/models/searches/query_transformer_spec.rb
diff --git a/spec/requests/search_spec.rb b/spec/requests/search_spec.rb

Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,6 @@`
`7`	`7`
`8`	`8`	`& a:hover,`
`9`	`9`	`& .selected {`
`10`		`- background-color: var(--joy-button-secondary);`
	`10`	`+ background-color: var(--joy-block-selected);`
`11`	`11`	`}`
`12`	`12`	`}`
Original file line number	Diff line number	Diff line change
`@@ -96,6 +96,10 @@ main > *,`
`96`	`96`	`column-gap: var(--space-m);`
`97`	`97`	`}`
`98`	`98`
	`99`	`+.col-gap-xs {`
	`100`	`+ column-gap: var(--space-xs);`
	`101`	`+}`
	`102`	`+`
`99`	`103`	`.col-gap-3xs {`
`100`	`104`	`column-gap: var(--space-3xs);`
`101`	`105`	`}`
Original file line number	Diff line number	Diff line change
`@@ -699,6 +699,10 @@`
`699`	`699`	`padding-top: 4rem;`
`700`	`700`	`}`
`701`	`701`
	`702`	`+.pt-2 {`
	`703`	`+ padding-top: 0.5rem;`
	`704`	`+}`
	`705`	`+`
`702`	`706`	`.pt-4 {`
`703`	`707`	`padding-top: 1rem;`
`704`	`708`	`}`