Add spellchecking to Solr. Fixes #60.

Author: kernelmethod

docker/solr/conf/managed-schema

@@ -124,7 +124,7 @@
     <!-- Combine the "title" and "abstract" fields so that we search both when performing basic search -->
     <copyField source="title" dest="basic_search" />
     <copyField source="abstract" dest="basic_search" />
-    <field name="basic_search" type="text_general" />
+    <field name="basic_search" type="text_general" indexed="true" stored="false" />
 
     <!--
     END COMPENDIUM FIELDS

docker/solr/conf/solrconfig.xml

@@ -849,7 +849,7 @@
     <!-- a spellchecker built from a field of the main index -->
     <lst name="spellchecker">
       <str name="name">default</str>
-      <str name="field">_text_</str>
+      <str name="field">title</str>
       <str name="classname">solr.DirectSolrSpellChecker</str>
       <!-- the spellcheck distance measure used, the default is the internal levenshtein -->
       <str name="distanceMeasure">internal</str>

requirements.txt

@@ -5,5 +5,5 @@ django[argon2] == 3.0.3
 gunicorn == 20.0.4
 psycopg2-binary == 2.8.4
 pylibmc == 1.6.1
-pysolr == 3.8.1
 python-dotenv == 0.11.0
+requests == 2.23.0

src/assets/css/style.css

@@ -119,7 +119,7 @@ a.anchor:hover {
 
 .searchbar {
   background-color: white;
-  border: 4px black;
+  border: 1px solid black;
 }
 
 /****************************

src/search/forms.py

@@ -19,11 +19,12 @@ class BasicSearchForm(forms.Form):
     query = forms.CharField(
         widget=SearchInput(
             attrs={
-                "class": "uk-input uk-form-large",
+                "class": "searchbar uk-input uk-form-large",
                 "type": "search",
                 "placeholder": "Search...",
             }
-        )
+        ),
+        required=False,
     )
 
     rows = forms.IntegerField(initial=20, widget=forms.HiddenInput(), required=False,)
@@ -72,7 +73,7 @@ def clean_page(self):
 
     def clean(self):
         cleaned_data = super().clean()
-        query_params = cleaned_data.pop("query")
+        query_params = cleaned_data.pop("query", {})
         cleaned_data.update(query_params)
         return cleaned_data
 

src/search/solr.py

@@ -2,8 +2,8 @@
 Code for integrating with Apache Solr for search on the site
 """
 
-import pysolr
-
+import logging
+import requests
 from typing import Dict, List
 
 
@@ -13,10 +13,10 @@ class SearchEngine:
     to and query Apache Solr.
     """
 
+    solr_logger = logging.getLogger("search.solr")
+
     def __init__(self):
-        self.solr = pysolr.Solr(
-            "http://tec-search:8983/solr/compendium", always_commit=False
-        )
+        self.solr_url = "http://tec-search:8983/solr/compendium"
 
     """
     Search functions
@@ -31,25 +31,39 @@ def basic_search(self, query: Dict[str, List[str]]):
         tokens = query.get("quoted_substrings", [])
         tokens += query.get("words", [])
 
+        # If tokens == [], we assume that we didn't receive any words at all,
+        # in which case we'll simply match everything in the compendium.
+        if len(tokens) == 0:
+            tokens = [""]
+
         # Combine tokens into a single search query for Solr
-        query_str = " && ".join(f"(abstract:{T!r} || title:{T!r})" for T in tokens)
+        query_str = " && ".join(f"basic_search:*{T}*" for T in tokens)
 
         # Add pagination
         page = query.get("page", 0)
         rows = query.get("rows", 20)
         start = page * rows
-        kwargs = {
+        data = {
+            "q": query_str,
             "rows": rows,
             "start": start,
             "fl": "id,title,abstract,slug,year,month,day",
         }
 
-        results = self.solr.search(query_str, **kwargs)
+        req = requests.get(f"{self.solr_url}/spell", params=data)
+        results = req.json()
 
-        # Add information about the start, page number, and rows for
-        # future handling
-        results.page = page
-        results.rows = rows
-        results.start = start
+        # Add some more useful data to the results dictionary
+        results["meta"] = {
+            "page": page,
+            "rows": rows,
+        }
+
+        # Do some logging to record the transaction
+        qtime = results["responseHeader"]["QTime"]
+        self.solr_logger.info(f"Solr query: {query_str}")
+        self.solr_logger.debug(
+            f"Solr query metadata: qtime={qtime}ms queryurl={req.url}"
+        )
 
         return results

src/search/views/mixins.py

@@ -4,6 +4,7 @@
 
 import abc
 import json
+import logging
 
 from django.core import serializers
 from django.db.models import QuerySet
@@ -89,7 +90,13 @@ class BasicSearchMixin(metaclass=abc.ABCMeta):
     in them.
     """
 
+    # Parent logger for search-related components
+    search_logger = logging.getLogger("search")
+
+    # Name of the button used to perform search
     basic_search_button_name = "search"
+
+    # Wrapper class for querying Solr
     search_engine = SearchEngine()
 
     def check_basic_search(self, request) -> bool:
@@ -102,23 +109,28 @@ def check_basic_search(self, request) -> bool:
 
         return request.method == "GET" and self.basic_search_button_name in request.GET
 
-    def create_search_form(self, request):
+    def create_search_form(self, data):
         """
         Return a form to display a basic search input on the page.
         """
-        form = BasicSearchForm(data=request.GET)
-        return form
+        if isinstance(data, dict):
+            self.search_logger.info(f"Received search params: {data}")
+            return BasicSearchForm(data=data)
+        else:
+            self.search_logger.info(f"Received search params: {data.GET.dict()}")
+            return BasicSearchForm(data=data.GET)
 
-    def execute_basic_search(self, params):
+    def execute_basic_search(self, request):
         """
         Run a basic search request. Return all compendium entries matching
         the input query.
         """
 
-        form = BasicSearchForm(data=params)
+        form = self.create_search_form(request)
 
         # TODO: more robust error checking
         if form.is_valid():
+            self.search_logger.debug(f"Cleaned search params: {form.cleaned_data}")
             results = self.search_engine.basic_search(form.cleaned_data)
         else:
             raise Exception(str(form.errors))

src/search/views/search_results.py

@@ -3,6 +3,7 @@
 """
 
 import math
+import re
 
 from django.core.paginator import Paginator
 from django.shortcuts import render, redirect
@@ -19,50 +20,95 @@ class SearchView(BasicSearchMixin, View):
     default_pagination = 10
 
     def get(self, request):
-        results = self.execute_basic_search(request.GET)
-
-        query = request.GET.get("query", None)
+        query = request.GET.dict()
+        query.setdefault("query", "")
+        self.search_logger.info(f"QUERY = {query}")
+        results = self.execute_basic_search(query)
+        search_form = self.create_search_form(query)
 
         # Populate some CompendiumEntry objects with the data that we found
         # from Solr
-        entries = [CompendiumEntry(**entry) for entry in results.docs]
+        entries = results["response"]["docs"]
+        entries = [CompendiumEntry(**entry) for entry in entries]
 
-        hits = results.hits
-        rows = results.rows  # Entries per page
-        page = results.page
+        hits = results["response"]["numFound"]
+        rows = results["meta"]["rows"]  # Entries per page
+        page = results["meta"]["page"]
 
         # Start/end result numbers
-        start = results.start + 1
-        end = results.start + len(entries)
+        start = min(results["response"]["start"] + 1, hits)
+        end = results["response"]["start"] + len(entries)
 
         if hits == 0:
             n_pages = 0
         else:
             n_pages = math.ceil(hits / rows)
 
         context = {
+            "query": query,
+            "search_form": search_form,
+            "qtime": results["responseHeader"]["QTime"],
             "hits": hits,
             "page": page,
             "n_pages": n_pages,
             "rows": rows,
             "start": start,
             "end": end,
             "entries": entries,
-            "query": query,
             "start": start,
             "rows": rows,
         }
-        return render(request, "entry_list.html", context)
 
-    def get_old(self, request):
-        entries = CompendiumEntry.objects.all()
+        # Check spelling
+        correctly_spelled, suggested_query = self._check_spelling(query, results)
 
-        # Paginate (don't show all of the results on a single page)
-        paginator = Paginator(entries, self.default_pagination)
-        page_obj = paginator.get_page(request.GET.get("page"))
-        page_number = page_obj.number
+        if not correctly_spelled:
+            context["suggested_query"] = suggested_query
 
-        context = {
-            "page_obj": page_obj,
-        }
         return render(request, "entry_list.html", context)
+
+    """
+    Internal API
+    """
+
+    def _check_spelling(self, query: str, results: dict):
+        """
+        Check the spelling of the results returned by execute_basic_search.
+        If the query is misspelled and there aren't many (or any) results
+        returned by the query, provide spelling suggestions.
+
+        Returns
+        -------
+        correctly_spelled : bool
+            Whether or not the words in the query were correctly spelled.
+            If no spelling mistakes where found, or there were sufficiently
+            many results returned, correctly_spelled is returned as True.
+
+        suggested_query : Optional[str]
+            A suggested query to replace the input query. Returns as None
+            if no suggested query could be generated.
+        """
+
+        self.search_logger.info(results)
+
+        hits = results["response"]["numFound"]
+        correctly_spelled = results.get("spellcheck", {}).get("correctlySpelled", True)
+        correctly_spelled = correctly_spelled or hits > 10
+
+        if correctly_spelled:
+            return correctly_spelled, None
+
+        # Create a "suggested query" by trying to fix every misspelled word
+        # that was found.
+        s = results.get("spellcheck", {}).get("suggestions", [])
+        if len(s) == 0:
+            return correctly_spelled, None
+
+        suggestions = [(s[ii], s[ii + 1]) for ii in range(len(s) // 2)]
+        for (word, suggestion) in suggestions:
+            # Find the top-suggested replacement and use regex to replace it
+            # in the query.
+            replacement = suggestion["suggestion"][0]["word"]
+            query = re.sub(f"\\b{word}\\b", replacement, query, flags=re.IGNORECASE)
+
+        return correctly_spelled, query

src/templates/entry_list.html

@@ -14,7 +14,18 @@ <h1 class="uk-h1">
     <hr>
 
     <div class="uk-container">
-      <p>Seeing results {{ start }} - {{ end }} out of {{ hits }}</p>
+      <div class="uk-grid-match" uk-grid>
+        <div class="uk-width-1-2@m">
+          <h3>Seeing results {{ start }} - {{ end }} out of {{ hits }}</h3>
+        </div>
+        <div class="uk-width-1-2@m uk-text-right uk-text-small">
+          <span class="monospace uk-text-muted">Search finished in {{ qtime }}ms</span>
+        </div>
+      </div>
+
+      <div class="uk-container uk-width-1-2@m uk-text-center">
+        {% include "includes/searchbar.html" with form=search_form only %}
+      </div>
 
       {% for entry in entries %}
         {% include "includes/entry_snippet.html" with entry=entry only %}
@@ -29,6 +40,20 @@ <h1 class="uk-h1">
   search results.
   {% endcomment %}
 
+  {% if hits == 0 %}
+  <div class="uk-text-center">
+    <h3>No results were found matching the query</h3>
+  </div>
+  {% endif %}
+
+  {% if suggested_query %}
+  <p class="uk-text-lead">
+    Did you mean to search for
+    <a href="?query={{ suggested_query }}&rows={{ rows }}" class="uk-text-italic">{{ suggested_query }}</a>?
+  </p>
+  {% endif %}
+
+  {% if hits > 0 %}
   <div>
     <div class="uk-grid uk-text-center uk-width-1-1 paginator">
       {# Display options to go to the next and previous page(s) #}
@@ -37,7 +62,7 @@ <h1 class="uk-h1">
         <a href="?query={{ query }}&rows={{ rows }}&page=0">
           <span uk-icon="icon:chevron-double-left;ratio:1.5"></span> first
         </a> |
-        <a href="?query={{ query }}&rows={{ rows }}page={{ page|add:-1 }}">previous</a>
+        <a href="?query={{ query }}&rows={{ rows }}&page={{ page|add:-1 }}">previous</a>
       {% endif %}
       </div>
 
@@ -55,6 +80,7 @@ <h1 class="uk-h1">
       </div>
     </div>
   </div>
+  {% endif %}
 
 </div>
 {% endblock %}