Adds support for xapian, whoosh, elastic and solr (all via django-haystack). Fixes ciur/papermerge#473

Author: ciur

.gitignore

@@ -38,3 +38,5 @@ queue/
 celerybeat-schedule
 launch.json
 .vscode
+xapian_index
+whoosh_index

docker/dev/config/settings.py

@@ -28,3 +28,31 @@
 LOCALE_PATHS = (
     PROJ_ROOT / Path('papermerge'),
 )
+
+search_engine = config.get('search', 'engine', default='xapian')
+
+if search_engine == 'xapian':
+    HAYSTACK_CONNECTIONS['default']['PATH'] = config.get(
+        'search',
+        'path',
+        default=os.path.join(PROJ_ROOT, 'xapian_index')
+    )
+elif search_engine == 'whoosh':
+    HAYSTACK_CONNECTIONS['default']['PATH'] = config.get(
+        'search',
+        'path',
+        default=os.path.join(PROJ_ROOT, 'whoosh_index')
+    )
+elif search_engine in (
+        'es7',
+        'es',
+        'elasticsearch7',
+        'elasticsearch',
+        'elastic',
+        'elastic7',
+        'solr'
+):
+    HAYSTACK_CONNECTIONS['default']['URL'] = config.get(
+        'search',
+        'url'
+    )

docker/dev/xapian_install.sh

@@ -0,0 +1,43 @@
+
+#!/usr/bin/env bash
+# first argument of the script is Xapian version (e.g. 1.2.19)
+VERSION=$1
+
+# prepare
+mkdir $VIRTUAL_ENV/packages && cd $VIRTUAL_ENV/packages
+
+CORE=xapian-core-$VERSION
+BINDINGS=xapian-bindings-$VERSION
+
+# download
+echo "Downloading source..."
+curl -O https://oligarchy.co.uk/xapian/$VERSION/${CORE}.tar.xz
+curl -O https://oligarchy.co.uk/xapian/$VERSION/${BINDINGS}.tar.xz
+
+# extract
+echo "Extracting source..."
+tar xf ${CORE}.tar.xz
+tar xf ${BINDINGS}.tar.xz
+
+# install
+echo "Installing Xapian-core..."
+cd $VIRTUAL_ENV/packages/${CORE}
+./configure --prefix=$VIRTUAL_ENV && make && make install
+
+PYV=`python -c "import sys;t='{v[0]}'.format(v=list(sys.version_info[:1]));sys.stdout.write(t)";`
+
+if [ $PYV = "2" ]; then
+    PYTHON_FLAG=--with-python
+else
+    PYTHON_FLAG=--with-python3
+fi
+
+echo "Installing Xapian-bindings..."
+cd $VIRTUAL_ENV/packages/${BINDINGS}
+./configure --prefix=$VIRTUAL_ENV $PYTHON_FLAG XAPIAN_CONFIG=$VIRTUAL_ENV/bin/xapian-config && make && make install
+
+# clean
+# rm -rf $VIRTUAL_ENV/packages
+
+# test
+python -c "import xapian"

docker/prod/config/settings.py

@@ -24,3 +24,31 @@
 LOCALE_PATHS = (
     PROJ_ROOT / Path('papermerge'),
 )
+
+search_engine = config.get('search', 'engine', default='xapian')
+
+if search_engine == 'xapian':
+    HAYSTACK_CONNECTIONS['default']['PATH'] = config.get(
+        'search',
+        'path',
+        default=os.path.join(PROJ_ROOT, 'xapian_index')
+    )
+elif search_engine == 'whoosh':
+    HAYSTACK_CONNECTIONS['default']['PATH'] = config.get(
+        'search',
+        'path',
+        default=os.path.join(PROJ_ROOT, 'whoosh_index')
+    )
+elif search_engine in (
+        'es7',
+        'es',
+        'elasticsearch7',
+        'elasticsearch',
+        'elastic',
+        'elastic7',
+        'solr'
+):
+    HAYSTACK_CONNECTIONS['default']['URL'] = config.get(
+        'search',
+        'url'
+    )

docker/services-plus-kibana.yml renamed to docker/services-plus-es.yml

@@ -16,20 +16,9 @@ services:
       - 6379:6379
     volumes:
       - redisdata:/data
-  es:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
-    environment:
-      - discovery.type=single-node
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    ports:
-      - 9200:9200
-      - 9300:9300
-    volumes:
-      - es-data:/usr/share/elasticsearch/data
 volumes:
   postgres_data2:
   redisdata:
-  es-data:
 
 networks:
   my_local:

docker/services.yml

@@ -19,9 +19,6 @@
 redis_host = config.get('redis', 'host', default='127.0.0.1')
 redis_port = config.get('redis', 'port', default=6379)
 
-es_hosts = config.get('elasticsearch', 'hosts', default=None)
-es_port = config.get('elasticsearch', 'port', default=None)
-
 CELERY_BROKER_URL = f"redis://{redis_host}:{redis_port}/0"
 CELERY_WORKER_HIJACK_ROOT_LOGGER = False
 CELERY_ACCEPT_CONTENT = ['json']
@@ -46,17 +43,6 @@
 
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
-if es_hosts and es_port:
-    ELASTICSEARCH_DSL = {
-        'default': {
-            'hosts': config.get(
-                'elasticsearch',
-                'hosts',
-                default=f"{es_hosts}:{es_port}"
-            )
-        },
-    }
-
 # Custom signal processor handles connections errors (to elasticsearch)
 # and reports them as warnings. This way, even when no connection to ES
 # is available, documents, folders, pages etc can still be used
@@ -140,6 +126,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'papermerge.core.apps.CoreConfig',
+    'papermerge.search.apps.SearchConfig',
     'papermerge.notifications.apps.NotificationsConfig',
     'django.contrib.contenttypes',
     'dynamic_preferences',
@@ -148,16 +135,12 @@
     'polymorphic',
     'mptt',
     'channels',
+    'haystack',
 ]
 
 # include elasticsearch apps only if PAPERMERGE_ELASTICSEARCH_HOSTS
 # and PAPERMERGE_ELASTICSEARCH_PORT are defined
 # and have non-empty value
-if es_hosts and es_port:
-    INSTALLED_APPS.extend([
-        'papermerge.search.apps.SearchConfig',
-        'django_elasticsearch_dsl'
-    ])
 
 MIDDLEWARE = [
     'corsheaders.middleware.CorsMiddleware',
@@ -347,3 +330,26 @@
     'VERSION': '2.1.0',
     'APPEND_COMPONENTS': JSONAPI_COMPONENTS
 }
+
+SEARCH_ENGINES_MAP = {
+    'elastic': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine',
+    'elastic7': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine',
+    'elasticsearch7': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine',
+    'elasticsearch': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine',
+    'es7': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine',
+    'es': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine',
+    'solr': 'haystack.backends.solr_backend.SolrEngine',
+    'whoosh': 'haystack.backends.whoosh_backend.WhooshEngine',
+    'xapian': 'xapian_backend.XapianEngine',
+}
+
+HAYSTACK_DOCUMENT_FIELD = 'indexed_content'
+
+search_engine = config.get('search', 'engine', default='xapian')
+
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': SEARCH_ENGINES_MAP[search_engine],
+    },
+}

papermerge/conf/settings.py

@@ -0,0 +1,2 @@
+TAGS_OP_ALL = 'all'
+TAGS_OP_ANY = 'any'