edit: docs and readme

hp4k1h5 · hp4k1h5 · commit 1762ec8d3c08 · 2020-06-19T13:41:34.000-04:00
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@ built/
 node_modules/
 yarn.lock
 docs/
+**/*.wiki
diff --git a/README.md b/README.md
@@ -1,6 +1,8 @@
 # AQLqueryBuilder.js
 > a typescript query builder for [arangodb](https://www.arangodb.com)'s [ArangoSearch](https://www.arangodb.com/docs/stable/arangosearch.html)
 
+##### !! warning !! experimental and unstable
+
 ## overview
 ArangoSearch provides a high-level API for interacting with Arango Search Views
 through the Arango Query Language (AQL). This library aims to provide a query
@@ -47,11 +49,83 @@ provided, optional terms are considered required, so as not to retrieve all
 documents.
 
 ## setup
+
+1) running generated AQL queries will require a working arangodb instance. in
+the future, it is hoped that this package can be imported and used in the
+`arangosh`, as well as client and server side. Currently there is only limited
+support for server-side use.
+
 ## installation
+
+!! packaging and export behavior is not stable, and is likely to change
+!! significantly in the short-term
+1) clone this repository in your es6 compatible project.
+2) run `yarn install` from the project directory.
+
 ## usage
+for better documentation, run `yarn doc && serve docs/` from the project
+  directory root.
+
+AQLqueryBuilder aims to provide collection-agnostic and language-agnostic
+boolean search capabilities to the library's user. Currently, this library
+makes a number of assumptions about the way your data is stored and indexed,
+but these are hopefully compatible with a wide range of setups.
+
+The primary assumption this library makes is that the data you are trying to
+query against is indexed by an ArangoSearch View, and that all documents index
+the same exact field. This field can be indexed by any number of analyzers,
+and the search be will run against all supplied collections simultaneously. This
+allows for true multi-language search provided that each collection is
+restricted to just one language and all documents index the same key as all
+other documents in the view. While there are plans to expand on this
+functionality to provide multi-key search, this library is primarily built for
+academic and textual searches, and is ideally suited for documents like books,
+articles, and other media where most of the data resides in a single place.
+
+This works best as a document query tool. Leveraging ArangoSearch's built-in
+language stemming analyzers allows for complex search phrases to be run
+against any number of language-specific collections simultaneously.
+
+For an example of a multi-lingual document ingest/parser, please see
+[ptolemy's curator](https://gitlab.com/HP4k1h5/nineveh/-/tree/master/ptolemy/dimitri/curator.js)
+
+__Example:__
+```javascript
+import {buildAQL} from 'path/to/AQLqueryBuilder'
+const queryObject =
+{
+  "view": "the_arango-search_view-name",
+  "collections": [{
+    "name": "collection_name",
+    "analyzer": "analyzer_name"
+  }],
+  "query": "+'query string' -for +parseQuery ?to parse"
+}
+const aqlQuery = buildAQL(queryObject)
+// ... const cursor = await db.query(aqlQuery)
+```
+`collections` is an array of `collection` objects. This allows searching and
+filtering across collections impacted by the search.
+
 ### query object
 
 `buildAQL` accepts an object with the following properties:
+
+• **view**: *string* (required): the name of the ArangoSearch view the query
+will be run against
+
+• **collections** (required): the names of the collections indexed by @view to query
+
+• **terms** (required): either an array of @term interfaces or a string to be
+parsed by @parseQuery
+
+• **key** (optional | default: "text"): the name of the Arango document key to search
+within.
+
+• **filters** (optional): a list of @filter interfaces
+
+___
+
 Example:
 ```json
 {
@@ -63,18 +137,72 @@ Example:
        "analyzer_name"
     }
   ],
+  "key": "text",
   "query": "either a +query ?\"string for parseQuery to parse\"",
   "query": [
            {"type": "phr", "op": "?", "val": "\"or a list of query objects\""},
            {"type": "tok", "op": "-", "val": "tokens"}
          ],
+  "filters": [
+    {
+      "field": "field_name",
+      "op": ">",
+      "val": 0
+    }
+  ],
   "limit":
     {
       "start": 0,
       "end": 20,
     }
 }
 ```
+
+### boolean search logic
+Quoting [mit's Database Search Tips](https://libguides.mit.edu/c.php?g=175963&p=1158594):
+> Boolean operators form the basis of mathematical sets and database logic.
+    They connect your search words together to either narrow or broaden your
+    set of results.  The three basic boolean operators are: AND, OR, and NOT.
+
+#### `+` AND
+* Mandatory terms and phrases. All results MUST INCLUDE these terms and
+* phrases.
+#### `?` OR
+* Optional terms and phrases. If there are ANDS or NOTS, these serve as
+* match score "boosters". If there are no ANDS or NOTS, ORS become required
+* in results.
+#### `-` NOT
+* Search results MUST NOT INCLUDE these terms and phrases. If a result that
+* would otherwise have matched, contains one or more terms or phrases, it
+* will not be included in the result set.
+
 ### default query syntax
+for more information on boolean search logic see
+  [above](#boolean-search-logic)
+
+The default syntax accepted by `AQLqueryBuilder`'s `query` object's `terms`
+key is as follows:
+
+1) Everything inside single or double quotes is considered a `PHRASE`
+2) Everything else is considered a word to be analyzed by `TOKENS`
+3) Every individual search word and quoted phrase may be optionally prefixed
+by one of the following symbols `+ ? -`, or the plus-sign, the question-mark,
+and the minus-sign. If a word has no operator prefix, it is considered
+optional and is counted as an `OR`.
+
+Example:
+input `one +two -"buckle my shoe"` and the queryParser will interpret as
+follows:
+
+|        | ANDS | ORS | NOTS             |
+| -      | -    | -   | -                |
+| PHRASE |      |     | "buckle my shoe" |
+| TOKENS | two  | one |                  |
+
+The generated AQL query, when run will bring back only results that contain
+"two", that do not contain variations on the phrase "buckle my shoe", and that
+optionally contain "one". In this case, documents that contain "one" will be
+likely to score higher than those that do not.
+
 ## bugs
 ## contributing
diff --git a/package.json b/package.json
@@ -6,7 +6,7 @@
   "scripts": {
     "test": "mocha -r ts-node/register",
     "tests": "mocha -r ts-node/register 'tests/*.ts'",
-    "doc": "typedoc src/"
+    "doc": "typedoc --plugin none src/"
   },
   "homepage": "https://github.com/HP4k1h5/AQLqueryBuilder.js",
   "repository": {
@@ -35,6 +35,7 @@
     "mocha": "^8.0.1",
     "ts-node": "^8.10.2",
     "typedoc": "^0.17.7",
+    "typedoc-plugin-markdown": "^2.3.1",
     "typescript": "^3.9.5"
   },
   "author": "HP4k1h5",
diff --git a/src/lib/structs.ts b/src/lib/structs.ts
@@ -12,6 +12,11 @@ export interface query {
   * */
   terms: term[] | string,
   /**
+  * the name of the document key to search, must be the same across all
+  * documents 
+  * */
+  key?: string,
+  /**
   * a list of @filter interfaces
   * */
   filters?: filter[],
@@ -28,33 +33,6 @@ export interface collection {
   analyzer: string,
 }
 
-/**
- * @terms are the basic boolean search logic terms applied to ArangoSearch
- * Views via the Arango Query Language (AQL). 
- * Example: {
- *  ANDS: {anas:[], phrs: []},
- * }
-**/
-export interface terms {
-  /**
-   * Mandatory terms and phrases. All results MUST INCLUDE these terms and
-   * phrases.
-  **/
-  ANDS?: term[],
-  /**
-   * Optional terms and phrases. If there are ANDS or NOTS, these serve as
-   * match score "boosters". If there are no ANDS or NOTS, ORS become required
-   * in results.
-  **/
-  ORS?: term[],
-  /**
-   * Search results MUST NOT INCLUDE these terms and phrases. If a result that
-   * would otherwise have matched, contains one or more terms or phrases, it
-   * will not be included in the result set.
-  **/
-  NOTS?: term[],
-}
-
 export interface term {
   type: string,
   val: string,
diff --git a/src/search.ts b/src/search.ts
@@ -10,9 +10,9 @@ export function buildSearch(query: query): any {
     : query.terms
 
   /* build boolean pieces */
-  let ANDS = buildOPS(query.collections, query.terms, '+')
-  let ORS = buildOPS(query.collections, query.terms, '?')
-  let NOTS = buildOPS(query.collections, query.terms, '-')
+  let ANDS = buildOPS(query.collections, query.terms, '+', query.key)
+  let ORS = buildOPS(query.collections, query.terms, '?', query.key)
+  let NOTS = buildOPS(query.collections, query.terms, '-', query.key)
 
   /* handle combinations */
   if (ANDS && ORS) {
@@ -37,15 +37,16 @@ export function buildSearch(query: query): any {
       SORT TFIDF(doc) DESC`
 }
 
-function buildOPS(collections: collection[], terms: term[], op: string): any {
+function buildOPS(collections: collection[], terms: term[], op: string, key:
+  string = 'text'): any {
   const opWord: string = op == '+' ? ' AND ' : ' OR '
 
   let queryTerms: any = terms.filter((t: term) => t.op == op)
   if (!queryTerms.length) return
 
   /* phrases */
   let phrases = queryTerms.filter((qT: term) => qT.type == 'phr')
-    .map((phrase: any) => buildPhrase(phrase, collections))
+    .map((phrase: any) => buildPhrase(phrase, collections, key))
   if (!phrases.length) {
     phrases = undefined
   } else {
@@ -54,21 +55,21 @@ function buildOPS(collections: collection[], terms: term[], op: string): any {
 
   /* tokens */
   let tokens = queryTerms.filter((qT: { type: string }) => qT.type === 'tok')
-  tokens = tokens && buildTokens(tokens, collections)
+  tokens = tokens && buildTokens(tokens, collections, key)
 
   if (!phrases && !tokens) return
   if (op == '-') return { phrases, tokens }
   if (phrases && tokens) return aql.join([ phrases, tokens ], opWord)
   return (tokens || phrases)
 }
 
-function buildPhrase(phrase: term, collections: collection[]): any {
+function buildPhrase(phrase: term, collections: collection[], key: string): any {
   return collections.map(coll => {
-    return aql`PHRASE(doc.text, ${phrase.val.slice(1, -1)}, ${coll.analyzer})`
+    return aql`PHRASE(doc${key}, ${phrase.val.slice(1, -1)}, ${coll.analyzer})`
   })
 }
 
-function buildTokens(tokens: term[], collections: collection[]): any {
+function buildTokens(tokens: term[], collections: collection[], key: string): any {
   if (!tokens.length) return
 
   const opWordMap = {
@@ -83,17 +84,19 @@ function buildTokens(tokens: term[], collections: collection[]): any {
     return a
   }, {})
 
-  const makeTokenAnalyzers = (tokens: term[], op: string, analyzer: string, field: string) => {
+  const makeTokenAnalyzers = (tokens: term[], op: string, analyzer: string,
+    key: string) => {
     return aql`
       ANALYZER(
         TOKENS(${tokens}, ${analyzer})
-        ${aql.literal(op)} IN doc.${field}, ${analyzer})`
+        ${aql.literal(op)} IN doc.${key}, ${analyzer})`
   }
 
   let remapped = []
   collections.forEach(coll => {
     remapped.push(
-      ...Object.keys(mapped).map(op => makeTokenAnalyzers(mapped[ op ], op, coll.analyzer, 'text'))
+      ...Object.keys(mapped).map(op => makeTokenAnalyzers(mapped[ op ], op,
+        coll.analyzer, key))
     )
   })
 
