Provide new initialization API endpoint (I63) (#64)

* i63:init index

* validate schema spec

* schema validation: bug fixes

Author: ptitzler

README.md

@@ -252,6 +252,50 @@ The response is similar to that of editing a row, although again note that the r
 }
 ```
 
+### Initializing the index
+
+To programatically delete all data and initialize the index
+
+```
+POST /initialize
+```
+
+including the `schema` property in the payload defining the following structure 
+
+```
+{ "fields": [
+    {
+      "name": "id",
+      "type": "string",
+      "example": "example_id",
+      "facet": true
+    },
+    {
+      "name": "score",
+      "type": "number",
+      "example": 8,
+      "facet": false
+    },
+    {
+      "name": "tags",
+      "type": "arrayofstrings",
+      "example": "example_tag_1,example_tag_2",
+      "facet": true
+    }
+  ]
+}
+
+> This example defines a schema containing three fields of which two will be enabled for faceted search.
+
+```
+Valid values:
+
+* Property `name`: any string
+* Property `type`: `number`, `boolean`, `string`, `arrayofstrings` (e.g. `val1,val2,val3`)
+* Property `example`: any valid value for this `type`
+* Property `facet`: `true` or `false`
+
+
 ## Privacy Notice
 
 The Simple Search Service web application includes code to track deployments to Bluemix and other Cloud Foundry platforms. The following information is sent to a [Deployment Tracker](https://github.com/IBM-Bluemix/cf-deployment-tracker-service) service on each deployment:

app.js

@@ -124,8 +124,6 @@ app.post('/fetch', bodyParser, isloggedin.auth, function(req, res){
 
 // import previously uploaded CSV
 app.post('/import', bodyParser, isloggedin.auth, function(req, res){
-  console.log("****",req.body.schema);
-  console.log("****");
 
   var currentUpload = app.locals.import[req.body.upload_id];
 
@@ -147,6 +145,59 @@ app.post('/import', bodyParser, isloggedin.auth, function(req, res){
   res.status(204).end();
 });
 
+// Initialize the SSS index by deleting all data and saving the new schema
+// Request parameters: 
+//  - schema: JSON defining a valid schema (required)
+// 
+app.post('/initialize', bodyParser, isloggedin.auth, function(req, res){
+
+  if((!req.body) || (! req.body.schema)) {
+    return res.status(400).json({error: "Request rejected", reason: "Schema definition is missing."});
+  }
+
+  var theschema = null;
+
+  try {
+    // parse schema
+    theschema = JSON.parse(req.body.schema);
+  }
+  catch(err) {
+    // payload is not valid JSON; return client error
+    console.error("/initialize: payload " + req.body.schema + " caused JSON parsing error: " + JSON.stringify(err));
+    return res.status(400).json({error: "Request rejected", reason: "Schema definition is not valid JSON."});
+  }
+
+  // validate schema definition
+  var validationErrors = schema.validateSchemaDef(theschema);
+  if(validationErrors) {
+    // payload is not valid schema definition; return client error
+    return res.status(400).json({error: "Request rejected", reason: "Schema validation failed. Errors: " + validationErrors.join("; ")});
+  }
+
+  var cache = require("./lib/cache.js")(app.locals.cache);
+  if (cache) {
+    cache.clearAll();
+  }
+
+  // re-create index database
+  db.deleteAndCreate(function(err) {
+    if(err) {
+      // index database could not be dropped/created; return server error
+      return res.status(500).json({error: "Request failed", reason: "Index database could not be re-initialized: " + JSON.stringify(err)});
+    }
+    // save schema definition
+    schema.save(theschema, function(err) {
+      if(err) {
+        // schema could not be saved; return server error
+        return res.status(500).json({error: "Request failed", reason: "Schema could not be saved: " + JSON.stringify(err)});
+      }
+      // initialization complete; return OK
+      return res.status(200).end();
+    });
+  });
+});
+
+
 app.get('/import/status', isloggedin.auth, function(req, res) {
   var status = dbimport.status();
   res.send(status);

internal-api-reference.md

@@ -178,4 +178,48 @@ where `"complete":true` indicates the completion of the import process.
 
 ## POST /deleteeverything
 
-Delete the database and start again.
+Delete the database and start again.
+
+
+## POST /initialize
+
+Delete the database and define schema.
+
+A form-encoded HTTP POST is expected to include a valid JSON payload describing the schema
+
+```
+	"fields": [
+		{
+			"name": "id",
+			"type": "string",
+			"facet": true,
+			"example": "4a9f23"
+		}, 
+		{
+			"name": "tags",
+			"type": "arrayofstrings",
+			"facet": true,
+			"example": "eins,zwei,drei"			
+		}, 
+		...
+	]
+```
+
+Each field specification must define the [field] `name`, [field] `type` and `facet` properties.
+The `example` property is optional. If set it should contain a valid value.
+
+> All property names are case sensitive.
+
+Valid values:
+
+ * `name`: any unique string
+ * `type`: "`string`" || "`number`" || "`boolean`" || "`arrayofstrings`"  (case sensitive)
+ * `facet`: `true` or `false`
+ * `example`: any string representing a valid value for the field
+
+Return codes and responses:
+
+* `200` Request was successfully processed.
+* `400` The schema definition is invalid. JSON response includes properties `error` and `reason`.
+* `500` Request processing failed. JSON response includes properties `error` and `reason`.
+

lib/schema.js

@@ -188,10 +188,103 @@ var validate = function(schema, row, editMode) {
   }
 
 };
- 
+
+/**
+ * Determines whether a schema definition adheres to the specification. Data structure:
+ * "fields" : [
+ *             {
+ *              "name": <unique_string>,
+ *              "type": <type_value>,
+ *              "facet": <facet_value>,
+ *              "example": <string> 
+ *             },
+ *             ...
+ *            ]
+ * where <type_value> one of ["string","number","boolean","arrayofstrings"] and <facet_value>
+ * one of [true, false]. All properties except example are required.
+ * @param {Object} schema - the schema definition to be validated
+ * @return {Array} null if the schema definition is valid or an array of Strings listing issues 
+ */
+var validateSchemaDef = function(schema) { 
+
+  var errors = [];
+
+  if((! schema) || ((! schema.hasOwnProperty("fields")))) {
+    // fatal schema definition error
+    errors.push("The schema must contain at least one field definition.");
+    return errors;
+  }
+
+  if(! Array.isArray(schema.fields)) {
+    // fatal schema definition error
+    errors.push("The property named `fields` must define a non-empty array of field definitions.");
+    return errors;    
+  }
+
+  if(schema.fields.length < 1) {
+    // fatal schema definition error
+    errors.push("The property named `fields` must define a non-empty array of field definitions.");
+    return errors;    
+  }
+
+  // schema specification (property name, data type, required, values)
+  const fieldSpecs = [
+                      {name: "name", type: "string", required: true},
+                      {name: "type", type: "string", required: true, values: ["string","number","boolean","arrayofstrings"]},
+                      {name: "facet", type: "boolean", required: true, values: [true, false]},
+                      {name: "example", type: "string", required: false}
+                    ];
+
+  // field counter
+  var count = 1; 
+  // iterate through all field definitions and verify that they adhere to the specification;
+  // identify all issues before returning an error
+  _.each(schema.fields,
+         function(field) {
+          _.each(fieldSpecs,
+                 function(fieldSpec) {
+                    if(! field.hasOwnProperty(fieldSpec.name)) {
+                      if(fieldSpec.required) {
+                        // a required property is missing
+                        errors.push("Field " + count + " - property `" + fieldSpec.name + "` is missing");
+                      }
+                    }
+                    else {
+                      // validate data type of property value                    
+                      if(typeof field[fieldSpec.name] !== fieldSpec.type) {
+                        // the data type of the field's property value is invalid
+                        errors.push("Field " + count + " - data type of property `" + fieldSpec.name + "` must be `" + fieldSpec.type + "`");
+                      }
+                      else {
+                        // validate property value
+                        if((fieldSpec.hasOwnProperty('values')) && (! _.contains(fieldSpec.values, field[fieldSpec.name]))) {
+                          // the property value is invalid
+                          errors.push("Field " + count + " - value of property `" + fieldSpec.name + "` must be one of " + _.map(fieldSpec.values, function(element) { return ("`" + element + "`");}).join(","));
+                        }
+                        // enforce constraint that no facet can be defined for fields with data type "number"
+                        if((field.type === "number") && (field.facet)) {
+                          errors.push("Field " + count + " - cannot facet field with data type `number`. Use data type `string` instead.");
+                        }
+                      }
+                  }
+          });
+          count++;
+  });
+
+  if(errors.length) {
+    // schema definition appears to be invalid; return error list 
+    return errors;
+  }
+  else {
+    // schema definition appears to be valid; return null
+    return null;
+  }
+};
+
 module.exports = {
   load: load,
   save: save,
   generateSearchIndex: generateSearchIndex,
-  validate: validate
+  validate: validate,
+  validateSchemaDef: validateSchemaDef
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "simple-search-service",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "A Node app that creates a faceted search engine, powered by IBM Cloudant",
   "scripts": {
     "start": "node app.js",