From c55cf28faf3be2878f222fc2e5261aa0d607f14c Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Fri, 3 Feb 2017 11:51:56 -0800 Subject: [PATCH 1/3] i63:init index --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ app.js | 40 ++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5a92071..1825705 100755 --- a/README.md +++ b/README.md @@ -252,6 +252,50 @@ The response is similar to that of editing a row, although again note that the r } ``` +### Initializing the index + +To programatically delete all data and initialize the index + +``` +POST /initialize +``` + +including the `schema` property in the payload defining the following structure + +``` +{ "fields": [ + { + "name": "id", + "type": "string", + "example": "example_id", + "facet": true + }, + { + "name": "score", + "type": "number", + "example": 8, + "facet": false + }, + { + "name": "tags", + "type": "arrayofstrings", + "example": "example_tag_1,example_tag_2", + "facet": true + } + ] +} + +> This example defines a schema containing three fields of which two will be enabled for faceted search. + +``` +Valid values: + +* Property `name`: any string +* Property `type`: `number`, `boolean`, `string`, `arrayofstrings` (e.g. `val1,val2,val3`) +* Property `example`: any valid value for this `type` +* Property `facet`: `true` or `false` + + ## Privacy Notice The Simple Search Service web application includes code to track deployments to Bluemix and other Cloud Foundry platforms. The following information is sent to a [Deployment Tracker](https://github.com/IBM-Bluemix/cf-deployment-tracker-service) service on each deployment: diff --git a/app.js b/app.js index b9b1a70..55cf8d7 100755 --- a/app.js +++ b/app.js @@ -147,6 +147,46 @@ app.post('/import', bodyParser, isloggedin.auth, function(req, res){ res.status(204).end(); }); +app.post('/initialize', bodyParser, isloggedin.auth, function(req, res){ + + if((!req.body) || (! req.body.schema)) { + return res.status(400).send('Schema definition missing.'); + } + + var theschema = null; + + try { + // parse schema + theschema = JSON.parse(req.body.schema); + // validate schema + // ... + } + catch(e) { + return res.status(400).send('Schema definition is invalid: ' + e); + } + + var cache = require('./lib/cache.js')(app.locals.cache); + if (cache) { + cache.clearAll(); + } + + // re-create index database + db.deleteAndCreate(function(err) { + if(err) { + return res.status(500).send('Index could not be re-initialized: ' + err); + } + // save schema definition + schema.save(theschema, function(err, d) { + if(err) { + return res.status(500).send('Schema could not be saved: ' + err); + } + console.log('schema saved',err,d); + return res.status(200).end(); + }); + }); +}); + + app.get('/import/status', isloggedin.auth, function(req, res) { var status = dbimport.status(); res.send(status); diff --git a/package.json b/package.json index 34e5c81..cbfb650 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "simple-search-service", - "version": "0.2.0", + "version": "0.2.1", "description": "A Node app that creates a faceted search engine, powered by IBM Cloudant", "scripts": { "start": "node app.js", From 6d1ac47f342335313f7add8b612c86c7877e7346 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Tue, 7 Feb 2017 14:19:42 -0800 Subject: [PATCH 2/3] validate schema spec --- app.js | 24 ++++++++---- internal-api-reference.md | 41 +++++++++++++++++++- lib/schema.js | 82 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 136 insertions(+), 11 deletions(-) diff --git a/app.js b/app.js index 55cf8d7..db6c1ce 100755 --- a/app.js +++ b/app.js @@ -147,10 +147,12 @@ app.post('/import', bodyParser, isloggedin.auth, function(req, res){ res.status(204).end(); }); +// Initialize the SSS index by deleting all data and loading the schema. +// Required request payload: a valid schema definition app.post('/initialize', bodyParser, isloggedin.auth, function(req, res){ if((!req.body) || (! req.body.schema)) { - return res.status(400).send('Schema definition missing.'); + return res.status(400).json({error: "Request rejected", reason: "Schema definition is missing."}); } var theschema = null; @@ -158,14 +160,20 @@ app.post('/initialize', bodyParser, isloggedin.auth, function(req, res){ try { // parse schema theschema = JSON.parse(req.body.schema); - // validate schema - // ... } - catch(e) { - return res.status(400).send('Schema definition is invalid: ' + e); + catch(err) { + console.error("/initialize: payload " + req.body.schema + " caused JSON parsing error: " + JSON.stringify(err)); + return res.status(400).json({error: "Request rejected", reason: "Schema definition is not valid JSON."}); } - var cache = require('./lib/cache.js')(app.locals.cache); + // validate schema + var validationErrors = schema.validateSchemaDef(theschema); + if(validationErrors) { + return res.status(400).json({error: "Request rejected", reason: "Schema validation failed. Errors: " + validationErrors.join("; ")}); + } + + + var cache = require("./lib/cache.js")(app.locals.cache); if (cache) { cache.clearAll(); } @@ -173,12 +181,12 @@ app.post('/initialize', bodyParser, isloggedin.auth, function(req, res){ // re-create index database db.deleteAndCreate(function(err) { if(err) { - return res.status(500).send('Index could not be re-initialized: ' + err); + return res.status(500).json({error: "Request failed", reason: "Index database could not be re-initialized: " + JSON.stringify(err)}); } // save schema definition schema.save(theschema, function(err, d) { if(err) { - return res.status(500).send('Schema could not be saved: ' + err); + return res.status(500).json({error: "Request failed", reason: "Schema could not be saved: " + JSON.stringify(err)}); } console.log('schema saved',err,d); return res.status(200).end(); diff --git a/internal-api-reference.md b/internal-api-reference.md index 963df60..a8876f0 100644 --- a/internal-api-reference.md +++ b/internal-api-reference.md @@ -178,4 +178,43 @@ where `"complete":true` indicates the completion of the import process. ## POST /deleteeverything -Delete the database and start again. \ No newline at end of file +Delete the database and start again. + + +## POST /initialize + +Delete the database and define schema. + +A form-encoded HTTP POST is expected to include a valid JSON payload describing the schema + +``` + "fields": [ + { + "name": "id", + "type": "string", + "facet": true + }, + { + "name": "tags", + "type": "arrayofstrings", + "facet": true + }, + ... + ] +``` + +Each field specification must define the [field] `name`, [field] `type` and `facet` properties. +> All property names are case sensitive. + +Valid values: + + * `name`: any unique string + * `type`: "`string`" || "`number`" || "`boolean`" || "`arrayofstrings`" (case sensitive) + * `facet`: `true` or `false` + +Return codes and responses: + +* `200` Request was successfully processed. +* `400` The schema definition is invalid. JSON response includes properties `error` and `reason`. +* `500` Request processing failed. JSON response includes properties `error` and `reason`. + diff --git a/lib/schema.js b/lib/schema.js index 7ae291e..31c4028 100644 --- a/lib/schema.js +++ b/lib/schema.js @@ -188,10 +188,88 @@ var validate = function(schema, row, editMode) { } }; - + +/** + * Determines whether a schema definition adheres to the specification. + * "fields" : [ + * { + * "name": "field_name" + * } + * ] + * @param {Object} schema - the schema definition to be validated + * @return {Array} null if valid or an array of Strings listing issues + */ +var validateSchemaDef = function(schema) { + + var errors = []; + + if((! schema) || ((! schema.hasOwnProperty("fields")))) { + errors.push("The schema must contain at least one field definition."); + return errors; + } + + if(! Array.isArray(schema.fields)) { + errors.push("The property named `fields` must define a non-empty array of field definitions."); + return errors; + } + + if(schema.fields.length < 1) { + errors.push("The property named `fields` must define a non-empty array of field definitions."); + return errors; + } + + const fieldSpecs = [ + {name: "name", type: "string", required: true}, + {name: "type", type: "string", required: true, values: ["string","number","boolean","arrayofstrings"]}, + {name: "facet", type: "boolean", required: true, values: [true, false]}, + {name: "example", type: "string", required: false} + ]; + + // iterate through all field definitions and perform property validation + var count = 1; + _.each(schema.fields, + function(field) { + _.each(fieldSpecs, + function(fieldSpec) { + if(! field.hasOwnProperty(fieldSpec.name)) { + if(fieldSpec.required) { + // required property is missing + errors.push("Field " + count + " - property `" + fieldSpec.name + "` is missing"); + } + } + else { + // validate data type of property value + if(typeof field[fieldSpec.name] !== fieldSpec.type) { + // the data type of the field's property value is invalid + errors.push("Field " + count + " - data type of property `" + fieldSpec.name + "` must be `" + fieldSpec.type + "`"); + } + else { + // validate property value + if((fieldSpec.hasOwnProperty('values')) && (! _.contains(fieldSpec.values, field[fieldSpec.name]))) { + // the data type of the field's property value is invalid + errors.push("Field " + count + " - value of property `" + fieldSpec.name + "` must be one of " + _.map(fieldSpec.values, function(element) { return ("`" + element + "`");}).join(",")); + } + } + } + }); + count++; + }); + + if(errors.length) { + // schema definition appears to be invalid; return error list + return errors; + } + else { + // schema definition appears to be valid + // return null to indicate that no errors were found + return null; + } +}; + module.exports = { load: load, save: save, generateSearchIndex: generateSearchIndex, - validate: validate + validate: validate, + validateSchemaDef: validateSchemaDef }; \ No newline at end of file From ee60b435cf03d56c823c330d8816415410a007da Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Wed, 8 Feb 2017 13:50:06 -0800 Subject: [PATCH 3/3] schema validation: bug fixes --- app.js | 19 +++++++++++-------- internal-api-reference.md | 9 +++++++-- lib/schema.js | 35 +++++++++++++++++++++++++---------- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/app.js b/app.js index db6c1ce..60924c5 100755 --- a/app.js +++ b/app.js @@ -124,8 +124,6 @@ app.post('/fetch', bodyParser, isloggedin.auth, function(req, res){ // import previously uploaded CSV app.post('/import', bodyParser, isloggedin.auth, function(req, res){ - console.log("****",req.body.schema); - console.log("****"); var currentUpload = app.locals.import[req.body.upload_id]; @@ -147,8 +145,10 @@ app.post('/import', bodyParser, isloggedin.auth, function(req, res){ res.status(204).end(); }); -// Initialize the SSS index by deleting all data and loading the schema. -// Required request payload: a valid schema definition +// Initialize the SSS index by deleting all data and saving the new schema +// Request parameters: +// - schema: JSON defining a valid schema (required) +// app.post('/initialize', bodyParser, isloggedin.auth, function(req, res){ if((!req.body) || (! req.body.schema)) { @@ -162,17 +162,18 @@ app.post('/initialize', bodyParser, isloggedin.auth, function(req, res){ theschema = JSON.parse(req.body.schema); } catch(err) { + // payload is not valid JSON; return client error console.error("/initialize: payload " + req.body.schema + " caused JSON parsing error: " + JSON.stringify(err)); return res.status(400).json({error: "Request rejected", reason: "Schema definition is not valid JSON."}); } - // validate schema + // validate schema definition var validationErrors = schema.validateSchemaDef(theschema); if(validationErrors) { + // payload is not valid schema definition; return client error return res.status(400).json({error: "Request rejected", reason: "Schema validation failed. Errors: " + validationErrors.join("; ")}); } - var cache = require("./lib/cache.js")(app.locals.cache); if (cache) { cache.clearAll(); @@ -181,14 +182,16 @@ app.post('/initialize', bodyParser, isloggedin.auth, function(req, res){ // re-create index database db.deleteAndCreate(function(err) { if(err) { + // index database could not be dropped/created; return server error return res.status(500).json({error: "Request failed", reason: "Index database could not be re-initialized: " + JSON.stringify(err)}); } // save schema definition - schema.save(theschema, function(err, d) { + schema.save(theschema, function(err) { if(err) { + // schema could not be saved; return server error return res.status(500).json({error: "Request failed", reason: "Schema could not be saved: " + JSON.stringify(err)}); } - console.log('schema saved',err,d); + // initialization complete; return OK return res.status(200).end(); }); }); diff --git a/internal-api-reference.md b/internal-api-reference.md index a8876f0..17c30cf 100644 --- a/internal-api-reference.md +++ b/internal-api-reference.md @@ -192,18 +192,22 @@ A form-encoded HTTP POST is expected to include a valid JSON payload describing { "name": "id", "type": "string", - "facet": true + "facet": true, + "example": "4a9f23" }, { "name": "tags", "type": "arrayofstrings", - "facet": true + "facet": true, + "example": "eins,zwei,drei" }, ... ] ``` Each field specification must define the [field] `name`, [field] `type` and `facet` properties. +The `example` property is optional. If set it should contain a valid value. + > All property names are case sensitive. Valid values: @@ -211,6 +215,7 @@ Valid values: * `name`: any unique string * `type`: "`string`" || "`number`" || "`boolean`" || "`arrayofstrings`" (case sensitive) * `facet`: `true` or `false` + * `example`: any string representing a valid value for the field Return codes and responses: diff --git a/lib/schema.js b/lib/schema.js index 31c4028..cd64863 100644 --- a/lib/schema.js +++ b/lib/schema.js @@ -190,34 +190,44 @@ var validate = function(schema, row, editMode) { }; /** - * Determines whether a schema definition adheres to the specification. + * Determines whether a schema definition adheres to the specification. Data structure: * "fields" : [ * { - * "name": "field_name" - * } + * "name": , + * "type": , + * "facet": , + * "example": + * }, + * ... * ] + * where one of ["string","number","boolean","arrayofstrings"] and + * one of [true, false]. All properties except example are required. * @param {Object} schema - the schema definition to be validated - * @return {Array} null if valid or an array of Strings listing issues + * @return {Array} null if the schema definition is valid or an array of Strings listing issues */ var validateSchemaDef = function(schema) { var errors = []; if((! schema) || ((! schema.hasOwnProperty("fields")))) { + // fatal schema definition error errors.push("The schema must contain at least one field definition."); return errors; } if(! Array.isArray(schema.fields)) { + // fatal schema definition error errors.push("The property named `fields` must define a non-empty array of field definitions."); return errors; } if(schema.fields.length < 1) { + // fatal schema definition error errors.push("The property named `fields` must define a non-empty array of field definitions."); return errors; } + // schema specification (property name, data type, required, values) const fieldSpecs = [ {name: "name", type: "string", required: true}, {name: "type", type: "string", required: true, values: ["string","number","boolean","arrayofstrings"]}, @@ -225,15 +235,17 @@ var validateSchemaDef = function(schema) { {name: "example", type: "string", required: false} ]; - // iterate through all field definitions and perform property validation - var count = 1; + // field counter + var count = 1; + // iterate through all field definitions and verify that they adhere to the specification; + // identify all issues before returning an error _.each(schema.fields, function(field) { _.each(fieldSpecs, function(fieldSpec) { if(! field.hasOwnProperty(fieldSpec.name)) { if(fieldSpec.required) { - // required property is missing + // a required property is missing errors.push("Field " + count + " - property `" + fieldSpec.name + "` is missing"); } } @@ -246,9 +258,13 @@ var validateSchemaDef = function(schema) { else { // validate property value if((fieldSpec.hasOwnProperty('values')) && (! _.contains(fieldSpec.values, field[fieldSpec.name]))) { - // the data type of the field's property value is invalid + // the property value is invalid errors.push("Field " + count + " - value of property `" + fieldSpec.name + "` must be one of " + _.map(fieldSpec.values, function(element) { return ("`" + element + "`");}).join(",")); } + // enforce constraint that no facet can be defined for fields with data type "number" + if((field.type === "number") && (field.facet)) { + errors.push("Field " + count + " - cannot facet field with data type `number`. Use data type `string` instead."); + } } } }); @@ -260,8 +276,7 @@ var validateSchemaDef = function(schema) { return errors; } else { - // schema definition appears to be valid - // return null to indicate that no errors were found + // schema definition appears to be valid; return null return null; } };