diff --git a/docs/docs/sql-syntax/objects.md b/docs/docs/sql-syntax/objects.md index 3e398ab..133f184 100644 --- a/docs/docs/sql-syntax/objects.md +++ b/docs/docs/sql-syntax/objects.md @@ -7,16 +7,16 @@ Specifying `$$ROOT` as a column alias sets the value to root object but only wor ???+ example "Example `$$ROOT` usage" ```sql - SELECT - t AS `$$ROOT` - FROM + SELECT + t AS `$$ROOT` + FROM ( - SELECT + SELECT id ,`First Name` ,`Last Name` - ,LENGTHOFARRAY(Rentals,'id') AS numRentals - FROM customers) + ,LENGTHOFARRAY(Rentals,'id') AS numRentals + FROM customers) AS t ``` @@ -27,9 +27,9 @@ Only available in aggregates. Use a `SELECT` without specifying a table to creat ???+ example "Creating a new object" ```sql - SELECT - (SELECT id,`First Name` AS Name) AS t - FROM + SELECT + (SELECT id,`First Name` AS Name) AS t + FROM customers ``` @@ -38,18 +38,16 @@ Create a new Object and assign to root ???+ example "Creating a new object and assigning to root" ```sql - SELECT + SELECT (SELECT id,`First Name` AS Name) AS t1 ,(SELECT id,`Last Name` AS LastName) AS t2 - ,MERGE_OBJECTS(t1,t2) AS `$$ROOT` - FROM + ,MERGE_OBJECTS(t1,t2) AS `$$ROOT` + FROM customers ``` ## Supported Object Functions - - ### PARSE_JSON `PARSE_JSON(expr)` @@ -59,11 +57,12 @@ Parses the JSON string. Use in conjunction with `ARRAY_TO_OBJECT` to convert an ???+ example "Example `PARSE_JSON` usage" ```sql - SELECT + SELECT id, ARRAY_TO_OBJECT(PARSE_JSON('[{"k":"val","v":1}]')) AS test FROM `customers`; ``` + ### MERGE_OBJECTS `MERGE_OBJECTS(expr)` @@ -71,7 +70,7 @@ Parses the JSON string. Use in conjunction with `ARRAY_TO_OBJECT` to convert an ???+ example "Example `MERGE_OBJECTS` usage" ```sql - SELECT + SELECT id, MERGE_OBJECTS(`Address`,PARSE_JSON('{"val":1}')) AS test FROM `customers`; @@ -80,7 +79,7 @@ Parses the JSON string. Use in conjunction with `ARRAY_TO_OBJECT` to convert an ???+ example "Example `MERGE_OBJECTS` usage with sub select" ```sql - SELECT + SELECT id, MERGE_OBJECTS(`Address`,(SELECT 1 AS val)) AS test FROM `customers`; @@ -115,27 +114,28 @@ Creates an empty object. FROM `customers`; ``` -[//]: # (todo add back when flatten implemented) -[//]: # (### FLATTEN) - -[//]: # () -[//]: # (`FLATTEN(field, prefix)`) - -[//]: # () -[//]: # (Flattens an object into a set of fields.) +### FLATTEN -[//]: # () -[//]: # (???+ example "Example `FLATTEN` usage") +`FLATTEN(field, prefix)` -[//]: # () -[//]: # ( ```sql) +Flattens an object into a set of fields. You can optionally add a -[//]: # ( SELECT) +???+ example "Example `FLATTEN` usage" -[//]: # ( id,) +```sql' + SELECT + id, + FLATTEN(`address`,'addr_') + FROM `customers`; +``` -[//]: # ( FLATTEN(`address`,'addr_')) +???+ example "Example `FLATTEN` usage with unset" -[//]: # ( FROM `customers`;) +```sql' + SELECT + id, + FLATTEN(`address`,'addr_',true) + FROM `customers`; +``` -[//]: # ( ```) +> Will remove the `address` field from the output and will only have the `addr_` prefixed fields. diff --git a/lib/MongoFunctions.js b/lib/MongoFunctions.js index 625d781..c8dd6a5 100644 --- a/lib/MongoFunctions.js +++ b/lib/MongoFunctions.js @@ -2586,103 +2586,120 @@ class AllowableFunctions { }, }, // todo add flatten back, has a error with no as which we're handling for unset - // { - // name: 'flatten', - // allowQuery: false, - // parse: (parameters) => { - // // todo fix return type - // - // if (!$check.array(parameters)) - // throw new Error('Invalid parameters for flatten'); - // if (parameters.length !== 2) { - // throw new Error( - // `Invalid parameter length for flatten, should be two but was ${parameters.length}` - // ); - // } - // const field = parameters[0]; - // const prefix = parameters[1]; - // if ($check.emptyString(field) || !$check.string(field)) { - // throw new Error( - // `The first parameter passed to flatten should be a non empty string but was ${field}` - // ); - // } - // if ($check.emptyString(prefix) || !$check.string(prefix)) { - // throw new Error( - // `The second parameter passed to flatten should be a non empty string but was ${prefix}` - // ); - // } - // // todo decide how to unset or if we should? Leave to user? - // - // if (field.indexOf('.') > -1) { - // const fieldParts = field.split('.'); - // const fieldName = fieldParts - // .slice(0, fieldParts.length - 1) - // .join('.'); - // - // return { - // $set: { - // [fieldName]: { - // $mergeObjects: [ - // '$' + fieldName, - // { - // $arrayToObject: { - // $map: { - // input: { - // $objectToArray: - // '$' + field, - // }, - // as: 'temp_flatten', - // in: { - // k: { - // $concat: [ - // prefix, - // '$$temp_flatten.k', - // ], - // }, - // v: '$$temp_flatten.v', - // }, - // }, - // }, - // }, - // ], - // }, - // }, - // }; - // } else { - // return { - // $replaceRoot: { - // newRoot: { - // $mergeObjects: [ - // '$$ROOT', - // { - // $arrayToObject: { - // $map: { - // input: { - // $objectToArray: - // '$' + field, - // }, - // as: 'temp_flatten', - // in: { - // k: { - // $concat: [ - // prefix, - // '$$temp_flatten.k', - // ], - // }, - // v: '$$temp_flatten.v', - // }, - // }, - // }, - // }, - // ], - // }, - // }, - // }; - // } - // }, - // requiresAs: false, - // jsonSchemaReturnType: 'null', - // }, + { + name: 'flatten', + allowQuery: true, + parse: (parameters) => { + // todo fix return type + + if (!$check.array(parameters)) + throw new Error( + 'Invalid parameters for flatten, should be an array' + ); + if (parameters.length < 2) { + throw new Error( + `Invalid parameter length for flatten, should be two but was ${parameters.length}` + ); + } + const unset = !!AllowableFunctions._getLiteral( + parameters[2] + ); + let field = parameters[0]; + if (field.startsWith('$')) { + field = field.substring(1); + } + const prefix = AllowableFunctions._getLiteral( + parameters[1] + ); + if ($check.emptyString(field) || !$check.string(field)) { + throw new Error( + `The first parameter passed to flatten should be a non empty string but was ${field}` + ); + } + if ($check.emptyString(prefix) || !$check.string(prefix)) { + throw new Error( + `The second parameter passed to flatten should be a non empty string but was ${prefix}` + ); + } + + // todo decide how to unset or if we should? Leave to user? + + if (field.indexOf('.') > -1) { + const fieldParts = field.split('.'); + const fieldName = fieldParts + .slice(0, fieldParts.length - 1) + .join('.'); + + const result = { + $set: { + [fieldName]: { + $mergeObjects: [ + '$' + fieldName, + { + $arrayToObject: { + $map: { + input: { + $objectToArray: + '$' + field, + }, + as: 'temp_flatten', + in: { + k: { + $concat: [ + prefix, + '$$temp_flatten.k', + ], + }, + v: '$$temp_flatten.v', + }, + }, + }, + }, + ], + }, + }, + }; + if (unset) { + result.unsetAfterReplaceOrSet = {$unset: [field]}; + } + return result; + } + const result = { + $replaceRoot: { + newRoot: { + $mergeObjects: [ + '$$ROOT', + { + $arrayToObject: { + $map: { + input: { + $objectToArray: '$' + field, + }, + as: 'temp_flatten', + in: { + k: { + $concat: [ + prefix, + '$$temp_flatten.k', + ], + }, + v: '$$temp_flatten.v', + }, + }, + }, + }, + ], + }, + }, + }; + if (unset) { + result.unsetAfterReplaceOrSet = {$unset: [field]}; + } + return result; + }, + requiresAs: false, + jsonSchemaReturnType: 'null', + }, /* #endregion */ // separate action diff --git a/lib/make/createResultObject.js b/lib/make/createResultObject.js index eb5d992..2e104de 100644 --- a/lib/make/createResultObject.js +++ b/lib/make/createResultObject.js @@ -19,6 +19,8 @@ function createResultObject() { countDistinct: null, windowFields: [], subQueryRootProjections: [], + set: null, + unsetAfterReplaceOrSet: null, }; } diff --git a/lib/make/makeAggregatePipeline.js b/lib/make/makeAggregatePipeline.js index 967aa79..68fd354 100644 --- a/lib/make/makeAggregatePipeline.js +++ b/lib/make/makeAggregatePipeline.js @@ -452,6 +452,12 @@ function makeAggregatePipeline(ast, context = {}) { if (result.count.length > 0) { result.count.forEach((countStep) => pipeline.push(countStep)); } + if (result.set) { + pipeline.push(result.set); + if (result.unsetAfterReplaceOrSet) { + pipeline.push(result.unsetAfterReplaceOrSet); + } + } if (result.unset) { pipeline.push(result.unset); } @@ -555,6 +561,9 @@ function makeAggregatePipeline(ast, context = {}) { if (result.replaceRoot) { pipeline.push(result.replaceRoot); + if (result.unsetAfterReplaceOrSet) { + pipeline.push(result.unsetAfterReplaceOrSet); + } } if (result.unwind && result.unwind.length > 0) { diff --git a/lib/make/projectColumnParser.js b/lib/make/projectColumnParser.js index b5deeaa..f2df51a 100644 --- a/lib/make/projectColumnParser.js +++ b/lib/make/projectColumnParser.js @@ -5,6 +5,7 @@ const makeCastPartModule = require('./makeCastPart'); const makeObjectFromSelectModule = require('./makeObjectFromSelect'); const makeProjectionExpressionPartModule = require('./makeProjectionExpressionPart'); const $check = require('check-types'); +const {functionByName} = require('../MongoFunctions'); exports.projectColumnParser = projectColumnParser; @@ -94,7 +95,13 @@ function projectColumnParser(column, result, context, tableAlias = '') { column.expr, context ); - result.unset = parsedExpr; + if (!result.unset) { + result.unset = parsedExpr; + } else if (result.unset.$unset && result.unset.$unset.length) { + result.unset.$unset.push(...parsedExpr.$unset); + } else { + result.unset = parsedExpr; + } return; } if (column.expr.type === 'function' && column.as) { @@ -193,7 +200,50 @@ function projectColumnParser(column, result, context, tableAlias = '') { result.parsedProject.$project[column.as] = `$${column.expr.value}`; return; } - + if (column.expr.type === 'function' && !column.as) { + const fn = functionByName(column.expr.name || column.expr.value); + if (fn.requiresAs) { + throw new Error( + `Require as for calculation:${ + column.expr.name || column.expr.value + }` + ); + } + const parsedExpr = + makeProjectionExpressionPartModule.makeProjectionExpressionPart( + column.expr, + context + ); + let applied = 0; + if (parsedExpr.$replaceRoot) { + result.replaceRoot = {$replaceRoot: parsedExpr.$replaceRoot}; + applied++; + } + if (parsedExpr.$unset) { + result.unset = {$unset: parsedExpr.$unset}; + applied++; + } + if (parsedExpr.$set) { + result.set = {$set: parsedExpr.$set}; + applied++; + } + if (parsedExpr.unsetAfterReplaceOrSet) { + result.unsetAfterReplaceOrSet = parsedExpr.unsetAfterReplaceOrSet; + applied++; + } + const keys = Object.keys(parsedExpr); + if (applied === 0) { + throw new Error( + `Logic Not implemented for function that does not requireAs and returned ${keys.join(', ')}` + ); + } + if (keys.length !== applied) { + throw new Error( + `Logic only partially implemented for function that does not requireAs and returned ${keys.join(', ')}` + ); + } + return; + } if (!column.as) { throw new Error( `Require as for calculation:${ diff --git a/lib/types.ts b/lib/types.ts index d200231..7ff9c0a 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -236,10 +236,12 @@ export interface ColumnParseResult { exprToMerge: (string | {[key: string]: string | {$literal: string}})[]; count: {$count: string}[]; unset: {$unset: string[]}; + set: {$set: any}; countDistinct: string; groupByProject?: object; windowFields: SetWindowFields[]; subQueryRootProjections: string[]; + unsetAfterReplaceOrSet: {$unset: string[]}; } export interface MongoQueryFunction { diff --git a/package.json b/package.json index a959bd2..292c0df 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@synatic/noql", - "version": "4.2.9", + "version": "4.2.10", "description": "Convert SQL statements to mongo queries or aggregates", "main": "index.js", "files": [ diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index d9d4d83..6b82ab5 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -313,5 +313,107 @@ } ] } + }, + "flatten": { + "case1": { + "expectedResults": [ + { + "First Name": "MARY", + "Address": { + "Address": "1913 Hanoi Way", + "City": "Sasebo", + "Country": "Japan", + "District": "Nagasaki" + }, + "Address_Address": "1913 Hanoi Way", + "Address_City": "Sasebo", + "Address_Country": "Japan", + "Address_District": "Nagasaki" + } + ] + }, + "case2": { + "expectedResults": [ + { + "First Name": "MARY", + "Address_Address": "1913 Hanoi Way", + "Address_City": "Sasebo", + "Address_Country": "Japan", + "Address_District": "Nagasaki" + } + ] + }, + "case3": { + "expectedResults": [ + { + "jsonObjValues": { + "string": "Testing", + "integer": 1, + "boolean": true, + "number": 1.5, + "nested": { + "string": "Testing", + "secondLevel": { + "string": "Testing" + } + }, + "nestedStrArray": [ + "A", + "b" + ], + "nestedObjArray": [ + { + "string": "Testing", + "integer": 1 + }, + { + "string": "", + "integer": 0 + }, + {} + ], + "stringOrInt": "asd", + "nested_string": "Testing", + "nested_secondLevel": { + "string": "Testing" + } + }, + "testId": "bugfix.schema-aware-queries.cast-json-array-to-varchar.case1" + } + ] + }, + "case4": { + "expectedResults": [ + { + "jsonObjValues": { + "string": "Testing", + "integer": 1, + "boolean": true, + "number": 1.5, + "nestedStrArray": [ + "A", + "b" + ], + "nestedObjArray": [ + { + "string": "Testing", + "integer": 1 + }, + { + "string": "", + "integer": 0 + }, + {} + ], + "stringOrInt": "asd", + "nested_string": "Testing", + "nested_secondLevel": { + "string": "Testing" + } + }, + "testId": "bugfix.schema-aware-queries.cast-json-array-to-varchar.case1" + } + ] + } } } \ No newline at end of file diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index 20755ac..97c13f9 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -770,4 +770,57 @@ describe('node-sql-parser upgrade tests', function () { }); }); }); + + describe('flatten', () => { + it('should flatten an object but keep the nested object by default', async () => { + const queryString = ` + SELECT "First Name", + "Address", + flatten(Address,'Address_'), + unset(Rentals) + FROM customers + WHERE id=1`; + await queryResultTester({ + queryString: queryString, + casePath: 'flatten.case1', + mode, + }); + }); + it('should flatten an object and unset the nested object if specified', async () => { + const queryString = ` + SELECT "First Name", + "Address", + flatten(Address,'Address_',true), + unset(Rentals) + FROM customers + WHERE id=1`; + await queryResultTester({ + queryString: queryString, + casePath: 'flatten.case2', + mode, + }); + }); + it('should flatten an objects nested property', async () => { + const queryString = ` + SELECT flatten(\`jsonObjValues.nested\`,'nested_'), unset(stringArray,numberArray,jsonArray,mixedArray,mixedPrimitiveArray,objOrArray,stringOrObject,commaTest,testCategory) + FROM "function-test-data" + WHERE testId='bugfix.schema-aware-queries.cast-json-array-to-varchar.case1'`; + await queryResultTester({ + queryString: queryString, + casePath: 'flatten.case3', + mode, + }); + }); + it('should flatten an objects nested property with unset', async () => { + const queryString = ` + SELECT flatten(\`jsonObjValues.nested\`,'nested_',true), unset(stringArray,numberArray,jsonArray,mixedArray,mixedPrimitiveArray,objOrArray,stringOrObject,commaTest,testCategory) + FROM "function-test-data" + WHERE testId='bugfix.schema-aware-queries.cast-json-array-to-varchar.case1'`; + await queryResultTester({ + queryString: queryString, + casePath: 'flatten.case4', + mode, + }); + }); + }); });