From 66e0b3929143b3d51305597b11e9a885fa152f8a Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Mon, 26 Aug 2024 15:16:55 +0200 Subject: [PATCH 01/18] adding start of full outer join --- test/exampleData/foj-customers.json | 14 ++++++++++++++ test/exampleData/foj-orders.json | 20 ++++++++++++++++++++ test/joins/joins.test.js | 20 ++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 test/exampleData/foj-customers.json create mode 100644 test/exampleData/foj-orders.json diff --git a/test/exampleData/foj-customers.json b/test/exampleData/foj-customers.json new file mode 100644 index 00000000..ce19c7ab --- /dev/null +++ b/test/exampleData/foj-customers.json @@ -0,0 +1,14 @@ +[ + { + "customerId": 1, + "customerName": "Alfreds Futterkiste" + }, + { + "customerId": 2, + "customerName": "Ana Trujillo Emparedados y helados" + }, + { + "customerId": 3, + "customerName": "Antonio Moreno Taquería" + } +] diff --git a/test/exampleData/foj-orders.json b/test/exampleData/foj-orders.json new file mode 100644 index 00000000..94572116 --- /dev/null +++ b/test/exampleData/foj-orders.json @@ -0,0 +1,20 @@ +[ + { + "orderId": 10308, + "customerId": 2, + "employeeId": 7, + "shipperId": 3 + }, + { + "orderId": 10309, + "customerId": 37, + "employeeId": 3, + "shipperId": 1 + }, + { + "orderId": 10310, + "customerId": 77, + "employeeId": 8, + "shipperId": 2 + } +] diff --git a/test/joins/joins.test.js b/test/joins/joins.test.js index f07678e7..0b4c660c 100644 --- a/test/joins/joins.test.js +++ b/test/joins/joins.test.js @@ -145,6 +145,7 @@ describe('joins', function () { }); }); }); + describe('left join', () => { it('should be able to do a left join', async () => { await queryResultTester({ @@ -608,6 +609,7 @@ describe('joins', function () { }); }); }); + describe('optimize', () => { const expectedPipeline = [ { @@ -689,4 +691,22 @@ describe('joins', function () { assert.deepStrictEqual(pipeline, expectedPipeline); }); }); + + describe('full outer join', () => { + it.skip('should work', async () => { + const queryString = ` + SELECT c.customerName as customerName, + o.orderId as orderId, + unset(_id) + FROM "foj-customers" c + FULL OUTER JOIN "foj-orders" o + ON c.customerId = o.customerId + ORDER BY c.customerName DESC`; + const {pipeline} = await queryResultTester({ + queryString, + casePath: 'full-outer-join.case1', + mode: 'write', + }); + }); + }); }); From f466a69d24e49c1f2ff7d15f9489100326fb8504 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Mon, 26 Aug 2024 20:44:47 +0200 Subject: [PATCH 02/18] adding expected pipeline --- test/individualTests/upgrade.json | 2 +- test/joins/joins.test.js | 65 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index 3275d514..3376bc51 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -98,4 +98,4 @@ } } } -} +} \ No newline at end of file diff --git a/test/joins/joins.test.js b/test/joins/joins.test.js index 0b4c660c..403ba52e 100644 --- a/test/joins/joins.test.js +++ b/test/joins/joins.test.js @@ -707,6 +707,71 @@ describe('joins', function () { casePath: 'full-outer-join.case1', mode: 'write', }); + const expectedPipeline = [ + { + $lookup: { + from: 'foj-customers', + localField: 'customerId', + foreignField: 'customerId', + as: 'customers', + }, + }, + { + $unwind: { + path: '$customers', + preserveNullAndEmptyArrays: true, + }, + }, + { + $unionWith: { + coll: 'foj-customers', + pipeline: [ + { + $lookup: { + from: 'foj-orders', + localField: 'customerId', + foreignField: 'customerId', + as: 'orders', + }, + }, + ], + }, + }, + { + $unwind: { + path: '$orders', + preserveNullAndEmptyArrays: true, + }, + }, + { + $project: { + customerName: { + $ifNull: [ + '$customerName', + '$customers.customerName', + ], + }, + orderId: { + $ifNull: ['$orderId', '$orders.orderId'], + }, + }, + }, + { + $group: { + _id: { + customerName: '$customerName', + orderId: '$orderId', + }, + }, + }, + { + $project: { + _id: 0, + customerName: '$_id.customerName', + orderId: '$_id.orderId', + }, + }, + ]; }); }); }); From 32decc1b2532bf9cdf520df30c234801ec1cbb07 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Tue, 27 Aug 2024 08:40:26 +0200 Subject: [PATCH 03/18] Checking in the full join for now --- lib/make/makeJoinForPipeline.js | 28 ++++++++++++++++++++++++++++ test/joins/joins.test.js | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/make/makeJoinForPipeline.js b/lib/make/makeJoinForPipeline.js index b29a2049..23b82294 100644 --- a/lib/make/makeJoinForPipeline.js +++ b/lib/make/makeJoinForPipeline.js @@ -354,6 +354,34 @@ function tableJoin( } } else if (join.join === 'LEFT JOIN') { // dont need anything + } else if (join.join === 'FULL JOIN') { + pipeline.push({ + $unwind: { + path: `$${localPart.table}`, + preserveNullAndEmptyArrays: true, + }, + }); + pipeline.push({ + $unionWith: { + coll: join.table, + pipeline: [ + { + $lookup: { + from: toTable, + localField: localPart.column, + foreignField: fromPart.column, + as: fromPart.table, + }, + }, + ], + }, + }); + pipeline.push({ + $unwind: { + path: `$${fromPart.table}`, + preserveNullAndEmptyArrays: true, + }, + }); } else { throw new Error(`Join not supported:${join.join}`); } diff --git a/test/joins/joins.test.js b/test/joins/joins.test.js index 403ba52e..47120ac7 100644 --- a/test/joins/joins.test.js +++ b/test/joins/joins.test.js @@ -693,7 +693,7 @@ describe('joins', function () { }); describe('full outer join', () => { - it.skip('should work', async () => { + it('should work', async () => { const queryString = ` SELECT c.customerName as customerName, o.orderId as orderId, From 76290ece6fe2bbdc89117edf24c6e74deeb463db Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Tue, 27 Aug 2024 11:44:10 +0200 Subject: [PATCH 04/18] checking in latest update to full join --- lib/make/makeJoinForPipeline.js | 6 --- test/joins/join-cases.json | 81 +++++++++++++++++++++++++++++++++ test/joins/joins.test.js | 4 +- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/lib/make/makeJoinForPipeline.js b/lib/make/makeJoinForPipeline.js index 23b82294..454c2684 100644 --- a/lib/make/makeJoinForPipeline.js +++ b/lib/make/makeJoinForPipeline.js @@ -355,12 +355,6 @@ function tableJoin( } else if (join.join === 'LEFT JOIN') { // dont need anything } else if (join.join === 'FULL JOIN') { - pipeline.push({ - $unwind: { - path: `$${localPart.table}`, - preserveNullAndEmptyArrays: true, - }, - }); pipeline.push({ $unionWith: { coll: join.table, diff --git a/test/joins/join-cases.json b/test/joins/join-cases.json index 38bf5a40..08e6dd87 100644 --- a/test/joins/join-cases.json +++ b/test/joins/join-cases.json @@ -1778,5 +1778,86 @@ } ] } + }, + "full-outer-join": { + "case1": { + "pipeline": [ + { + "$project": { + "c": "$$ROOT" + } + }, + { + "$lookup": { + "from": "foj-orders", + "as": "o", + "localField": "c.customerId", + "foreignField": "customerId" + } + }, + { + "$unionWith": { + "coll": "foj-orders", + "pipeline": [ + { + "$lookup": { + "from": "foj-orders", + "localField": "customerId", + "foreignField": "customerId", + "as": "o" + } + } + ] + } + }, + { + "$unwind": { + "path": "$o", + "preserveNullAndEmptyArrays": true + } + }, + { + "$group": { + "_id": { + "customerName": "$c.customerName", + "orderId": "$o.orderId" + } + } + }, + { + "$project": { + "customerName": "$_id.customerName", + "orderId": "$_id.orderId", + "_id": 0 + } + }, + { + "$sort": { + "customerName": -1 + } + } + ], + "expectedResults": [ + { + "customerName": "Antonio Moreno Taquería" + }, + { + "customerName": "Ana Trujillo Emparedados y helados", + "orderId": 10308 + }, + { + "customerName": "Alfreds Futterkiste" + }, + { + "orderId": 10308 + }, + { + "orderId": 10309 + }, + { + "orderId": 10310 + } + ] + } } } \ No newline at end of file diff --git a/test/joins/joins.test.js b/test/joins/joins.test.js index 47120ac7..1981a467 100644 --- a/test/joins/joins.test.js +++ b/test/joins/joins.test.js @@ -696,8 +696,7 @@ describe('joins', function () { it('should work', async () => { const queryString = ` SELECT c.customerName as customerName, - o.orderId as orderId, - unset(_id) + o.orderId as orderId FROM "foj-customers" c FULL OUTER JOIN "foj-orders" o ON c.customerId = o.customerId @@ -706,6 +705,7 @@ describe('joins', function () { queryString, casePath: 'full-outer-join.case1', mode: 'write', + outputPipeline: true, }); const expectedPipeline = [ { From bbfc96b8800bca27c5d447a47806b8e0514df9ff Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Tue, 3 Sep 2024 14:57:22 +0200 Subject: [PATCH 05/18] feat(pivot): adding logic and tests --- .cursorrules | 21 +++ lib/make/apply-pivot.js | 153 ++++++++++++++++++ lib/make/makeAggregatePipeline.js | 28 ++-- test/exampleData/Production_Product.json | 6 + .../Purchasing_PurchaseOrderHeader.json | 12 ++ test/individualTests/upgrade.json | 25 +++ test/individualTests/upgrade.test.js | 71 +++++++- 7 files changed, 304 insertions(+), 12 deletions(-) create mode 100644 .cursorrules create mode 100644 lib/make/apply-pivot.js create mode 100644 test/exampleData/Production_Product.json create mode 100644 test/exampleData/Purchasing_PurchaseOrderHeader.json diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 00000000..94f8df90 --- /dev/null +++ b/.cursorrules @@ -0,0 +1,21 @@ +# Original instructions: https://forum.cursor.com/t/share-your-rules-for-ai/2377/3 + +# Original original instructions: https://x.com/NickADobos/status/1814596357879177592 + +You are an expert AI programming assistant that primarily focuses on producing clear, readable Node.js and MongoDB code. + +You always use the latest version of Node.js, and MongoDB, and you are familiar with the latest features and best practices. + +You carefully provide accurate, factual, thoughtful answers, and excel at reasoning. + +This project is a translator library that takes in a SQL dialect called `noql` documented in the `/docs` folder, the objective of the library is to use the provided SQL statement to generate a valid and performant mongodb aggregation pipeline that can be executed to get data. + +- Follow the user’s requirements carefully & to the letter. +- First think step-by-step - describe your plan for what to build in javascript, written out in great detail. +- Confirm, then write code! +- Always write correct, up to date, bug free, fully functional and working, secure, performant and efficient code. +- Focus on readability over being performant. +- Fully implement all requested functionality. +- Leave NO todo’s, placeholders or missing pieces. +- Be concise. Minimize any other prose. +- If you think there might not be a correct answer, you say so. If you do not know the answer, say so instead of guessing. diff --git a/lib/make/apply-pivot.js b/lib/make/apply-pivot.js new file mode 100644 index 00000000..8f6d5ac3 --- /dev/null +++ b/lib/make/apply-pivot.js @@ -0,0 +1,153 @@ +const {functionByName} = require('../MongoFunctions'); +module.exports = {applyPivot}; + +/** + * + * @param {string} pivotString + * @param {import('../types').PipelineFn[]} pipeline + * @param {import('../types').NoqlContext} context + */ +function applyPivot(pivotString, pipeline, context) { + const pivot = createJSONFromPivotString(pivotString); + + pivot.fields = pivot.fields.map((field) => { + const functionName = field.split('(')[0]; + const foundFunction = functionByName(functionName); + if (!foundFunction) { + throw new Error( + `Unable to find function "${functionName}" in pivot.fields."` + ); + } + let argumentString = field.replace(functionName, ''); + let name = ''; + if (argumentString.indexOf(' as ') >= 0) { + const parts = argumentString.split(' as '); + argumentString = parts[0]; + name = parts[1]; + } + argumentString = argumentString.substring(1, argumentString.length - 1); + const rawArguments = argumentString.split(','); + if (!name) { + name = rawArguments[0]; + } + const parsedArguments = rawArguments.map((arg) => + isNumeric(arg) ? parseFloat(arg) : `$${arg}` + ); + return { + name, + foundFunction, + parsedArguments, + }; + }); + pipeline.push({ + $group: { + _id: `$${pivot.for}`, + ...pivot.fields.reduce((previousValue, currentValue) => { + const res = currentValue.foundFunction.parse( + ...currentValue.parsedArguments + ); + previousValue[currentValue.name] = res; + return previousValue; + }, {}), + }, + }); + pipeline.push({ + $group: { + _id: null, + data: { + $push: { + k: { + $toString: '$_id', + }, + v: { + ...pivot.fields.reduce( + (previousValue, currentValue) => { + previousValue[currentValue.name] = + `$${currentValue.name}`; + return previousValue; + }, + {} + ), + }, + }, + }, + }, + }); + pipeline.push({ + $project: { + _id: 0, + data: { + $arrayToObject: '$data', + }, + }, + }); + pipeline.push({ + $project: { + result: { + $mergeObjects: [ + { + ...pivot.columns.reduce( + (previousValue, currentValue) => { + previousValue[currentValue] = null; + return previousValue; + }, + {} + ), + }, + '$data', + ], + }, + }, + }); + pipeline.push({ + $replaceRoot: { + newRoot: '$result', + }, + }); +} + +/** + * + * @param {string} str + * @returns {boolean} + */ +function isNumeric(str) { + if (typeof str != 'string') { + return false; + } + return !isNaN(str) && !isNaN(parseFloat(str)); +} + +/** + * + * @param {string} inputString + * @returns {{columns: (*|*[]), for: (*|string), fields: *[]}} + */ +function createJSONFromPivotString(inputString) { + // Split the string by '|' + const parts = inputString.split('|'); + + // Extract the field from the pivot function + let pivotPart = parts[1]; + const fieldMatch = pivotPart.match(/pivot\(\[(.*?)\]/); + const fields = fieldMatch + ? fieldMatch[1].split(',').map((f) => f.trim()) + : []; + + pivotPart = pivotPart.replace(fieldMatch[0], ''); + // Extract the 'for' part + const forMatch = pivotPart.match(/,(.*?),\[/); + const forPart = forMatch ? forMatch[1] : ''; + + pivotPart = pivotPart.replace(forMatch[0], ''); + // Extract the columns + const columnsMatch = pivotPart.match(/(.*?)\]/); + const columns = columnsMatch ? columnsMatch[1].split(',').map(Number) : []; + + // Construct the JSON object + return { + fields: fields, + for: forPart, + columns: columns, + }; +} diff --git a/lib/make/makeAggregatePipeline.js b/lib/make/makeAggregatePipeline.js index a23797aa..8084cd48 100644 --- a/lib/make/makeAggregatePipeline.js +++ b/lib/make/makeAggregatePipeline.js @@ -15,7 +15,7 @@ const { const $copy = require('clone-deep'); const {optimizeJoinAndWhere} = require('./optimize-join-and-where'); - +const {applyPivot} = require('./apply-pivot'); exports.makeAggregatePipeline = makeAggregatePipeline; /** @@ -412,11 +412,22 @@ function makeAggregatePipeline(ast, context = {}) { if (!ast.from[0].as) { throw new Error(`AS not specified for initial sub query`); } - const tableAs = stripJoinHints(ast.from[0].as); - pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context) - .concat([{$project: {[tableAs]: '$$ROOT'}}]) - .concat(pipeline); + const as = ast.from[0].as; + const tableAs = stripJoinHints(as); + result.subQueryRootProjections.push(tableAs); + if (as.indexOf('|pivot(') >= 0) { + // const prevPipeline = pipeline; + pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context); + applyPivot(as, pipeline, context); + // pipeline = pipeline + // .concat([{$project: {[tableAs]: '$$ROOT'}}]) + // .concat(prevPipeline); + } else { + pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context) + .concat([{$project: {[tableAs]: '$$ROOT'}}]) + .concat(pipeline); + } } if (result.replaceRoot) { @@ -789,12 +800,7 @@ function handleExcept(ast, context, pipeline) { * @returns {string} */ function stripJoinHints(input) { - return input - .replace('|first', '') - .replace('|last', '') - .replace('|unwind', '') - .replace('|optimize', '') - .replace('|nooptimize', ''); + return input.split('|')[0]; } /** diff --git a/test/exampleData/Production_Product.json b/test/exampleData/Production_Product.json new file mode 100644 index 00000000..5f5a0c0e --- /dev/null +++ b/test/exampleData/Production_Product.json @@ -0,0 +1,6 @@ +[ + {"DaysToManufacture": 0, "StandardCost": 5.0885}, + {"DaysToManufacture": 1, "StandardCost": 223.88}, + {"DaysToManufacture": 2, "StandardCost": 359.1082}, + {"DaysToManufacture": 4, "StandardCost": 949.4105} +] diff --git a/test/exampleData/Purchasing_PurchaseOrderHeader.json b/test/exampleData/Purchasing_PurchaseOrderHeader.json new file mode 100644 index 00000000..456c6aa4 --- /dev/null +++ b/test/exampleData/Purchasing_PurchaseOrderHeader.json @@ -0,0 +1,12 @@ +[ + {"PurchaseOrderID": 1, "EmployeeID": 250, "VendorID": 1492}, + {"PurchaseOrderID": 2, "EmployeeID": 251, "VendorID": 1492}, + {"PurchaseOrderID": 3, "EmployeeID": 256, "VendorID": 1492}, + {"PurchaseOrderID": 4, "EmployeeID": 257, "VendorID": 1492}, + {"PurchaseOrderID": 5, "EmployeeID": 260, "VendorID": 1492}, + {"PurchaseOrderID": 6, "EmployeeID": 250, "VendorID": 1494}, + {"PurchaseOrderID": 7, "EmployeeID": 251, "VendorID": 1494}, + {"PurchaseOrderID": 8, "EmployeeID": 256, "VendorID": 1494}, + {"PurchaseOrderID": 9, "EmployeeID": 257, "VendorID": 1494}, + {"PurchaseOrderID": 10, "EmployeeID": 260, "VendorID": 1494} +] diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index 3376bc51..82036d4d 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -97,5 +97,30 @@ ] } } + }, + "pivot": { + "case1": { + "expectedResults": [ + { + "0": { + "StandardCost": 5.0885, + "MaxCost": 5.0885 + }, + "1": { + "StandardCost": 223.88, + "MaxCost": 223.88 + }, + "2": { + "StandardCost": 359.1082, + "MaxCost": 359.1082 + }, + "3": null, + "4": { + "StandardCost": 949.4105, + "MaxCost": 949.4105 + } + } + ] + } } } \ No newline at end of file diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index 39661f23..430f324b 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -392,9 +392,78 @@ describe('node-sql-parser upgrade tests', function () { await queryResultTester({ queryString: queryString, casePath: 'new.left.case1', - mode: 'write', + mode, outputPipeline: false, }); }); }); + describe('PIVOT and UNPIVOT', () => { + describe('PIVOT', () => { + it('should pivot DaysToManufacture to columns', async () => { + const queryText = ` + SELECT * + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot([avg(StandardCost), max(StandardCost) as MaxCost],DaysToManufacture,[0,1,2,3,4])' + `; + + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode: 'write', + outputPipeline: false, + }); + // const expected = { + // 0: 5.0885, + // 1: 223.88, + // 2: 359.1082, + // 3: null, + // 4: 949.4105, + // }; + }); + }); + + describe.skip('UNPIVOT', () => { + it('should unpivot employee columns to rows', async () => { + const queryText = ` + SELECT VendorID, Employee, Orders + FROM ( + SELECT VendorID, Emp1, Emp2, Emp3, Emp4, Emp5 + FROM ( + SELECT VendorID, + [250] AS Emp1, + [251] AS Emp2, + [256] AS Emp3, + [257] AS Emp4, + [260] AS Emp5 + FROM ( + SELECT PurchaseOrderID, + EmployeeID, VendorID + FROM Purchasing_PurchaseOrderHeader + ) p + PIVOT ( + COUNT(PurchaseOrderID) + FOR EmployeeID IN ([250], [251], [256], [257], [260]) + ) AS pvt + ) p + ) AS PivotTable + UNPIVOT ( + Orders FOR Employee IN (Emp1, Emp2, Emp3, Emp4, Emp5) + ) AS UnpivotTable + ORDER BY VendorID, Employee + `; + + await queryResultTester({ + queryString: queryText, + casePath: 'unpivot.case1', + mode, + outputPipeline: false, + }); + }); + }); + }); }); From 5a43ac6c1475a5212834e613c598200a9a051063 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Tue, 3 Sep 2024 15:35:28 +0200 Subject: [PATCH 06/18] feat(unpivot): start of logic --- lib/make/apply-pivot.js | 13 ++++- .../Purchasing_PurchaseOrderHeader.json | 12 ---- test/exampleData/pvt.json | 7 +++ test/individualTests/upgrade.json | 56 +++++++++++++++++++ test/individualTests/upgrade.test.js | 32 +++-------- 5 files changed, 82 insertions(+), 38 deletions(-) delete mode 100644 test/exampleData/Purchasing_PurchaseOrderHeader.json create mode 100644 test/exampleData/pvt.json diff --git a/lib/make/apply-pivot.js b/lib/make/apply-pivot.js index 8f6d5ac3..42dc9b9d 100644 --- a/lib/make/apply-pivot.js +++ b/lib/make/apply-pivot.js @@ -1,5 +1,5 @@ const {functionByName} = require('../MongoFunctions'); -module.exports = {applyPivot}; +module.exports = {applyPivot, applyUnpivot}; /** * @@ -151,3 +151,14 @@ function createJSONFromPivotString(inputString) { columns: columns, }; } + +/** + * + * @param {string} pivotString + * @param {import('../types').PipelineFn[]} pipeline + * @param {import('../types').NoqlContext} context + */ +function applyUnpivot(pivotString, pipeline, context) { + const unpivot = createJSONFromPivotString(pivotString); + console.log(unpivot); +} diff --git a/test/exampleData/Purchasing_PurchaseOrderHeader.json b/test/exampleData/Purchasing_PurchaseOrderHeader.json deleted file mode 100644 index 456c6aa4..00000000 --- a/test/exampleData/Purchasing_PurchaseOrderHeader.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - {"PurchaseOrderID": 1, "EmployeeID": 250, "VendorID": 1492}, - {"PurchaseOrderID": 2, "EmployeeID": 251, "VendorID": 1492}, - {"PurchaseOrderID": 3, "EmployeeID": 256, "VendorID": 1492}, - {"PurchaseOrderID": 4, "EmployeeID": 257, "VendorID": 1492}, - {"PurchaseOrderID": 5, "EmployeeID": 260, "VendorID": 1492}, - {"PurchaseOrderID": 6, "EmployeeID": 250, "VendorID": 1494}, - {"PurchaseOrderID": 7, "EmployeeID": 251, "VendorID": 1494}, - {"PurchaseOrderID": 8, "EmployeeID": 256, "VendorID": 1494}, - {"PurchaseOrderID": 9, "EmployeeID": 257, "VendorID": 1494}, - {"PurchaseOrderID": 10, "EmployeeID": 260, "VendorID": 1494} -] diff --git a/test/exampleData/pvt.json b/test/exampleData/pvt.json new file mode 100644 index 00000000..f2459806 --- /dev/null +++ b/test/exampleData/pvt.json @@ -0,0 +1,7 @@ +[ + {"VendorID": 1, "Emp1": 4, "Emp2": 3, "Emp3": 5, "Emp4": 4, "Emp5": 4}, + {"VendorID": 2, "Emp1": 4, "Emp2": 1, "Emp3": 5, "Emp4": 5, "Emp5": 5}, + {"VendorID": 3, "Emp1": 4, "Emp2": 3, "Emp3": 5, "Emp4": 4, "Emp5": 4}, + {"VendorID": 4, "Emp1": 4, "Emp2": 2, "Emp3": 5, "Emp4": 5, "Emp5": 4}, + {"VendorID": 5, "Emp1": 5, "Emp2": 1, "Emp3": 5, "Emp4": 5, "Emp5": 5} +] diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index 82036d4d..40b3fcfc 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -122,5 +122,61 @@ } ] } + }, + "unpivot": { + "case1": { + "expectedResults": [ + { + "pvt": { + "VendorID": 4, + "Emp1": 4, + "Emp2": 2, + "Emp3": 5, + "Emp4": 5, + "Emp5": 4 + } + }, + { + "pvt": { + "VendorID": 2, + "Emp1": 4, + "Emp2": 1, + "Emp3": 5, + "Emp4": 5, + "Emp5": 5 + } + }, + { + "pvt": { + "VendorID": 1, + "Emp1": 4, + "Emp2": 3, + "Emp3": 5, + "Emp4": 4, + "Emp5": 4 + } + }, + { + "pvt": { + "VendorID": 3, + "Emp1": 4, + "Emp2": 3, + "Emp3": 5, + "Emp4": 4, + "Emp5": 4 + } + }, + { + "pvt": { + "VendorID": 5, + "Emp1": 5, + "Emp2": 1, + "Emp3": 5, + "Emp4": 5, + "Emp5": 5 + } + } + ] + } } } \ No newline at end of file diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index 430f324b..e2083e7b 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -427,40 +427,22 @@ describe('node-sql-parser upgrade tests', function () { }); }); - describe.skip('UNPIVOT', () => { + describe('UNPIVOT', () => { it('should unpivot employee columns to rows', async () => { const queryText = ` - SELECT VendorID, Employee, Orders + SELECT * FROM ( SELECT VendorID, Emp1, Emp2, Emp3, Emp4, Emp5 - FROM ( - SELECT VendorID, - [250] AS Emp1, - [251] AS Emp2, - [256] AS Emp3, - [257] AS Emp4, - [260] AS Emp5 - FROM ( - SELECT PurchaseOrderID, - EmployeeID, VendorID - FROM Purchasing_PurchaseOrderHeader - ) p - PIVOT ( - COUNT(PurchaseOrderID) - FOR EmployeeID IN ([250], [251], [256], [257], [260]) - ) AS pvt - ) p - ) AS PivotTable - UNPIVOT ( - Orders FOR Employee IN (Emp1, Emp2, Emp3, Emp4, Emp5) - ) AS UnpivotTable - ORDER BY VendorID, Employee + FROM pvt + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'unpvt|unpivot([Orders],Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])' `; await queryResultTester({ queryString: queryText, casePath: 'unpivot.case1', - mode, + mode: 'write', outputPipeline: false, }); }); From 140a359bb3fda1cd4a02366c6a0aa1983d80f901 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Tue, 3 Sep 2024 19:54:15 +0200 Subject: [PATCH 07/18] Getting unpivot working --- lib/make/apply-pivot.js | 78 +++++++- lib/make/makeAggregatePipeline.js | 7 +- test/AAA_ALL_Profitability_Activities v2.sql | 189 +++++++++++++++++++ test/individualTests/upgrade.json | 155 +++++++++++---- test/individualTests/upgrade.test.js | 4 +- 5 files changed, 383 insertions(+), 50 deletions(-) create mode 100644 test/AAA_ALL_Profitability_Activities v2.sql diff --git a/lib/make/apply-pivot.js b/lib/make/apply-pivot.js index 42dc9b9d..aa423fe3 100644 --- a/lib/make/apply-pivot.js +++ b/lib/make/apply-pivot.js @@ -8,7 +8,7 @@ module.exports = {applyPivot, applyUnpivot}; * @param {import('../types').NoqlContext} context */ function applyPivot(pivotString, pipeline, context) { - const pivot = createJSONFromPivotString(pivotString); + const pivot = createJSONFromPivotString('pivot', pivotString); pivot.fields = pivot.fields.map((field) => { const functionName = field.split('(')[0]; @@ -120,16 +120,23 @@ function isNumeric(str) { /** * + * @param {'pivot'|'unpivot'} type * @param {string} inputString * @returns {{columns: (*|*[]), for: (*|string), fields: *[]}} */ -function createJSONFromPivotString(inputString) { +function createJSONFromPivotString(type, inputString) { // Split the string by '|' const parts = inputString.split('|'); // Extract the field from the pivot function let pivotPart = parts[1]; - const fieldMatch = pivotPart.match(/pivot\(\[(.*?)\]/); + let fieldMatch; + if (type === 'pivot') { + fieldMatch = pivotPart.match(/pivot\(\[(.*?)\]/); + } else { + fieldMatch = pivotPart.match(/unpivot\(\[(.*?)\]/); + } + const fields = fieldMatch ? fieldMatch[1].split(',').map((f) => f.trim()) : []; @@ -142,7 +149,9 @@ function createJSONFromPivotString(inputString) { pivotPart = pivotPart.replace(forMatch[0], ''); // Extract the columns const columnsMatch = pivotPart.match(/(.*?)\]/); - const columns = columnsMatch ? columnsMatch[1].split(',').map(Number) : []; + const columns = columnsMatch + ? columnsMatch[1].split(',').map((c) => c.trim()) + : []; // Construct the JSON object return { @@ -159,6 +168,63 @@ function createJSONFromPivotString(inputString) { * @param {import('../types').NoqlContext} context */ function applyUnpivot(pivotString, pipeline, context) { - const unpivot = createJSONFromPivotString(pivotString); - console.log(unpivot); + const unpivot = createJSONFromPivotString('unpivot', pivotString); + const projection = pipeline + .slice() + .reverse() + .find((p) => !!p.$project); + const columnsToExclude = [unpivot.for, '_id'].concat(unpivot.columns); + const columns = Object.keys(projection.$project).filter( + (val) => columnsToExclude.indexOf(val) === -1 + ); + pipeline.push({ + $project: { + ...columns.reduce((previousValue, currentValue) => { + previousValue[currentValue] = `$${currentValue}`; + return previousValue; + }, {}), + fields: { + $objectToArray: '$$ROOT', + }, + }, + }); + pipeline.push({ + $project: { + ...columns.reduce((previousValue, currentValue) => { + previousValue[currentValue] = 1; + return previousValue; + }, {}), + fields: { + $filter: { + input: '$fields', + as: 'field', + cond: { + $and: columns.map((c) => ({$ne: ['$$field.k', c]})), + }, + }, + }, + }, + }); + pipeline.push({ + $unwind: '$fields', + }); + pipeline.push({ + $project: { + ...columns.reduce((previousValue, currentValue) => { + previousValue[currentValue] = 1; + return previousValue; + }, {}), + [unpivot.for]: '$fields.k', + [unpivot.fields[0]]: '$fields.v', // Assuming 'Orders' should be numeric + }, + }); + pipeline.push({ + $sort: { + ...columns.reduce((previousValue, currentValue) => { + previousValue[currentValue] = 1; + return previousValue; + }, {}), + [unpivot.for]: 1, + }, + }); } diff --git a/lib/make/makeAggregatePipeline.js b/lib/make/makeAggregatePipeline.js index 8084cd48..234b03df 100644 --- a/lib/make/makeAggregatePipeline.js +++ b/lib/make/makeAggregatePipeline.js @@ -15,7 +15,7 @@ const { const $copy = require('clone-deep'); const {optimizeJoinAndWhere} = require('./optimize-join-and-where'); -const {applyPivot} = require('./apply-pivot'); +const {applyPivot, applyUnpivot} = require('./apply-pivot'); exports.makeAggregatePipeline = makeAggregatePipeline; /** @@ -345,7 +345,7 @@ function makeAggregatePipeline(ast, context = {}) { column, result, context, - ast.from && ast.from[0] ? ast.from[0].as : null + ast.from && ast.from[0] ? stripJoinHints(ast.from[0].as) : null ); }); if (result.count.length > 0) { @@ -423,6 +423,9 @@ function makeAggregatePipeline(ast, context = {}) { // pipeline = pipeline // .concat([{$project: {[tableAs]: '$$ROOT'}}]) // .concat(prevPipeline); + } else if (as.indexOf('|unpivot(') >= 0) { + pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context); + applyUnpivot(as, pipeline, context); } else { pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context) .concat([{$project: {[tableAs]: '$$ROOT'}}]) diff --git a/test/AAA_ALL_Profitability_Activities v2.sql b/test/AAA_ALL_Profitability_Activities v2.sql new file mode 100644 index 00000000..3bc1bfb1 --- /dev/null +++ b/test/AAA_ALL_Profitability_Activities v2.sql @@ -0,0 +1,189 @@ +SELECT DISTINCT + [Uniq Client ID], + [Lookup Code], + [Client Name], + UniqActivity, + ActivityCode, + [Activity Description], + [Who/Owner], + ID, + Event_Uniq, + Event_Date, + CONVERT(varchar, IIF(Event_Type = 1, 'Activity Created', + IIF(Event_Type = 2, 'Activity Closed', + IIF(Event_Type = 3, 'Note Created', + IIF(Event_Type = 4, 'Task Created', + IIF(Event_Type = 5, 'Attachment Created', '')))))) AS 'Event Type', + Event_Desc, + IIF(Event_Type = 5, FileExtension,'') AS 'File Type', + Activity_Status, + [Activity Dept], + [Branch], + [Profit Center], + [RiskAdvisor], + [AccountMgr], + [RACode], + [AMCode], + GETDATE() AS 'Last Refreshed' + +FROM + (SELECT + dbo.Client.UniqEntity AS 'Uniq Client ID', + dbo.Client.LookupCode AS 'Lookup Code', + dbo.Client.NameOf AS 'Client Name', + dbo.Activity.UniqActivity, + dbo.ActivityCode.ActivityCode, + IIF(dbo.Activity.UniqEmployee = -1,dbo.Workgroup.LookupCode,Activity_Owner.LookupCode) AS 'Who/Owner', + dbo.Activity.DescriptionOf AS 'Activity Description', + IIF(dbo.Activity.ClosedDate IS NULL, 'Open', 'Closed') AS Activity_Status, + IIF(dbo.Activity.UniqDepartment <> -1,Act_Dept.DepartmentCode,Pol_Dept.DepartmentCode) AS 'Activity Dept', + IIF(dbo.Activity.UniqBranch <> -1,Act_Branch.BranchCode,Pol_Branch.BranchCode) AS 'Branch', + IIF(dbo.Activity.UniqProfitCenter <> -1,Act_PC.ProfitCenterCode,Pol_PC.ProfitCenterCode) AS 'Profit Center', + + IIF(dbo.ActivityTask.StatusCode IS NULL, NULL, + IIF(dbo.ActivityTask.StatusCode = 'I', 'In Progress', + IIF(dbo.ActivityTask.StatusCode = 'A', 'Cancelled', + IIF(dbo.ActivityTask.StatusCode = 'N', 'Not Started', + IIF(dbo.ActivityTask.StatusCode = 'P', 'Marked N/A', + IIF(dbo.ActivityTask.StatusCode = 'C', 'Complete', 'PICNIC')))))) AS Task_Status, + + CONVERT(varchar(50),dbo.Activity.InsertedByCode) AS 'Act_Ins_By', + CONVERT(varchar(50),dbo.Activity.ClosedByCode) AS 'Act_Cls_By', + CONVERT(varchar(50),dbo.ActivityNote.InsertedByCode) AS 'Note_Ins_By', + CONVERT(varchar(50),dbo.ActivityTask.InsertedByCode) AS 'Task_Ins_By', + CONVERT(varchar(50),dbo.Attachment.InsertedByCode) AS 'Att_Ins_By', + + Act_Ins_ID.UniqSecurityUser AS 'Act_Ins_ID', + Act_Cls_ID.UniqSecurityUser AS 'Act_Cls_ID', + Note_Ins_ID.UniqSecurityUser AS 'Note_Ins_ID', + Task_Ins_ID.UniqSecurityUser AS 'Task_Ins_ID', + Att_Ins_ID.UniqSecurityUser AS 'Att_Ins_ID', + + DATEADD(HOUR, - 8, dbo.Activity.InsertedDate) AS 'Act_Ins_Date', + DATEADD(HOUR, - 8, dbo.Activity.ClosedDate) AS 'Act_Cls_Date', + DATEADD(HOUR, - 8, dbo.ActivityNote.InsertedDate) AS 'Note_Ins_Date', + DATEADD(HOUR, - 8, dbo.ActivityTask.InsertedDate) AS 'Task_Ins_Date', + DATEADD(HOUR, - 8, dbo.Attachment.AttachedDate) AS 'Att_Ins_Date', + + IIF(dbo.Activity.InsertedDate IS NULL, NULL, 1) AS 'Act_Ins_Event', + IIF(dbo.Activity.ClosedDate IS NULL, NULL, 2) AS 'Act_Cls_Event', + IIF(dbo.ActivityNote.InsertedDate IS NULL, NULL, 3) AS 'Note_Ins_Event', + IIF(dbo.ActivityTask.InsertedDate IS NULL, NULL, 4) AS 'Task_Ins_Event', + IIF(dbo.Attachment.AttachedDate IS NULL, NULL, 5) AS 'Att_Ins_Event', + + CONVERT(varchar(100), dbo.Activity.DescriptionOf) AS 'Act_Ins_Desc', + CONVERT(varchar(100), dbo.Activity.DescriptionOf) AS 'Act_Cls_Desc', + CONVERT(varchar(100), dbo.ActivityNote.Note) AS 'Note_Ins_Desc', + CONVERT(varchar(100), dbo.ActivityTask.DescriptionOf) AS 'Task_Ins_Desc', + CONVERT(varchar(100), CONCAT(dbo.Attachment.DescriptionOf,dbo.Attachment.FileExtension)) AS 'Att_Ins_Desc', + + dbo.Activity.UniqActivity AS 'Act_Ins_Uniq', + dbo.Activity.UniqActivity AS 'Act_Cls_Uniq', + dbo.ActivityNote.UniqActivityNote AS 'Note_Ins_Uniq', + dbo.ActivityTask.UniqActivityTask AS 'Task_Ins_Uniq', + dbo.Attachment.UniqAttachment AS 'Att_Ins_Uniq', + dbo.Attachment.FileExtension, + Svc_Role.RiskAdvisor, + Svc_Role.AccountMgr, + Svc_Role.RACode, + Svc_Role.AMCode + + + FROM dbo.Activity + LEFT OUTER JOIN dbo.Employee AS Activity_Owner ON dbo.Activity.UniqEmployee = Activity_Owner.UniqEntity + LEFT OUTER JOIN dbo.Workgroup ON dbo.Activity.UniqWorkGroup = dbo.Workgroup.UniqWorkGroup + LEFT OUTER JOIN dbo.ConfigureLkLanguageResource AS Workgroup_Name ON dbo.Workgroup.ConfigureLkLanguageResourceID = Workgroup_Name.ConfigureLkLanguageResourceID AND Workgroup_Name.CultureCode = 'en-US' + FULL OUTER JOIN dbo.ActivityCode ON dbo.Activity.UniqActivityCode = dbo.ActivityCode.UniqActivityCode + LEFT OUTER JOIN dbo.Client ON dbo.Activity.UniqEntity = dbo.Client.UniqEntity + LEFT OUTER JOIN dbo.Department AS Act_Dept ON dbo.Activity.UniqDepartment = Act_Dept.UniqDepartment + LEFT OUTER JOIN dbo.Branch AS Act_Branch ON dbo.Activity.UniqBranch = Act_Branch.UniqBranch + LEFT OUTER JOIN dbo.ProfitCenter AS Act_PC ON dbo.Activity.UniqProfitCenter = Act_PC.UniqProfitCenter + + LEFT OUTER JOIN dbo.SecurityUser AS Act_Ins_ID ON dbo.Activity.InsertedByCode = Act_Ins_ID.UserCode + LEFT OUTER JOIN dbo.SecurityUser AS Act_Cls_ID ON dbo.Activity.InsertedByCode = Act_Cls_ID.UserCode + + FULL OUTER JOIN dbo.ActivityNote ON dbo.Activity.UniqActivity = dbo.ActivityNote.UniqActivity --AND (dbo.ActivityNote.InsertedDate BETWEEN DATEADD(MONTH,-13,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)) AND EOMONTH(DATEADD(MONTH,-2,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)))) + LEFT OUTER JOIN dbo.SecurityUser AS Note_Ins_ID ON dbo.ActivityNote.InsertedByCode = Note_Ins_ID.UserCode + + LEFT OUTER JOIN dbo.ActivityTask ON dbo.Activity.UniqActivity = dbo.ActivityTask.UniqActivity --AND (dbo.ActivityTask.InsertedDate BETWEEN DATEADD(MONTH,-13,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)) AND EOMONTH(DATEADD(MONTH,-2,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)))) + LEFT OUTER JOIN dbo.SecurityUser AS Task_Ins_ID ON dbo.ActivityTask.InsertedByCode = Task_Ins_ID.UserCode + + LEFT OUTER JOIN dbo.AttachmentAttachedTo ON dbo.Activity.UniqActivity = dbo.AttachmentAttachedTo.UniqActivity + LEFT OUTER JOIN dbo.Attachment ON dbo.AttachmentAttachedTo.UniqAttachment = dbo.Attachment.UniqAttachment --AND ((NOT(dbo.AttachmentAttachedTo.UniqActivity = - 1)) AND (dbo.Attachment.AttachedDate BETWEEN DATEADD(MONTH,-13,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)) AND EOMONTH(DATEADD(MONTH,-2,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0))))) + LEFT OUTER JOIN dbo.SecurityUser AS Att_Ins_ID ON dbo.Attachment.InsertedByCode = Att_Ins_ID.UserCode + LEFT OUTER JOIN dbo.ActivityPolicyLineJT ON dbo.Activity.UniqActivity = dbo.ActivityPolicyLineJT.UniqActivity + + LEFT OUTER JOIN dbo.Policy ON dbo.ActivityPolicyLineJT.UniqPolicy = dbo.Policy.UniqPolicy + LEFT OUTER JOIN dbo.Line ON dbo.Policy.UniqPolicy = dbo.Line.UniqPolicy + LEFT OUTER JOIN dbo.Department AS Pol_Dept ON dbo.Activity.UniqDepartment = Pol_Dept.UniqDepartment + LEFT OUTER JOIN dbo.Branch AS Pol_Branch ON dbo.Activity.UniqBranch = Pol_Branch.UniqBranch + LEFT OUTER JOIN dbo.ProfitCenter AS Pol_PC ON dbo.Activity.UniqProfitCenter = Pol_PC.UniqProfitCenter + + LEFT OUTER JOIN + (SELECT + dbo.Line.UniqPolicy, + MAX(CASE WHEN dbo.CdServicingRole.CdServicingRoleCode = 'RA0' THEN dbo.Employee.NameOf ELSE NULL END) AS RiskAdvisor, + MAX(CASE WHEN dbo.CdServicingRole.CdServicingRoleCode = 'AM0' THEN dbo.Employee.NameOf ELSE NULL END) AS AccountMgr, + MAX(CASE WHEN dbo.CdServicingRole.CdServicingRoleCode = 'RA0' THEN dbo.SecurityUser.UniqSecurityUser ELSE NULL END) AS RACode, + MAX(CASE WHEN dbo.CdServicingRole.CdServicingRoleCode = 'AM0' THEN dbo.SecurityUser.UniqSecurityUser ELSE NULL END) AS AMCode + FROM dbo.Line + LEFT OUTER JOIN dbo.LineEmployeeServicingJT ON dbo.Line.UniqLine = dbo.LineEmployeeServicingJT.UniqLine + LEFT OUTER JOIN dbo.CdServicingRole ON dbo.CdServicingRole.UniqCdServicingRole = dbo.LineEmployeeServicingJT.UniqCdServicingRole + AND dbo.CdServicingRole.CdServicingRoleCode IN ('RA0', 'AM0','AA0','BC0','MA0') + LEFT OUTER JOIN dbo.Employee ON dbo.LineEmployeeServicingJT.UniqEntity = dbo.Employee.UniqEntity + LEFT OUTER JOIN dbo.SecurityUser ON dbo.Employee.UniqEntity = dbo.SecurityUser.UniqEmployee + GROUP BY dbo.Line.UniqPolicy + ) AS Svc_Role ON dbo.ActivityPolicyLineJT.UniqPolicy = Svc_Role.UniqPolicy + + WHERE (NOT(dbo.Activity.LkActivityUnsuccessfulReason LIKE 'Incorrect%')) + AND (NOT(dbo.Activity.DescriptionOf LIKE '%Duplicate%')) + AND (dbo.Client.LookupCode <> '') + AND (NOT (dbo.Client.LookupCode LIKE '%TESTACCT%')) + AND (NOT (dbo.Client.LookupCode LIKE '%PLUGACCT%')) + AND (NOT (dbo.Client.LookupCode LIKE 'MORR&GA-01')) + --AND ((dbo.Activity.InsertedDate BETWEEN DATEADD(MONTH,-13,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)) AND EOMONTH(DATEADD(MONTH,-2,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)))) + -- OR (dbo.Activity.ClosedDate BETWEEN DATEADD(MONTH,-13,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)) AND EOMONTH(DATEADD(MONTH,-2,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)))) + -- OR (dbo.ActivityNote.InsertedDate BETWEEN DATEADD(MONTH,-13,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)) AND EOMONTH(DATEADD(MONTH,-2,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)))) + -- OR (dbo.ActivityTask.InsertedDate BETWEEN DATEADD(MONTH,-13,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)) AND EOMONTH(DATEADD(MONTH,-2,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)))) + -- OR (dbo.Attachment.InsertedDate BETWEEN DATEADD(MONTH,-13,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)) AND EOMONTH(DATEADD(MONTH,-2,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0))))) + ) AS Activity_Events + +UNPIVOT + ( + Employee FOR Employees IN (Act_Ins_By, Act_Cls_By, Note_Ins_By, Task_Ins_By, Att_Ins_By) + ) AS eeup + +UNPIVOT + ( + ID FOR IDs IN (Act_Ins_ID, Act_Cls_ID, Note_Ins_ID, Task_Ins_ID, Att_Ins_ID) + ) AS idup + +UNPIVOT + ( + Event_Date FOR Event_Dates IN (Act_Ins_Date, Act_Cls_Date, Note_Ins_Date, Task_Ins_Date, Att_Ins_Date) + ) AS dateup + +UNPIVOT + ( + Event_Type FOR Event_Types IN (Act_Ins_Event, Act_Cls_Event, Note_Ins_Event, Task_Ins_Event, Att_Ins_Event) + ) AS typeup + +UNPIVOT + ( + Event_Desc FOR Event_Descs IN (Act_Ins_Desc, Act_Cls_Desc, Note_Ins_Desc, Task_Ins_Desc, Att_Ins_Desc) + ) AS descup +UNPIVOT + ( + Event_Uniq FOR Event_Uniqs IN (Act_Ins_Uniq,Act_Cls_Uniq,Note_Ins_Uniq,Task_Ins_Uniq,Att_Ins_Uniq) + ) AS uniqup + +WHERE + LEFT(Employees, 6) = LEFT(IDs, 6) + AND LEFT(Employees, 6) = LEFT(Event_Dates, 6) + AND LEFT(Employees, 6) = LEFT(Event_Types, 6) + AND LEFT(Event_Types, 6) = LEFT(Event_Descs, 6) + AND LEFT(Event_Uniqs,6) = LEFT(Event_Types,6) + AND (Event_Date BETWEEN DATEADD(MONTH,-13,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)) AND EOMONTH(DATEADD(MONTH,-2,DATEADD(MONTH,DATEDIFF(MONTH,0,GETDATE()),0)))) + AND (NOT([Employee] IN ('XHR001','APPLIED','APPLIEDSUPPORT','APPLIED1','APPLIED3','ZZZZAP','DOWNLOAD','ENTERPRISEADMIN','CONFIG','EPICSDKUSER','GOLIVE','SYSTEM','ZYWAE1'))) + +ORDER BY Event_Date \ No newline at end of file diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index 40b3fcfc..830886cd 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -127,54 +127,129 @@ "case1": { "expectedResults": [ { - "pvt": { - "VendorID": 4, - "Emp1": 4, - "Emp2": 2, - "Emp3": 5, - "Emp4": 5, - "Emp5": 4 - } + "VendorID": 1, + "Employee": "Emp1", + "Orders": 4 }, { - "pvt": { - "VendorID": 2, - "Emp1": 4, - "Emp2": 1, - "Emp3": 5, - "Emp4": 5, - "Emp5": 5 - } + "VendorID": 1, + "Employee": "Emp2", + "Orders": 3 }, { - "pvt": { - "VendorID": 1, - "Emp1": 4, - "Emp2": 3, - "Emp3": 5, - "Emp4": 4, - "Emp5": 4 - } + "VendorID": 1, + "Employee": "Emp3", + "Orders": 5 }, { - "pvt": { - "VendorID": 3, - "Emp1": 4, - "Emp2": 3, - "Emp3": 5, - "Emp4": 4, - "Emp5": 4 - } + "VendorID": 1, + "Employee": "Emp4", + "Orders": 4 }, { - "pvt": { - "VendorID": 5, - "Emp1": 5, - "Emp2": 1, - "Emp3": 5, - "Emp4": 5, - "Emp5": 5 - } + "VendorID": 1, + "Employee": "Emp5", + "Orders": 4 + }, + { + "VendorID": 2, + "Employee": "Emp1", + "Orders": 4 + }, + { + "VendorID": 2, + "Employee": "Emp2", + "Orders": 1 + }, + { + "VendorID": 2, + "Employee": "Emp3", + "Orders": 5 + }, + { + "VendorID": 2, + "Employee": "Emp4", + "Orders": 5 + }, + { + "VendorID": 2, + "Employee": "Emp5", + "Orders": 5 + }, + { + "VendorID": 3, + "Employee": "Emp1", + "Orders": 4 + }, + { + "VendorID": 3, + "Employee": "Emp2", + "Orders": 3 + }, + { + "VendorID": 3, + "Employee": "Emp3", + "Orders": 5 + }, + { + "VendorID": 3, + "Employee": "Emp4", + "Orders": 4 + }, + { + "VendorID": 3, + "Employee": "Emp5", + "Orders": 4 + }, + { + "VendorID": 4, + "Employee": "Emp1", + "Orders": 4 + }, + { + "VendorID": 4, + "Employee": "Emp2", + "Orders": 2 + }, + { + "VendorID": 4, + "Employee": "Emp3", + "Orders": 5 + }, + { + "VendorID": 4, + "Employee": "Emp4", + "Orders": 5 + }, + { + "VendorID": 4, + "Employee": "Emp5", + "Orders": 4 + }, + { + "VendorID": 5, + "Employee": "Emp1", + "Orders": 5 + }, + { + "VendorID": 5, + "Employee": "Emp2", + "Orders": 1 + }, + { + "VendorID": 5, + "Employee": "Emp3", + "Orders": 5 + }, + { + "VendorID": 5, + "Employee": "Emp4", + "Orders": 5 + }, + { + "VendorID": 5, + "Employee": "Emp5", + "Orders": 5 } ] } diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index e2083e7b..cfe3e0af 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -414,7 +414,7 @@ describe('node-sql-parser upgrade tests', function () { await queryResultTester({ queryString: queryText, casePath: 'pivot.case1', - mode: 'write', + mode, outputPipeline: false, }); // const expected = { @@ -430,7 +430,7 @@ describe('node-sql-parser upgrade tests', function () { describe('UNPIVOT', () => { it('should unpivot employee columns to rows', async () => { const queryText = ` - SELECT * + SELECT VendorID, Employee, Orders FROM ( SELECT VendorID, Emp1, Emp2, Emp3, Emp4, Emp5 FROM pvt From cf9fb848434a5ff750627f8f5b1563dccfbb08d0 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Wed, 4 Sep 2024 08:33:46 +0200 Subject: [PATCH 08/18] fix(pivot): when only one field, don't return an object --- lib/make/apply-pivot.js | 25 +++++++++++++---------- test/individualTests/upgrade.json | 11 ++++++++++ test/individualTests/upgrade.test.js | 30 ++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/lib/make/apply-pivot.js b/lib/make/apply-pivot.js index aa423fe3..74dd624d 100644 --- a/lib/make/apply-pivot.js +++ b/lib/make/apply-pivot.js @@ -39,6 +39,7 @@ function applyPivot(pivotString, pipeline, context) { parsedArguments, }; }); + pipeline.push({ $group: { _id: `$${pivot.for}`, @@ -59,16 +60,19 @@ function applyPivot(pivotString, pipeline, context) { k: { $toString: '$_id', }, - v: { - ...pivot.fields.reduce( - (previousValue, currentValue) => { - previousValue[currentValue.name] = - `$${currentValue.name}`; - return previousValue; - }, - {} - ), - }, + v: + pivot.fields.length > 1 + ? { + ...pivot.fields.reduce( + (previousValue, currentValue) => { + previousValue[currentValue.name] = + `$${currentValue.name}`; + return previousValue; + }, + {} + ), + } + : `$${pivot.fields[0].name}`, }, }, }, @@ -104,6 +108,7 @@ function applyPivot(pivotString, pipeline, context) { newRoot: '$result', }, }); + console.log(''); } /** diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index 830886cd..ec3f73f9 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -100,6 +100,17 @@ }, "pivot": { "case1": { + "expectedResults": [ + { + "0": 5.0885, + "1": 223.88, + "2": 359.1082, + "3": null, + "4": 949.4105 + } + ] + }, + "case2": { "expectedResults": [ { "0": { diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index cfe3e0af..3b2e2afb 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -399,7 +399,7 @@ describe('node-sql-parser upgrade tests', function () { }); describe('PIVOT and UNPIVOT', () => { describe('PIVOT', () => { - it('should pivot DaysToManufacture to columns', async () => { + it('should pivot DaysToManufacture to columns with one aggregation function', async () => { const queryText = ` SELECT * FROM ( @@ -408,7 +408,7 @@ describe('node-sql-parser upgrade tests', function () { FROM Production_Product GROUP BY DaysToManufacture, StandardCost ORDER BY DaysToManufacture, StandardCost - ) 'pvt|pivot([avg(StandardCost), max(StandardCost) as MaxCost],DaysToManufacture,[0,1,2,3,4])' + ) 'pvt|pivot([avg(StandardCost)],DaysToManufacture,[0,1,2,3,4])' `; await queryResultTester({ @@ -425,6 +425,32 @@ describe('node-sql-parser upgrade tests', function () { // 4: 949.4105, // }; }); + it('should pivot DaysToManufacture to columns with two aggregation functions, one with an as', async () => { + const queryText = ` + SELECT * + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot([avg(StandardCost), max(StandardCost) as MaxCost],DaysToManufacture,[0,1,2,3,4])' + `; + + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case2', + mode, + outputPipeline: false, + }); + // const expected = { + // 0: 5.0885, + // 1: 223.88, + // 2: 359.1082, + // 3: null, + // 4: 949.4105, + // }; + }); }); describe('UNPIVOT', () => { From c660b6dd12b955c706665889ec395e5bc4780fbc Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Wed, 4 Sep 2024 14:01:54 +0200 Subject: [PATCH 09/18] Cleaning up pivot/unpivot --- lib/make/apply-pivot.js | 10 -------- lib/make/makeAggregatePipeline.js | 15 ++++++++--- test/individualTests/upgrade.json | 31 +++++++++++++++++++++-- test/individualTests/upgrade.test.js | 37 ++++++++++++---------------- 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/lib/make/apply-pivot.js b/lib/make/apply-pivot.js index 74dd624d..dca740fa 100644 --- a/lib/make/apply-pivot.js +++ b/lib/make/apply-pivot.js @@ -108,7 +108,6 @@ function applyPivot(pivotString, pipeline, context) { newRoot: '$result', }, }); - console.log(''); } /** @@ -223,13 +222,4 @@ function applyUnpivot(pivotString, pipeline, context) { [unpivot.fields[0]]: '$fields.v', // Assuming 'Orders' should be numeric }, }); - pipeline.push({ - $sort: { - ...columns.reduce((previousValue, currentValue) => { - previousValue[currentValue] = 1; - return previousValue; - }, {}), - [unpivot.for]: 1, - }, - }); } diff --git a/lib/make/makeAggregatePipeline.js b/lib/make/makeAggregatePipeline.js index 234b03df..617bbf73 100644 --- a/lib/make/makeAggregatePipeline.js +++ b/lib/make/makeAggregatePipeline.js @@ -417,15 +417,19 @@ function makeAggregatePipeline(ast, context = {}) { result.subQueryRootProjections.push(tableAs); if (as.indexOf('|pivot(') >= 0) { - // const prevPipeline = pipeline; + const prevPipeline = pipeline; pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context); applyPivot(as, pipeline, context); - // pipeline = pipeline - // .concat([{$project: {[tableAs]: '$$ROOT'}}]) - // .concat(prevPipeline); + pipeline = pipeline + .concat([{$project: {[tableAs]: '$$ROOT'}}]) + .concat(prevPipeline); } else if (as.indexOf('|unpivot(') >= 0) { + const prevPipeline = pipeline; pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context); applyUnpivot(as, pipeline, context); + pipeline = pipeline + .concat([{$project: {[tableAs]: '$$ROOT'}}]) + .concat(prevPipeline); } else { pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context) .concat([{$project: {[tableAs]: '$$ROOT'}}]) @@ -803,6 +807,9 @@ function handleExcept(ast, context, pipeline) { * @returns {string} */ function stripJoinHints(input) { + if (!input) { + return input; + } return input.split('|')[0]; } diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index ec3f73f9..c52ddb50 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -106,7 +106,8 @@ "1": 223.88, "2": 359.1082, "3": null, - "4": 949.4105 + "4": 949.4105, + "CostSortedByProductionDays": "AverageCost" } ] }, @@ -129,7 +130,8 @@ "4": { "StandardCost": 949.4105, "MaxCost": 949.4105 - } + }, + "CostSortedByProductionDays": "Costs" } ] } @@ -162,6 +164,11 @@ "Employee": "Emp5", "Orders": 4 }, + { + "VendorID": 1, + "Employee": "_id", + "Orders": "66d84bd59c3487b892c5c7db" + }, { "VendorID": 2, "Employee": "Emp1", @@ -187,6 +194,11 @@ "Employee": "Emp5", "Orders": 5 }, + { + "VendorID": 2, + "Employee": "_id", + "Orders": "66d84bd59c3487b892c5c7dc" + }, { "VendorID": 3, "Employee": "Emp1", @@ -212,6 +224,11 @@ "Employee": "Emp5", "Orders": 4 }, + { + "VendorID": 3, + "Employee": "_id", + "Orders": "66d84bd59c3487b892c5c7dd" + }, { "VendorID": 4, "Employee": "Emp1", @@ -237,6 +254,11 @@ "Employee": "Emp5", "Orders": 4 }, + { + "VendorID": 4, + "Employee": "_id", + "Orders": "66d84bd59c3487b892c5c7de" + }, { "VendorID": 5, "Employee": "Emp1", @@ -261,6 +283,11 @@ "VendorID": 5, "Employee": "Emp5", "Orders": 5 + }, + { + "VendorID": 5, + "Employee": "_id", + "Orders": "66d84bd59c3487b892c5c7df" } ] } diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index 3b2e2afb..dd3109d1 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -401,14 +401,19 @@ describe('node-sql-parser upgrade tests', function () { describe('PIVOT', () => { it('should pivot DaysToManufacture to columns with one aggregation function', async () => { const queryText = ` - SELECT * + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" FROM ( SELECT DaysToManufacture, StandardCost FROM Production_Product GROUP BY DaysToManufacture, StandardCost ORDER BY DaysToManufacture, StandardCost - ) 'pvt|pivot([avg(StandardCost)],DaysToManufacture,[0,1,2,3,4])' + ) 'pvt|pivot([avg(StandardCost) as AverageCost],DaysToManufacture,[0,1,2,3,4])' `; await queryResultTester({ @@ -417,17 +422,15 @@ describe('node-sql-parser upgrade tests', function () { mode, outputPipeline: false, }); - // const expected = { - // 0: 5.0885, - // 1: 223.88, - // 2: 359.1082, - // 3: null, - // 4: 949.4105, - // }; }); it('should pivot DaysToManufacture to columns with two aggregation functions, one with an as', async () => { const queryText = ` - SELECT * + SELECT 'Costs' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" FROM ( SELECT DaysToManufacture, StandardCost @@ -440,29 +443,21 @@ describe('node-sql-parser upgrade tests', function () { await queryResultTester({ queryString: queryText, casePath: 'pivot.case2', - mode, + mode: 'write', outputPipeline: false, }); - // const expected = { - // 0: 5.0885, - // 1: 223.88, - // 2: 359.1082, - // 3: null, - // 4: 949.4105, - // }; }); }); describe('UNPIVOT', () => { it('should unpivot employee columns to rows', async () => { const queryText = ` - SELECT VendorID, Employee, Orders + SELECT VendorID, Employee, Orders, unset(_id) FROM ( SELECT VendorID, Emp1, Emp2, Emp3, Emp4, Emp5 FROM pvt - GROUP BY DaysToManufacture, StandardCost - ORDER BY DaysToManufacture, StandardCost ) 'unpvt|unpivot([Orders],Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])' + ORDER BY VendorID, Employee `; await queryResultTester({ From a405396f652bed6ec45fd64a6133ec9750486e85 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Wed, 4 Sep 2024 14:19:09 +0200 Subject: [PATCH 10/18] Finishing up pivot and unpivot --- test/individualTests/upgrade.json | 25 ------------------------- test/individualTests/upgrade.test.js | 8 ++++---- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index c52ddb50..00bc59b9 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -164,11 +164,6 @@ "Employee": "Emp5", "Orders": 4 }, - { - "VendorID": 1, - "Employee": "_id", - "Orders": "66d84bd59c3487b892c5c7db" - }, { "VendorID": 2, "Employee": "Emp1", @@ -194,11 +189,6 @@ "Employee": "Emp5", "Orders": 5 }, - { - "VendorID": 2, - "Employee": "_id", - "Orders": "66d84bd59c3487b892c5c7dc" - }, { "VendorID": 3, "Employee": "Emp1", @@ -224,11 +214,6 @@ "Employee": "Emp5", "Orders": 4 }, - { - "VendorID": 3, - "Employee": "_id", - "Orders": "66d84bd59c3487b892c5c7dd" - }, { "VendorID": 4, "Employee": "Emp1", @@ -254,11 +239,6 @@ "Employee": "Emp5", "Orders": 4 }, - { - "VendorID": 4, - "Employee": "_id", - "Orders": "66d84bd59c3487b892c5c7de" - }, { "VendorID": 5, "Employee": "Emp1", @@ -283,11 +263,6 @@ "VendorID": 5, "Employee": "Emp5", "Orders": 5 - }, - { - "VendorID": 5, - "Employee": "_id", - "Orders": "66d84bd59c3487b892c5c7df" } ] } diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index dd3109d1..28617005 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -443,7 +443,7 @@ describe('node-sql-parser upgrade tests', function () { await queryResultTester({ queryString: queryText, casePath: 'pivot.case2', - mode: 'write', + mode, outputPipeline: false, }); }); @@ -452,9 +452,9 @@ describe('node-sql-parser upgrade tests', function () { describe('UNPIVOT', () => { it('should unpivot employee columns to rows', async () => { const queryText = ` - SELECT VendorID, Employee, Orders, unset(_id) + SELECT VendorID, Employee, Orders FROM ( - SELECT VendorID, Emp1, Emp2, Emp3, Emp4, Emp5 + SELECT VendorID, Emp1, Emp2, Emp3, Emp4, Emp5, unset(_id) FROM pvt ) 'unpvt|unpivot([Orders],Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])' ORDER BY VendorID, Employee @@ -463,7 +463,7 @@ describe('node-sql-parser upgrade tests', function () { await queryResultTester({ queryString: queryText, casePath: 'unpivot.case1', - mode: 'write', + mode, outputPipeline: false, }); }); From 8e73b35dc729f3a262e44eae6760bac5b8c82178 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Thu, 5 Sep 2024 08:44:19 +0200 Subject: [PATCH 11/18] Trying to get multiple unwinds working --- lib/make/apply-pivot.js | 12 ++++++------ lib/make/makeAggregatePipeline.js | 28 ++++++++++++++++++++++++---- test/individualTests/upgrade.test.js | 18 ++++++++++++++++++ 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/lib/make/apply-pivot.js b/lib/make/apply-pivot.js index dca740fa..fe682334 100644 --- a/lib/make/apply-pivot.js +++ b/lib/make/apply-pivot.js @@ -169,15 +169,15 @@ function createJSONFromPivotString(type, inputString) { * * @param {string} pivotString * @param {import('../types').PipelineFn[]} pipeline + * @param {import('../types').PipelineFn} projection * @param {import('../types').NoqlContext} context */ -function applyUnpivot(pivotString, pipeline, context) { +function applyUnpivot(pivotString, pipeline, projection, context) { const unpivot = createJSONFromPivotString('unpivot', pivotString); - const projection = pipeline - .slice() - .reverse() - .find((p) => !!p.$project); - const columnsToExclude = [unpivot.for, '_id'].concat(unpivot.columns); + + const columnsToExclude = [unpivot.for, '_id'] + .concat(unpivot.columns) + .concat(unpivot.fields); const columns = Object.keys(projection.$project).filter( (val) => columnsToExclude.indexOf(val) === -1 ); diff --git a/lib/make/makeAggregatePipeline.js b/lib/make/makeAggregatePipeline.js index 617bbf73..81c2b7e1 100644 --- a/lib/make/makeAggregatePipeline.js +++ b/lib/make/makeAggregatePipeline.js @@ -426,10 +426,30 @@ function makeAggregatePipeline(ast, context = {}) { } else if (as.indexOf('|unpivot(') >= 0) { const prevPipeline = pipeline; pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context); - applyUnpivot(as, pipeline, context); - pipeline = pipeline - .concat([{$project: {[tableAs]: '$$ROOT'}}]) - .concat(prevPipeline); + const projection = prevPipeline + .slice() + .reverse() + .find((p) => !!p.$project); + const unpivots = as + .split('|unpivot') + .filter((u) => u.startsWith('(')) + .map((u) => '|unpivot' + u); + if (unpivots.length === 1) { + applyUnpivot(unpivots[0], pipeline, projection, context); + pipeline = pipeline + .concat([{$project: {[tableAs]: '$$ROOT'}}]) + .concat(prevPipeline); + } + const facet = {$facet: {}}; + pipeline.push(facet); + let counter = 0; + for (const unpivot of unpivots) { + const subPipeline = []; + applyUnpivot(unpivot, subPipeline, projection, context); + facet.$facet[counter.toString()] = subPipeline; + counter++; + } + console.log(''); } else { pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context) .concat([{$project: {[tableAs]: '$$ROOT'}}]) diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index 28617005..95ce8829 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -467,6 +467,24 @@ describe('node-sql-parser upgrade tests', function () { outputPipeline: false, }); }); + it('should support multiple unpivots', async () => { + // See https://dba.stackexchange.com/a/222745 + const queryText = ` + SELECT VendorID, Employee, Orders + FROM ( + SELECT VendorID, Emp1, Emp2, Emp3, Emp4, Emp5, unset(_id) + FROM pvt + ) 'unpvt|unpivot([Orders],Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])|unpivot([Orders2],Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])' + ORDER BY VendorID, Employee + `; + + await queryResultTester({ + queryString: queryText, + casePath: 'unpivot.case2', + mode: 'write', + outputPipeline: false, + }); + }); }); }); }); From cb1b0b41f134400f799b0ea66dba4c1aee6c0fc9 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Mon, 9 Sep 2024 13:08:28 +0200 Subject: [PATCH 12/18] basic multi unpivot logic --- lib/make/apply-pivot.js | 157 ++++++++++++++++++- lib/make/makeAggregatePipeline.js | 21 ++- test/exampleData/multiple-unpivot.json | 9 ++ test/individualTests/upgrade.json | 201 +++++++++++++++++++++++++ test/individualTests/upgrade.test.js | 17 ++- 5 files changed, 385 insertions(+), 20 deletions(-) create mode 100644 test/exampleData/multiple-unpivot.json diff --git a/lib/make/apply-pivot.js b/lib/make/apply-pivot.js index fe682334..f366dde2 100644 --- a/lib/make/apply-pivot.js +++ b/lib/make/apply-pivot.js @@ -1,5 +1,7 @@ const {functionByName} = require('../MongoFunctions'); -module.exports = {applyPivot, applyUnpivot}; +const merge = require('lodash/merge'); + +module.exports = {applyPivot, applyUnpivot, applyMultipleUnpivots}; /** * @@ -203,7 +205,15 @@ function applyUnpivot(pivotString, pipeline, projection, context) { input: '$fields', as: 'field', cond: { - $and: columns.map((c) => ({$ne: ['$$field.k', c]})), + $and: columns + .map((c) => ({$ne: ['$$field.k', c]})) + .concat([ + { + $or: unpivot.columns.map((c) => ({ + $eq: ['$$field.k', c], + })), + }, + ]), }, }, }, @@ -223,3 +233,146 @@ function applyUnpivot(pivotString, pipeline, projection, context) { }, }); } + +/** + * + * @param {string[]} pivotStrings + * @param pivotString + * @param {import('../types').PipelineFn[]} pipeline + * @param {import('../types').PipelineFn} projection + * @param {import('../types').NoqlContext} context + */ +function applyMultipleUnpivots(pivotStrings, pipeline, projection, context) { + const unpivots = pivotStrings.map((s) => + createJSONFromPivotString('unpivot', s) + ); + const unpivot = unpivots.reduce( + (previousValue, currentValue) => { + previousValue.for.push(currentValue.for); + previousValue.fields.push(...currentValue.fields); + previousValue.columns.push(...currentValue.columns); + return previousValue; + }, + {fields: [], for: [], columns: []} + ); + const columnsToExclude = [...unpivot.for, '_id'] + .concat(unpivot.columns) + .concat(unpivot.fields); + const columns = Object.entries(projection.$project) + .filter( + ([key, expression]) => + columnsToExclude.indexOf(key) === -1 && + expression.indexOf('.') > 0 + ) + .map(([key]) => key); + pipeline.push({ + $project: { + ...columns.reduce((previousValue, currentValue) => { + previousValue[currentValue] = `$${currentValue}`; + return previousValue; + }, {}), + fields: { + $objectToArray: '$$ROOT', + }, + }, + }); + pipeline.push({ + $project: { + ...columns.reduce((previousValue, currentValue) => { + previousValue[currentValue] = 1; + return previousValue; + }, {}), + fields: { + $filter: { + input: '$fields', + as: 'field', + cond: { + $and: columns + .map((c) => ({$ne: ['$$field.k', c]})) + .concat([ + { + $or: unpivot.columns.map((c) => ({ + $eq: ['$$field.k', c], + })), + }, + ]), + }, + }, + }, + }, + }); + pipeline.push({ + $unwind: '$fields', + }); + const columnProjection = { + $project: { + ...columns.reduce((previousValue, currentValue) => { + previousValue[currentValue] = 1; + return previousValue; + }, {}), + }, + }; + for (const unpivotOp of unpivots) { + columnProjection.$project[unpivotOp.fields[0]] = { + $cond: { + if: { + $or: unpivotOp.columns.map((c) => ({ + $eq: ['$fields.k', c], + })), + }, + then: '$fields.v', + else: '$$REMOVE', + }, + }; + } + pipeline.push(columnProjection); + pipeline.push({ + $group: { + _id: columns.reduce((previousValue, currentValue) => { + previousValue[currentValue] = `$${currentValue}`; + return previousValue; + }, {}), + ...unpivots.reduce((previousValue, currentValue) => { + previousValue[currentValue.for] = { + $push: `$${currentValue.fields[0]}`, + }; + return previousValue; + }, {}), + }, + }); + pipeline.push({ + $unwind: { + path: `$${unpivot.for[0]}`, + includeArrayIndex: '__unwindIndex', + preserveNullAndEmptyArrays: false, + }, + }); + const restOfFor = unpivot.for.splice(1); + const restOfFields = unpivot.fields.splice(1); + pipeline.push({ + $project: { + _id: 0, + ...columns.reduce((previousValue, currentValue) => { + previousValue[currentValue] = `$_id.${currentValue}`; + return previousValue; + }, {}), + [unpivot.fields[0]]: `$${unpivot.for[0]}`, + ...restOfFields.reduce( + (previousValue, currentValue, currentIndex) => { + previousValue[currentValue] = { + $arrayElemAt: [ + `$${restOfFor[currentIndex]}`, + '$__unwindIndex', + ], + }; + return previousValue; + }, + {} + ), + }, + }); + console.log(''); + /** + * + */ +} diff --git a/lib/make/makeAggregatePipeline.js b/lib/make/makeAggregatePipeline.js index 81c2b7e1..b3d90086 100644 --- a/lib/make/makeAggregatePipeline.js +++ b/lib/make/makeAggregatePipeline.js @@ -15,7 +15,11 @@ const { const $copy = require('clone-deep'); const {optimizeJoinAndWhere} = require('./optimize-join-and-where'); -const {applyPivot, applyUnpivot} = require('./apply-pivot'); +const { + applyPivot, + applyUnpivot, + applyMultipleUnpivots, +} = require('./apply-pivot'); exports.makeAggregatePipeline = makeAggregatePipeline; /** @@ -439,17 +443,12 @@ function makeAggregatePipeline(ast, context = {}) { pipeline = pipeline .concat([{$project: {[tableAs]: '$$ROOT'}}]) .concat(prevPipeline); + } else { + applyMultipleUnpivots(unpivots, pipeline, projection, context); + pipeline = pipeline + .concat([{$project: {[tableAs]: '$$ROOT'}}]) + .concat(prevPipeline); } - const facet = {$facet: {}}; - pipeline.push(facet); - let counter = 0; - for (const unpivot of unpivots) { - const subPipeline = []; - applyUnpivot(unpivot, subPipeline, projection, context); - facet.$facet[counter.toString()] = subPipeline; - counter++; - } - console.log(''); } else { pipeline = makeAggregatePipeline(ast.from[0].expr.ast, context) .concat([{$project: {[tableAs]: '$$ROOT'}}]) diff --git a/test/exampleData/multiple-unpivot.json b/test/exampleData/multiple-unpivot.json new file mode 100644 index 00000000..b7466277 --- /dev/null +++ b/test/exampleData/multiple-unpivot.json @@ -0,0 +1,9 @@ +[{ + "SalesID": 1001, + "Order1Name": "first", + "Order1Date": "2018-01-01", + "Order1Amt": 111.00, + "Order2Name": "second", + "Order2Date": "2018-02-01", + "Order2Amt": 222.00 +}] diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index 00bc59b9..cc278cf7 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -265,6 +265,207 @@ "Orders": 5 } ] + }, + "case2": { + "expectedResults": [ + { + "SalesID": 1001, + "OrderNum": 1, + "OrderName": "first", + "OrderDate": "2018-01-01" + }, + { + "SalesID": 1001, + "OrderNum": 2, + "OrderName": "second", + "OrderDate": "2018-02-01" + } + ], + "pipeline": [ + { + "$unset": [ + "_id" + ] + }, + { + "$project": { + "SalesID": "$SalesID", + "Order1Name": "$Order1Name", + "Order2Name": "$Order2Name", + "Order1Date": "$Order1Date", + "Order2Date": "$Order2Date", + "Order1Amt": "$Order1Amt", + "Order2Amt": "$Order2Amt" + } + }, + { + "$project": { + "SalesID": "$SalesID", + "fields": { + "$objectToArray": "$$ROOT" + } + } + }, + { + "$project": { + "SalesID": 1, + "fields": { + "$filter": { + "input": "$fields", + "as": "field", + "cond": { + "$and": [ + { + "$ne": [ + "$$field.k", + "SalesID" + ] + }, + { + "$or": [ + { + "$eq": [ + "$$field.k", + "Order1Name" + ] + }, + { + "$eq": [ + "$$field.k", + "Order2Name" + ] + }, + { + "$eq": [ + "$$field.k", + "Order1Date" + ] + }, + { + "$eq": [ + "$$field.k", + "Order2Date" + ] + } + ] + } + ] + } + } + } + } + }, + { + "$unwind": "$fields" + }, + { + "$project": { + "SalesID": 1, + "OrderName": { + "$cond": { + "if": { + "$or": [ + { + "$eq": [ + "$fields.k", + "Order1Name" + ] + }, + { + "$eq": [ + "$fields.k", + "Order2Name" + ] + } + ] + }, + "then": "$fields.v", + "else": "$$REMOVE" + } + }, + "OrderDate": { + "$cond": { + "if": { + "$or": [ + { + "$eq": [ + "$fields.k", + "Order1Date" + ] + }, + { + "$eq": [ + "$fields.k", + "Order2Date" + ] + } + ] + }, + "then": "$fields.v", + "else": "$$REMOVE" + } + } + } + }, + { + "$group": { + "_id": { + "SalesID": "$SalesID" + }, + "OrderNames": { + "$push": "$OrderName" + }, + "OrderDates": { + "$push": "$OrderDate" + } + } + }, + { + "$unwind": { + "path": "$OrderNames", + "includeArrayIndex": "__unwindIndex", + "preserveNullAndEmptyArrays": false + } + }, + { + "$project": { + "_id": 0, + "SalesID": "$_id.SalesID", + "OrderName": "$OrderNames", + "OrderDate": { + "$arrayElemAt": [ + "$OrderDates", + "$__unwindIndex" + ] + } + } + }, + { + "$project": { + "unpvt": "$$ROOT" + } + }, + { + "$setWindowFields": { + "sortBy": { + "OrderName": 1 + }, + "output": { + "OrderNum": { + "$documentNumber": {} + } + } + } + }, + { + "$project": { + "SalesID": "$unpvt.SalesID", + "OrderNum": "$OrderNum", + "OrderName": "$unpvt.OrderName", + "OrderDate": "$unpvt.OrderDate" + } + } + ] } } } \ No newline at end of file diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index 95ce8829..ed2b5b22 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -470,19 +470,22 @@ describe('node-sql-parser upgrade tests', function () { it('should support multiple unpivots', async () => { // See https://dba.stackexchange.com/a/222745 const queryText = ` - SELECT VendorID, Employee, Orders + SELECT SalesID, + ROW_NUMBER() OVER ( + ORDER BY OrderName + ) OrderNum, + OrderName, + OrderDate FROM ( - SELECT VendorID, Emp1, Emp2, Emp3, Emp4, Emp5, unset(_id) - FROM pvt - ) 'unpvt|unpivot([Orders],Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])|unpivot([Orders2],Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])' - ORDER BY VendorID, Employee + SELECT SalesID, Order1Name, Order2Name, Order1Date, Order2Date, Order1Amt, Order2Amt, unset(_id) + FROM multiple-unpivot + ) 'unpvt|unpivot([OrderName],OrderNames,[Order1Name, Order2Name])|unpivot([OrderDate],OrderDates,[Order1Date, Order2Date])' `; - await queryResultTester({ queryString: queryText, casePath: 'unpivot.case2', mode: 'write', - outputPipeline: false, + outputPipeline: true, }); }); }); From 045c6fcf60208938c20f487681786ea09dcdae52 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Mon, 9 Sep 2024 13:12:46 +0200 Subject: [PATCH 13/18] updating test to not be in write mode --- test/individualTests/upgrade.json | 191 +-------------------------- test/individualTests/upgrade.test.js | 9 +- 2 files changed, 9 insertions(+), 191 deletions(-) diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index cc278cf7..b61f016a 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -272,198 +272,15 @@ "SalesID": 1001, "OrderNum": 1, "OrderName": "first", - "OrderDate": "2018-01-01" + "OrderDate": "2018-01-01", + "OrderAmt": 111 }, { "SalesID": 1001, "OrderNum": 2, "OrderName": "second", - "OrderDate": "2018-02-01" - } - ], - "pipeline": [ - { - "$unset": [ - "_id" - ] - }, - { - "$project": { - "SalesID": "$SalesID", - "Order1Name": "$Order1Name", - "Order2Name": "$Order2Name", - "Order1Date": "$Order1Date", - "Order2Date": "$Order2Date", - "Order1Amt": "$Order1Amt", - "Order2Amt": "$Order2Amt" - } - }, - { - "$project": { - "SalesID": "$SalesID", - "fields": { - "$objectToArray": "$$ROOT" - } - } - }, - { - "$project": { - "SalesID": 1, - "fields": { - "$filter": { - "input": "$fields", - "as": "field", - "cond": { - "$and": [ - { - "$ne": [ - "$$field.k", - "SalesID" - ] - }, - { - "$or": [ - { - "$eq": [ - "$$field.k", - "Order1Name" - ] - }, - { - "$eq": [ - "$$field.k", - "Order2Name" - ] - }, - { - "$eq": [ - "$$field.k", - "Order1Date" - ] - }, - { - "$eq": [ - "$$field.k", - "Order2Date" - ] - } - ] - } - ] - } - } - } - } - }, - { - "$unwind": "$fields" - }, - { - "$project": { - "SalesID": 1, - "OrderName": { - "$cond": { - "if": { - "$or": [ - { - "$eq": [ - "$fields.k", - "Order1Name" - ] - }, - { - "$eq": [ - "$fields.k", - "Order2Name" - ] - } - ] - }, - "then": "$fields.v", - "else": "$$REMOVE" - } - }, - "OrderDate": { - "$cond": { - "if": { - "$or": [ - { - "$eq": [ - "$fields.k", - "Order1Date" - ] - }, - { - "$eq": [ - "$fields.k", - "Order2Date" - ] - } - ] - }, - "then": "$fields.v", - "else": "$$REMOVE" - } - } - } - }, - { - "$group": { - "_id": { - "SalesID": "$SalesID" - }, - "OrderNames": { - "$push": "$OrderName" - }, - "OrderDates": { - "$push": "$OrderDate" - } - } - }, - { - "$unwind": { - "path": "$OrderNames", - "includeArrayIndex": "__unwindIndex", - "preserveNullAndEmptyArrays": false - } - }, - { - "$project": { - "_id": 0, - "SalesID": "$_id.SalesID", - "OrderName": "$OrderNames", - "OrderDate": { - "$arrayElemAt": [ - "$OrderDates", - "$__unwindIndex" - ] - } - } - }, - { - "$project": { - "unpvt": "$$ROOT" - } - }, - { - "$setWindowFields": { - "sortBy": { - "OrderName": 1 - }, - "output": { - "OrderNum": { - "$documentNumber": {} - } - } - } - }, - { - "$project": { - "SalesID": "$unpvt.SalesID", - "OrderNum": "$OrderNum", - "OrderName": "$unpvt.OrderName", - "OrderDate": "$unpvt.OrderDate" - } + "OrderDate": "2018-02-01", + "OrderAmt": 222 } ] } diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index ed2b5b22..d4ee70cb 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -475,17 +475,18 @@ describe('node-sql-parser upgrade tests', function () { ORDER BY OrderName ) OrderNum, OrderName, - OrderDate + OrderDate, + OrderAmt FROM ( SELECT SalesID, Order1Name, Order2Name, Order1Date, Order2Date, Order1Amt, Order2Amt, unset(_id) FROM multiple-unpivot - ) 'unpvt|unpivot([OrderName],OrderNames,[Order1Name, Order2Name])|unpivot([OrderDate],OrderDates,[Order1Date, Order2Date])' + ) 'unpvt|unpivot([OrderName],OrderNames,[Order1Name, Order2Name])|unpivot([OrderDate],OrderDates,[Order1Date, Order2Date])|unpivot([OrderAmt],OrderAmts,[Order1Amt, Order2Amt])' `; await queryResultTester({ queryString: queryText, casePath: 'unpivot.case2', - mode: 'write', - outputPipeline: true, + mode, + outputPipeline: false, }); }); }); From bf80df55b6d3ee23f3a8fe5fb04eecf95422770d Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Mon, 9 Sep 2024 14:09:57 +0200 Subject: [PATCH 14/18] adding checks for the formatted pivot string --- lib/make/apply-pivot.js | 33 +++- test/individualTests/upgrade.test.js | 257 ++++++++++++++++++++++++++- 2 files changed, 282 insertions(+), 8 deletions(-) diff --git a/lib/make/apply-pivot.js b/lib/make/apply-pivot.js index f366dde2..ccf60ec0 100644 --- a/lib/make/apply-pivot.js +++ b/lib/make/apply-pivot.js @@ -1,5 +1,5 @@ const {functionByName} = require('../MongoFunctions'); -const merge = require('lodash/merge'); +const $check = require('check-types'); module.exports = {applyPivot, applyUnpivot, applyMultipleUnpivots}; @@ -134,30 +134,51 @@ function createJSONFromPivotString(type, inputString) { // Split the string by '|' const parts = inputString.split('|'); + const formatErrorMessage = `The ${type} operation had the wrong format`; // Extract the field from the pivot function let pivotPart = parts[1]; let fieldMatch; if (type === 'pivot') { fieldMatch = pivotPart.match(/pivot\(\[(.*?)\]/); } else { - fieldMatch = pivotPart.match(/unpivot\(\[(.*?)\]/); + fieldMatch = pivotPart.match(/unpivot\((.*?),/); + } + if (!fieldMatch || !fieldMatch[1] || !fieldMatch[1].trim()) { + throw new Error(formatErrorMessage); } const fields = fieldMatch - ? fieldMatch[1].split(',').map((f) => f.trim()) + ? fieldMatch[1] + .split(',') + .map((f) => f.trim()) + .filter(Boolean) : []; + if (!fields.length) { + throw new Error(formatErrorMessage); + } pivotPart = pivotPart.replace(fieldMatch[0], ''); + if (type === 'unpivot') { + pivotPart = ',' + pivotPart; + } // Extract the 'for' part const forMatch = pivotPart.match(/,(.*?),\[/); - const forPart = forMatch ? forMatch[1] : ''; - + const forPart = forMatch ? forMatch[1].trim() : ''; + if (!forPart) { + throw new Error(formatErrorMessage); + } pivotPart = pivotPart.replace(forMatch[0], ''); // Extract the columns const columnsMatch = pivotPart.match(/(.*?)\]/); const columns = columnsMatch - ? columnsMatch[1].split(',').map((c) => c.trim()) + ? columnsMatch[1] + .split(',') + .map((c) => c.trim()) + .filter(Boolean) : []; + if (!columns.length) { + throw new Error(formatErrorMessage); + } // Construct the JSON object return { diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index d4ee70cb..42e20707 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -447,6 +447,259 @@ describe('node-sql-parser upgrade tests', function () { outputPipeline: false, }); }); + const formatErrorSearchString = 'operation had the wrong format'; + it('should throw an error if the pivot is en empty array', async () => { + const queryText = ` + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot([ ],DaysToManufacture,[0,1,2,3,4])' + `; + try { + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode, + outputPipeline: false, + }); + assert.fail('should not pass'); + } catch (err) { + assert(err.message.indexOf(formatErrorSearchString) !== -1); + } + }); + it('should throw an error if the pivot is not provided', async () => { + const queryText = ` + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot(,DaysToManufacture,[0,1,2,3,4])' + `; + try { + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode, + outputPipeline: false, + }); + assert.fail('should not pass'); + } catch (err) { + assert(err.message.indexOf(formatErrorSearchString) !== -1); + } + }); + it('should throw an error if the pivot has only 2 arguments', async () => { + const queryText = ` + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot(DaysToManufacture,[0,1,2,3,4])' + `; + try { + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode, + outputPipeline: false, + }); + assert.fail('should not pass'); + } catch (err) { + assert(err.message.indexOf(formatErrorSearchString) !== -1); + } + }); + it('should throw an error if the pivot has only 2 arguments and the first is empty', async () => { + const queryText = ` + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot(,[0,1,2,3,4])' + `; + try { + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode, + outputPipeline: false, + }); + assert.fail('should not pass'); + } catch (err) { + assert(err.message.indexOf(formatErrorSearchString) !== -1); + } + }); + it('should throw an error if the pivot has only 1 argument', async () => { + const queryText = ` + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot([0,1,2,3,4])' + `; + try { + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode, + outputPipeline: false, + }); + assert.fail('should not pass'); + } catch (err) { + assert(err.message.indexOf(formatErrorSearchString) !== -1); + } + }); + it('should throw an error if the pivot is missing the for argument', async () => { + const queryText = ` + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot([avg(StandardCost) as AverageCost], [0,1,2,3,4])' + `; + try { + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode, + outputPipeline: false, + }); + assert.fail('should not pass'); + } catch (err) { + assert(err.message.indexOf(formatErrorSearchString) !== -1); + } + }); + it('should throw an error if the pivot is missing the columns argument', async () => { + const queryText = ` + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot([avg(StandardCost) as AverageCost],DaysToManufacture' + `; + try { + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode, + outputPipeline: false, + }); + assert.fail('should not pass'); + } catch (err) { + assert(err.message.indexOf(formatErrorSearchString) !== -1); + } + }); + it('should throw an error if the pivot has an empty columns argument', async () => { + const queryText = ` + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot([avg(StandardCost) as AverageCost],DaysToManufacture,[]' + `; + try { + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode, + outputPipeline: false, + }); + assert.fail('should not pass'); + } catch (err) { + assert(err.message.indexOf(formatErrorSearchString) !== -1); + } + }); + it('should throw an error if the pivot has an empty columns argument', async () => { + const queryText = ` + SELECT 'AverageCost' as CostSortedByProductionDays, + "0", + "1", + "2", + "3", + "4" + FROM ( + SELECT DaysToManufacture, + StandardCost + FROM Production_Product + GROUP BY DaysToManufacture, StandardCost + ORDER BY DaysToManufacture, StandardCost + ) 'pvt|pivot([avg(StandardCost) as AverageCost],DaysToManufacture,[,,]' + `; + try { + await queryResultTester({ + queryString: queryText, + casePath: 'pivot.case1', + mode, + outputPipeline: false, + }); + assert.fail('should not pass'); + } catch (err) { + assert(err.message.indexOf(formatErrorSearchString) !== -1); + } + }); }); describe('UNPIVOT', () => { @@ -456,7 +709,7 @@ describe('node-sql-parser upgrade tests', function () { FROM ( SELECT VendorID, Emp1, Emp2, Emp3, Emp4, Emp5, unset(_id) FROM pvt - ) 'unpvt|unpivot([Orders],Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])' + ) 'unpvt|unpivot(Orders,Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])' ORDER BY VendorID, Employee `; @@ -480,7 +733,7 @@ describe('node-sql-parser upgrade tests', function () { FROM ( SELECT SalesID, Order1Name, Order2Name, Order1Date, Order2Date, Order1Amt, Order2Amt, unset(_id) FROM multiple-unpivot - ) 'unpvt|unpivot([OrderName],OrderNames,[Order1Name, Order2Name])|unpivot([OrderDate],OrderDates,[Order1Date, Order2Date])|unpivot([OrderAmt],OrderAmts,[Order1Amt, Order2Amt])' + ) 'unpvt|unpivot(OrderName,OrderNames,[Order1Name, Order2Name])|unpivot(OrderDate,OrderDates,[Order1Date, Order2Date])|unpivot(OrderAmt,OrderAmts,[Order1Amt, Order2Amt])' `; await queryResultTester({ queryString: queryText, From 8808907be2d4d8f1289367e43ffc54d3989de3f5 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Mon, 9 Sep 2024 14:25:05 +0200 Subject: [PATCH 15/18] 4.1.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a9782276..0745a623 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@synatic/noql", - "version": "4.1.6", + "version": "4.1.7", "description": "Convert SQL statements to mongo queries or aggregates", "main": "index.js", "files": [ From 6f9de263949122728a1f6db7ef1064b6e197e74f Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Tue, 10 Sep 2024 11:33:24 +0200 Subject: [PATCH 16/18] feat(full-join): adding missing steps, adding the ability to specify that the projection has already been done --- lib/make/makeAggregatePipeline.js | 120 ++++++++++++++++-------------- lib/make/makeJoinForPipeline.js | 70 ++++++++++++++++- lib/types.ts | 1 + test/joins/join-cases.json | 67 ++++------------- test/joins/joins.test.js | 98 +++++++----------------- 5 files changed, 170 insertions(+), 186 deletions(-) diff --git a/lib/make/makeAggregatePipeline.js b/lib/make/makeAggregatePipeline.js index b3d90086..72bd7ce4 100644 --- a/lib/make/makeAggregatePipeline.js +++ b/lib/make/makeAggregatePipeline.js @@ -21,7 +21,7 @@ const { applyMultipleUnpivots, } = require('./apply-pivot'); exports.makeAggregatePipeline = makeAggregatePipeline; - +exports.stripJoinHints = stripJoinHints; /** * *Checks whether the query needs to force a group by @@ -341,67 +341,73 @@ function makeAggregatePipeline(ast, context = {}) { !isSelectAll(ast.columns) && ast.columns.length > 0 ) { - /** @type {import('../types').Column[]} */ - // @ts-ignore - const columns = ast.columns; - columns.forEach((column) => { - projectColumnParserModule.projectColumnParser( - column, - result, - context, - ast.from && ast.from[0] ? stripJoinHints(ast.from[0].as) : null - ); - }); - if (result.count.length > 0) { - result.count.forEach((countStep) => pipeline.push(countStep)); - } - if (result.unset) { - pipeline.push(result.unset); - } - if (result.windowFields) { - for (const windowField of result.windowFields) { - pipeline.push({ - $setWindowFields: windowField, - }); + if (!context.projectionAlreadyAdded) { + /** @type {import('../types').Column[]} */ + // @ts-ignore + const columns = ast.columns; + columns.forEach((column) => { + projectColumnParserModule.projectColumnParser( + column, + result, + context, + ast.from && ast.from[0] + ? stripJoinHints(ast.from[0].as) + : null + ); + }); + if (result.count.length > 0) { + result.count.forEach((countStep) => pipeline.push(countStep)); } - result.windowFields = []; - } - if (!$check.emptyObject(result.parsedProject.$project)) { - if (result.exprToMerge && result.exprToMerge.length > 0) { - pipeline.push({ - $replaceRoot: { - newRoot: { - $mergeObjects: result.exprToMerge.concat( - result.parsedProject.$project - ), - }, - }, - }); - } else { - if ( - (ast.distinct && - ast.distinct.toLowerCase && - ast.distinct.toLowerCase() === 'distinct') || - (ast.distinct && - ast.distinct.type && - ast.distinct.type.toLowerCase && - ast.distinct.type.toLowerCase() === 'distinct') - ) { + if (result.unset) { + pipeline.push(result.unset); + } + if (result.windowFields && result.windowFields.length) { + for (const windowField of result.windowFields) { pipeline.push({ - $group: {_id: result.parsedProject.$project}, + $setWindowFields: windowField, + }); + } + result.windowFields = []; + } + if (!$check.emptyObject(result.parsedProject.$project)) { + if (result.exprToMerge && result.exprToMerge.length > 0) { + pipeline.push({ + $replaceRoot: { + newRoot: { + $mergeObjects: result.exprToMerge.concat( + result.parsedProject.$project + ), + }, + }, }); - const newProject = {}; - for (const k in result.parsedProject.$project) { - // eslint-disable-next-line no-prototype-builtins - if (!result.parsedProject.$project.hasOwnProperty(k)) { - continue; + } else { + if ( + (ast.distinct && + ast.distinct.toLowerCase && + ast.distinct.toLowerCase() === 'distinct') || + (ast.distinct && + ast.distinct.type && + ast.distinct.type.toLowerCase && + ast.distinct.type.toLowerCase() === 'distinct') + ) { + pipeline.push({ + $group: {_id: result.parsedProject.$project}, + }); + const newProject = {}; + for (const k in result.parsedProject.$project) { + // eslint-disable-next-line no-prototype-builtins + if ( + !result.parsedProject.$project.hasOwnProperty(k) + ) { + continue; + } + newProject[k] = `$_id.${k}`; } - newProject[k] = `$_id.${k}`; + newProject['_id'] = 0; + pipeline.push({$project: newProject}); + } else { + pipeline.push(result.parsedProject); } - newProject['_id'] = 0; - pipeline.push({$project: newProject}); - } else { - pipeline.push(result.parsedProject); } } } diff --git a/lib/make/makeJoinForPipeline.js b/lib/make/makeJoinForPipeline.js index 6bd986a6..a226408e 100644 --- a/lib/make/makeJoinForPipeline.js +++ b/lib/make/makeJoinForPipeline.js @@ -2,6 +2,8 @@ const makeFilterConditionModule = require('./makeFilterCondition'); const $check = require('check-types'); const $json = require('@synatic/json-magic'); const makeAggregatePipelineModule = require('./makeAggregatePipeline'); +const projectColumnParserModule = require('./projectColumnParser'); +const {createResultObject} = require('./createResultObject'); exports.makeJoinForPipeline = makeJoinForPipeline; @@ -359,16 +361,24 @@ function tableJoin( } else if (join.join === 'LEFT JOIN') { // dont need anything } else if (join.join === 'FULL JOIN') { + const toCollection = toAs || toTable; + const localTable = previousJoin.table || localPart.table; + pipeline.push({ + $unwind: { + path: `$${toCollection}`, + preserveNullAndEmptyArrays: true, + }, + }); pipeline.push({ $unionWith: { - coll: join.table, + coll: toTable, pipeline: [ { $lookup: { - from: toTable, + from: localTable, localField: localPart.column, foreignField: fromPart.column, - as: fromPart.table, + as: localPart.table, }, }, ], @@ -376,10 +386,62 @@ function tableJoin( }); pipeline.push({ $unwind: { - path: `$${fromPart.table}`, + path: `$${localPart.table}`, preserveNullAndEmptyArrays: true, }, }); + const result = createResultObject(); + const ast = context.fullAst.ast; + const columns = ast.columns; + columns.forEach((column) => { + projectColumnParserModule.projectColumnParser( + column, + result, + context, + ast.from && ast.from[0] + ? makeAggregatePipelineModule.stripJoinHints(ast.from[0].as) + : null + ); + }); + const project = (result.parsedProject || {}).$project; + if (!project) { + throw new Error(`Unable to get $projection for full outer join`); + } + // todo need the projection part and the schema + pipeline.push({ + $project: { + ...Object.entries(project).reduce((previous, current) => { + const [key, value] = current; + previous[key] = { + $ifNull: [`$${key}`, value], + }; + return previous; + }, {}), + }, + }); + pipeline.push({ + $group: { + _id: { + ...Object.keys(project).reduce((previous, current) => { + previous[current] = `$${current}`; + return previous; + }, {}), + }, + }, + }); + pipeline.push({ + $project: { + _id: 0, + ...Object.keys(project).reduce((previous, current) => { + previous[current] = `$_id.${current}`; + return previous; + }, {}), + }, + }); + context.projectionAlreadyAdded = true; + if (result.unset) { + pipeline.push(result.unset); + } } else { throw new Error(`Join not supported:${join.join}`); } diff --git a/lib/types.ts b/lib/types.ts index 52a71404..f75d1878 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -200,6 +200,7 @@ export interface NoqlContext extends ParserOptions { tables: string[]; fullAst: TableColumnAst; joinHints?: string[]; + projectionAlreadyAdded?: boolean; } export interface ParseResult { diff --git a/test/joins/join-cases.json b/test/joins/join-cases.json index 08e6dd87..29e3d588 100644 --- a/test/joins/join-cases.json +++ b/test/joins/join-cases.json @@ -1781,81 +1781,42 @@ }, "full-outer-join": { "case1": { - "pipeline": [ - { - "$project": { - "c": "$$ROOT" - } - }, - { - "$lookup": { - "from": "foj-orders", - "as": "o", - "localField": "c.customerId", - "foreignField": "customerId" - } - }, + "expectedResults": [ { - "$unionWith": { - "coll": "foj-orders", - "pipeline": [ - { - "$lookup": { - "from": "foj-orders", - "localField": "customerId", - "foreignField": "customerId", - "as": "o" - } - } - ] - } + "orderId": 10309 }, { - "$unwind": { - "path": "$o", - "preserveNullAndEmptyArrays": true - } + "orderId": 10310 }, { - "$group": { - "_id": { - "customerName": "$c.customerName", - "orderId": "$o.orderId" - } - } + "customerName": "Alfreds Futterkiste" }, { - "$project": { - "customerName": "$_id.customerName", - "orderId": "$_id.orderId", - "_id": 0 - } + "customerName": "Ana Trujillo Emparedados y helados", + "orderId": 10308 }, { - "$sort": { - "customerName": -1 - } + "customerName": "Antonio Moreno Taquería" } - ], + ] + }, + "case2": { "expectedResults": [ { - "customerName": "Antonio Moreno Taquería" + "orderId": 10309 }, { - "customerName": "Ana Trujillo Emparedados y helados", - "orderId": 10308 + "orderId": 10310 }, { "customerName": "Alfreds Futterkiste" }, { + "customerName": "Ana Trujillo Emparedados y helados", "orderId": 10308 }, { - "orderId": 10309 - }, - { - "orderId": 10310 + "customerName": "Antonio Moreno Taquería" } ] } diff --git a/test/joins/joins.test.js b/test/joins/joins.test.js index f1ecf8fb..d5949565 100644 --- a/test/joins/joins.test.js +++ b/test/joins/joins.test.js @@ -693,85 +693,39 @@ describe('joins', function () { }); describe('full outer join', () => { - it('should work', async () => { + it('should work - case 1', async () => { + // https://www.w3schools.com/Sql/sql_join_full.asp const queryString = ` SELECT c.customerName as customerName, - o.orderId as orderId + o.orderId as orderId, + unset(_id) FROM "foj-customers" c FULL OUTER JOIN "foj-orders" o ON c.customerId = o.customerId - ORDER BY c.customerName DESC`; - const {pipeline} = await queryResultTester({ + ORDER BY c.customerName ASC, orders.orderId ASC`; + await queryResultTester({ queryString, casePath: 'full-outer-join.case1', - mode: 'write', - outputPipeline: true, - }); - const expectedPipeline = [ - { - $lookup: { - from: 'foj-customers', - localField: 'customerId', - foreignField: 'customerId', - as: 'customers', - }, - }, - { - $unwind: { - path: '$customers', - preserveNullAndEmptyArrays: true, - }, - }, - { - $unionWith: { - coll: 'foj-customers', - pipeline: [ - { - $lookup: { - from: 'foj-orders', - localField: 'customerId', - foreignField: 'customerId', - as: 'orders', - }, - }, - ], - }, - }, - { - $unwind: { - path: '$orders', - preserveNullAndEmptyArrays: true, - }, - }, - { - $project: { - customerName: { - $ifNull: [ - '$customerName', - '$customers.customerName', - ], - }, - orderId: { - $ifNull: ['$orderId', '$orders.orderId'], - }, - }, - }, - { - $group: { - _id: { - customerName: '$customerName', - orderId: '$orderId', - }, - }, - }, - { - $project: { - _id: 0, - customerName: '$_id.customerName', - orderId: '$_id.orderId', - }, - }, - ]; + mode, + outputPipeline: false, + }); + }); + it('should work - case 2', async () => { + // https://www.w3schools.com/Sql/sql_join_full.asp + const queryString = ` + SELECT customers.customerName as customerName, + orders.orderId as orderId, + unset(_id) + FROM "foj-orders" orders + FULL OUTER JOIN "foj-customers" customers + ON orders.customerId = customers.customerId + ORDER BY customers.customerName ASC, orders.orderId ASC`; + await queryResultTester({ + queryString, + casePath: 'full-outer-join.case2', + mode, + outputPipeline: false, + }); }); }); }); From 95c4a9794c972600a41bcf3c43fbe022b2c5f6fc Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Tue, 10 Sep 2024 20:05:24 +0200 Subject: [PATCH 17/18] Fixing up additional test cases and associated logic --- lib/make/makeJoinForPipeline.js | 22 ++++++++++++++------ test/individualTests/upgrade.json | 29 +++++++++++++++++++++++++++ test/individualTests/upgrade.test.js | 26 ++++++++++++++++++++++++ test/joins/join-cases.json | 30 ++++++++++++++++++++++++++++ test/joins/joins.test.js | 19 ++++++++++++++++++ 5 files changed, 120 insertions(+), 6 deletions(-) diff --git a/lib/make/makeJoinForPipeline.js b/lib/make/makeJoinForPipeline.js index a226408e..1db4d606 100644 --- a/lib/make/makeJoinForPipeline.js +++ b/lib/make/makeJoinForPipeline.js @@ -46,7 +46,14 @@ function makeJoinForPipeline(ast, context) { .map((a) => a.split('|')[0]); for (let i = 1; i < ast.from.length; i++) { - makeJoinPart(ast.from[i], ast.from[i - 1], aliases, pipeline, context); + makeJoinPart( + ast.from[i], + ast.from[i - 1], + aliases, + pipeline, + context, + ast + ); } return pipeline; @@ -59,9 +66,10 @@ function makeJoinForPipeline(ast, context) { * @param {string[]} aliases - the aliases used in the joins * @param {import('../types').PipelineFn[]} pipeline * @param {import('../types').NoqlContext} context - The Noql context to use when generating the output + * @param {import('../types').TableColumnAst} ast * @returns {void} */ -function makeJoinPart(join, previousJoin, aliases, pipeline, context) { +function makeJoinPart(join, previousJoin, aliases, pipeline, context, ast) { let toTable = join.table || ''; let toAs = join.as || ''; @@ -89,7 +97,8 @@ function makeJoinPart(join, previousJoin, aliases, pipeline, context) { toTable, toAs, joinHints, - context + context, + ast ); } const prefixLeft = shouldPrefixSide('left'); @@ -279,6 +288,7 @@ function sanitizeOnCondition(condition, joinAliases) { * @param {string} toAs * @param {string[]} joinHints * @param {import('../types').NoqlContext} context - The Noql context to use when generating the output + * @param {import('../types').TableColumnAst} ast * @returns {void} */ function tableJoin( @@ -288,7 +298,8 @@ function tableJoin( toTable, toAs, joinHints, - context + context, + ast ) { let localPart; let fromPart; @@ -391,7 +402,6 @@ function tableJoin( }, }); const result = createResultObject(); - const ast = context.fullAst.ast; const columns = ast.columns; columns.forEach((column) => { projectColumnParserModule.projectColumnParser( @@ -404,7 +414,7 @@ function tableJoin( ); }); const project = (result.parsedProject || {}).$project; - if (!project) { + if (!project || Object.keys(project).length === 0) { throw new Error(`Unable to get $projection for full outer join`); } // todo need the projection part and the schema diff --git a/test/individualTests/upgrade.json b/test/individualTests/upgrade.json index b61f016a..d9d4d837 100644 --- a/test/individualTests/upgrade.json +++ b/test/individualTests/upgrade.json @@ -283,6 +283,35 @@ "OrderAmt": 222 } ] + }, + "full-outer-join": { + "expectedResults": [ + { + "VendorID": 1, + "Employee": "Emp1", + "Orders": 4 + }, + { + "VendorID": 1, + "Employee": "Emp2", + "Orders": 3 + }, + { + "VendorID": 1, + "Employee": "Emp3", + "Orders": 5 + }, + { + "VendorID": 1, + "Employee": "Emp4", + "Orders": 4 + }, + { + "VendorID": 1, + "Employee": "Emp5", + "Orders": 4 + } + ] } } } \ No newline at end of file diff --git a/test/individualTests/upgrade.test.js b/test/individualTests/upgrade.test.js index 42e20707..20755ac3 100644 --- a/test/individualTests/upgrade.test.js +++ b/test/individualTests/upgrade.test.js @@ -742,6 +742,32 @@ describe('node-sql-parser upgrade tests', function () { outputPipeline: false, }); }); + it('should support unpivot on an outer join', async () => { + const queryString = ` + SELECT VendorID, Employee, Orders + FROM ( + SELECT c.customerName as customerName, + o.orderId as orderId, + 1 as VendorID, + 4 as Emp1, + 3 as Emp2, + 5 as Emp3, + 4 as Emp4, + 4 as Emp5, + unset(_id,orderId,customerName) + FROM "foj-customers" c + FULL OUTER JOIN "foj-orders" o + ON c.customerId = o.customerId + ORDER BY c.customerName ASC, orders.orderId ASC + LIMIT 1 + ) 'unpvt|unpivot(Orders,Employee,[Emp1, Emp2, Emp3, Emp4, Emp5])'`; + await queryResultTester({ + queryString, + casePath: 'unpivot.full-outer-join', + mode, + outputPipeline: false, + }); + }); }); }); }); diff --git a/test/joins/join-cases.json b/test/joins/join-cases.json index 29e3d588..6979b76f 100644 --- a/test/joins/join-cases.json +++ b/test/joins/join-cases.json @@ -1819,6 +1819,36 @@ "customerName": "Antonio Moreno Taquería" } ] + }, + "case3-sub-select": { + "expectedResults": [ + { + "foj": { + "orderId": 10309 + } + }, + { + "foj": { + "orderId": 10310 + } + }, + { + "foj": { + "customerName": "Alfreds Futterkiste" + } + }, + { + "foj": { + "customerName": "Ana Trujillo Emparedados y helados", + "orderId": 10308 + } + }, + { + "foj": { + "customerName": "Antonio Moreno Taquería" + } + } + ] } } } \ No newline at end of file diff --git a/test/joins/joins.test.js b/test/joins/joins.test.js index d5949565..e880563f 100644 --- a/test/joins/joins.test.js +++ b/test/joins/joins.test.js @@ -727,5 +727,24 @@ describe('joins', function () { outputPipeline: false, }); }); + it('should support a full outer join in a subselect', async () => { + const queryString = ` + SELECT * + FROM ( + SELECT c.customerName as customerName, + o.orderId as orderId, + unset(_id) + FROM "foj-customers" c + FULL OUTER JOIN "foj-orders" o + ON c.customerId = o.customerId + ORDER BY c.customerName ASC, orders.orderId ASC + ) foj`; + await queryResultTester({ + queryString, + casePath: 'full-outer-join.case3-sub-select', + mode, + outputPipeline: false, + }); + }); }); }); From 072d370f2adbae1f2109abb2df131c0797315795 Mon Sep 17 00:00:00 2001 From: Ryan Kotzen Date: Tue, 10 Sep 2024 20:19:38 +0200 Subject: [PATCH 18/18] fixing lint issue --- lib/make/apply-pivot.js | 1 - lib/make/makeAggregatePipeline.js | 121 ++++++++++++++---------------- 2 files changed, 58 insertions(+), 64 deletions(-) diff --git a/lib/make/apply-pivot.js b/lib/make/apply-pivot.js index ccf60ec0..7fa1efbf 100644 --- a/lib/make/apply-pivot.js +++ b/lib/make/apply-pivot.js @@ -1,5 +1,4 @@ const {functionByName} = require('../MongoFunctions'); -const $check = require('check-types'); module.exports = {applyPivot, applyUnpivot, applyMultipleUnpivots}; diff --git a/lib/make/makeAggregatePipeline.js b/lib/make/makeAggregatePipeline.js index 72bd7ce4..926ba076 100644 --- a/lib/make/makeAggregatePipeline.js +++ b/lib/make/makeAggregatePipeline.js @@ -339,75 +339,70 @@ function makeAggregatePipeline(ast, context = {}) { } else if ( ast.columns && !isSelectAll(ast.columns) && - ast.columns.length > 0 + ast.columns.length > 0 && + !context.projectionAlreadyAdded ) { - if (!context.projectionAlreadyAdded) { - /** @type {import('../types').Column[]} */ - // @ts-ignore - const columns = ast.columns; - columns.forEach((column) => { - projectColumnParserModule.projectColumnParser( - column, - result, - context, - ast.from && ast.from[0] - ? stripJoinHints(ast.from[0].as) - : null - ); - }); - if (result.count.length > 0) { - result.count.forEach((countStep) => pipeline.push(countStep)); - } - if (result.unset) { - pipeline.push(result.unset); - } - if (result.windowFields && result.windowFields.length) { - for (const windowField of result.windowFields) { - pipeline.push({ - $setWindowFields: windowField, - }); - } - result.windowFields = []; + /** @type {import('../types').Column[]} */ + // @ts-ignore + const columns = ast.columns; + columns.forEach((column) => { + projectColumnParserModule.projectColumnParser( + column, + result, + context, + ast.from && ast.from[0] ? stripJoinHints(ast.from[0].as) : null + ); + }); + if (result.count.length > 0) { + result.count.forEach((countStep) => pipeline.push(countStep)); + } + if (result.unset) { + pipeline.push(result.unset); + } + if (result.windowFields && result.windowFields.length) { + for (const windowField of result.windowFields) { + pipeline.push({ + $setWindowFields: windowField, + }); } - if (!$check.emptyObject(result.parsedProject.$project)) { - if (result.exprToMerge && result.exprToMerge.length > 0) { - pipeline.push({ - $replaceRoot: { - newRoot: { - $mergeObjects: result.exprToMerge.concat( - result.parsedProject.$project - ), - }, + result.windowFields = []; + } + if (!$check.emptyObject(result.parsedProject.$project)) { + if (result.exprToMerge && result.exprToMerge.length > 0) { + pipeline.push({ + $replaceRoot: { + newRoot: { + $mergeObjects: result.exprToMerge.concat( + result.parsedProject.$project + ), }, + }, + }); + } else { + if ( + (ast.distinct && + ast.distinct.toLowerCase && + ast.distinct.toLowerCase() === 'distinct') || + (ast.distinct && + ast.distinct.type && + ast.distinct.type.toLowerCase && + ast.distinct.type.toLowerCase() === 'distinct') + ) { + pipeline.push({ + $group: {_id: result.parsedProject.$project}, }); - } else { - if ( - (ast.distinct && - ast.distinct.toLowerCase && - ast.distinct.toLowerCase() === 'distinct') || - (ast.distinct && - ast.distinct.type && - ast.distinct.type.toLowerCase && - ast.distinct.type.toLowerCase() === 'distinct') - ) { - pipeline.push({ - $group: {_id: result.parsedProject.$project}, - }); - const newProject = {}; - for (const k in result.parsedProject.$project) { - // eslint-disable-next-line no-prototype-builtins - if ( - !result.parsedProject.$project.hasOwnProperty(k) - ) { - continue; - } - newProject[k] = `$_id.${k}`; + const newProject = {}; + for (const k in result.parsedProject.$project) { + // eslint-disable-next-line no-prototype-builtins + if (!result.parsedProject.$project.hasOwnProperty(k)) { + continue; } - newProject['_id'] = 0; - pipeline.push({$project: newProject}); - } else { - pipeline.push(result.parsedProject); + newProject[k] = `$_id.${k}`; } + newProject['_id'] = 0; + pipeline.push({$project: newProject}); + } else { + pipeline.push(result.parsedProject); } } }