Skip to content

Commit 98c7928

Browse files
rotem-codefreshrotem-codefresh
authored
offline-logs-improvements (#713)
- fixed example string - added env variables names in --help - exposed error types to users - createIndex: will be performed on background - adjustments to older mongo versions - each archive collection name will be pre-fixed with the date range
1 parent dc81849 commit 98c7928

File tree

7 files changed

+347
-181
lines changed

7 files changed

+347
-181
lines changed

‎lib/interface/cli/commands/offline-logs/base.cmd.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@ const command = new Command({
1616
.env("RUNTIME_MONGO")
1717
.option("uri", {
1818
describe:
19-
"Mongodb URI. If not provided, will be parsed from environment variables.",
19+
"Mongodb URI. If not provided, will be parsed from environment variable RUNTIME_MONGO_URI.",
2020
type: "string",
2121
})
2222
.option("db", {
2323
describe:
24-
"Database name. If not provided, will be parsed from environment variables.",
24+
"Database name. If not provided, will be parsed from environment variable RUNTIME_MONGO_DB.",
2525
type: "string",
2626
});
2727
},

‎lib/interface/cli/commands/offline-logs/ensure-index.cmd.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,24 @@ const command = new Command({
1818
),
1919
handler: async (argv) => {
2020
const { uri, db } = argv;
21-
const client = new MongoClient(uri);
21+
const client = new MongoClient( uri, { useUnifiedTopology: true } );
2222
await client.connect();
2323
const database = client.db(db);
2424
const failedCollections = [];
25+
const errors = [];
2526
for (const collection of utils.defaultCollections) {
2627
try {
2728
await utils.ensureIndex(database, collection);
2829
} catch (error) {
2930
console.error(`failed to ensure index of collection '${collection}', error: ${error.message}`);
3031
failedCollections.push(collection);
32+
errors.push(error);
3133
}
3234
}
3335
client.close();
3436
if (failedCollections.length) {
3537
throw new Error(
36-
`failed to ensure indexes of ${failedCollections.join(", ")}`
38+
`failed to ensure indexes of ${failedCollections.join(', ')}, ${errors.join(', ')}`
3739
);
3840
}
3941
},

‎lib/interface/cli/commands/offline-logs/ensure-index.spec.js

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
// const {Db, Collection} = require('mongodb');
2-
const { Collection } = require('yaml/types');
31
const utils = require('./utils')
42

53
describe("ensure-index", () => {
@@ -29,6 +27,7 @@ describe("ensure-index", () => {
2927
utils.getUserInput.mockReturnValue(true);
3028
const collection = 'whatever';
3129
const expectedIndexObj = { accountId: 1, jobId: 1 }
30+
const expectedOptions = { background: true }
3231

3332
//execute
3433
await utils.ensureIndex(mockDatabase, collection);
@@ -39,7 +38,7 @@ describe("ensure-index", () => {
3938
expect(mockCollection.estimatedDocumentCount).toBeCalledTimes(1);
4039
expect(utils.getUserInput).toBeCalledTimes(1);
4140
expect(mockCollection.createIndex).toBeCalledTimes(1);
42-
expect(mockCollection.createIndex).toBeCalledWith(expectedIndexObj)
41+
expect(mockCollection.createIndex).toBeCalledWith(expectedIndexObj, expectedOptions)
4342
});
4443
it("index does not exists and user says no on prompt", async () => {
4544
//setup

‎lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js

Lines changed: 177 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -1,96 +1,188 @@
11
const { MongoClient, ObjectId } = require("mongodb");
2-
const moment = require('moment')
3-
const Command = require('../../Command');
4-
const cmd = require('./base.cmd');
5-
const { objectIdFromDate, defaultCollections } = require('./utils')
6-
7-
const offloadToCollection = async function(sourceDBObj, collection, targetDB, cutoffDate) {
8-
const sourceCollectionObj = sourceDBObj.collection(collection)
9-
const targetCollection = `archive-${collection}`
10-
11-
const cutoffDateObj = moment(cutoffDate)
12-
.add(1, 'days')
13-
.startOf("day")
14-
15-
if(!cutoffDateObj.isValid()){
16-
throw new Error('please enter a valid date in ISO 8601 format')
2+
const moment = require("moment");
3+
const semver = require("semver");
4+
const Command = require("../../Command");
5+
const cmd = require("./base.cmd");
6+
const {
7+
objectIdFromDate,
8+
defaultCollections,
9+
findMinLog,
10+
checkCursorState,
11+
createRangeStr,
12+
getMinDate,
13+
} = require("./utils");
14+
15+
const MERGE_RELEASE_VERSION = "4.2.0";
16+
17+
const offloadToCollection = async function (
18+
sourceDBObj,
19+
collection,
20+
targetDB,
21+
cutoffDate
22+
) {
23+
const sourceCollectionObj = sourceDBObj.collection(collection);
24+
25+
const cutoffDateObj = moment(cutoffDate).add(1, "days").startOf("day");
26+
27+
if (!cutoffDateObj.isValid()) {
28+
throw new Error("please enter a valid date in ISO 8601 format");
1729
}
1830

19-
const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate())
31+
const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate());
2032

21-
var result = sourceCollectionObj.aggregate([
22-
{ $match: { _id: { $lte: ObjectId(cutoffDateId) } } },
23-
{ $merge: {
24-
into: {db: targetDB, coll: targetCollection},
25-
on: "_id",
26-
whenMatched: "keepExisting",
27-
whenNotMatched: "insert"
28-
}}
29-
])
30-
31-
await result.toArray()
32-
33-
if (!result.cursorState.killed){
34-
console.info(`logs from '${collection}' were archived to '${targetCollection}'`)
35-
await sourceCollectionObj.deleteMany({_id: {$lte: ObjectId(cutoffDateId)}})
33+
const minLog = await findMinLog(
34+
sourceCollectionObj,
35+
cutoffDateId,
36+
collection
37+
);
38+
39+
if (!minLog) {
40+
console.info(
41+
`No logs to archive in collection ${collection} from the given date.`
42+
);
43+
return;
3644
}
37-
else {
38-
console.error("Cursor error. Archiving operation may not be completed.")
39-
console.error("The old logs were not deleted from the source collection.")
45+
46+
const lowerBound = getMinDate(minLog);
47+
const upperBound = moment(cutoffDateObj);
48+
const targetCollection = `${createRangeStr(
49+
lowerBound,
50+
upperBound,
51+
collection
52+
)}-archive`;
53+
54+
const mongoInfo = await sourceDBObj.admin().serverInfo();
55+
const mongoServerVersion = await mongoInfo.version;
56+
57+
if (semver.lte(MERGE_RELEASE_VERSION, mongoServerVersion)) {
58+
await archiveWithMerge(
59+
sourceCollectionObj,
60+
cutoffDateId,
61+
targetDB,
62+
targetCollection,
63+
collection
64+
);
65+
} else {
66+
await archiveWithOut(
67+
sourceCollectionObj,
68+
cutoffDateId,
69+
targetCollection,
70+
collection
71+
);
4072
}
41-
}
73+
};
74+
///////////////////////////////////////////////////////////////////////////////////
75+
const archiveWithMerge = async function (
76+
sourceCollectionObj,
77+
cutoffDateId,
78+
targetDB,
79+
targetCollection,
80+
collection
81+
) {
82+
var result = sourceCollectionObj.aggregate([
83+
{ $match: { _id: { $lte: ObjectId(cutoffDateId) } } },
84+
{
85+
$merge: {
86+
into: { db: targetDB, coll: targetCollection },
87+
on: "_id",
88+
whenMatched: "keepExisting",
89+
whenNotMatched: "insert",
90+
},
91+
},
92+
]);
4293

94+
await result.toArray();
95+
96+
checkCursorState(result, collection, targetCollection);
97+
98+
await sourceCollectionObj.deleteMany({
99+
_id: { $lte: ObjectId(cutoffDateId) },
100+
});
101+
};
102+
///////////////////////////////////////////////////////////////////////////////////
103+
const archiveWithOut = async function (
104+
sourceCollectionObj,
105+
cutoffDateId,
106+
targetCollection,
107+
collection
108+
) {
109+
var result = sourceCollectionObj.aggregate([
110+
{ $match: { _id: { $lte: ObjectId(cutoffDateId) } } },
111+
{ $out: targetCollection },
112+
]);
113+
114+
await result.toArray();
115+
116+
checkCursorState(result, collection, targetCollection);
117+
118+
await sourceCollectionObj.deleteMany({
119+
_id: { $lte: ObjectId(cutoffDateId) },
120+
});
121+
};
122+
///////////////////////////////////////////////////////////////////////////////////
43123
const command = new Command({
44-
command: 'offload-to-collection',
45-
parent: cmd,
46-
description: 'Archiving logs from one or more source collections to target collections.',
47-
webDocs: {
48-
category: 'Logs',
49-
title: 'Offload To Collection',
50-
},
51-
builder: yargs => yargs
52-
.option('targetDB', {
53-
alias: 'tdb',
54-
describe: "Target database name, if none inserted, db will be defined as target.",
55-
type: "string",
56-
})
57-
.option('cutoffDate', {
58-
alias: "cod",
59-
describe:
60-
"The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
61-
demandOption: true,
62-
type: "string",
63-
})
64-
.example('codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --c logs foo --cod "2021-07-08" '),
65-
handler: async (argv) => {
66-
const {
67-
uri,
68-
db,
69-
targetDB,
70-
cutoffDate,
71-
} = argv
72-
73-
const client = new MongoClient(uri);
74-
try{
75-
await client.connect()
76-
const failedCollections = [];
77-
const sourceDBObj = client.db(db);
78-
const promises = defaultCollections.map( async (collection) => {
79-
try{
80-
await offloadToCollection(sourceDBObj, collection, targetDB || db, cutoffDate);
81-
} catch (error) {
82-
failedCollections.push(collection)
83-
}
84-
})
85-
await Promise.all(promises)
86-
87-
if (failedCollections.length){
88-
throw new Error(`failed to offload from collections: ${failedCollections.join(', ')}`)
124+
command: "offload-to-collection",
125+
parent: cmd,
126+
description:
127+
"Archiving logs from one or more source collections to target collections.",
128+
webDocs: {
129+
category: "Logs",
130+
title: "Offload To Collection",
131+
},
132+
builder: (yargs) =>
133+
yargs
134+
.option("targetDB", {
135+
alias: "tdb",
136+
describe:
137+
"This option is available only for mongodb version 4.2 and up. for older versions it will be ignored. \
138+
Target database name, if none inserted, db will be defined as target.",
139+
type: "string",
140+
})
141+
.option("cutoffDate", {
142+
alias: "cod",
143+
describe:
144+
"The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
145+
demandOption: true,
146+
type: "string",
147+
})
148+
.example(
149+
'codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --cod "2021-07-08" '
150+
),
151+
handler: async (argv) => {
152+
const { uri, db, targetDB, cutoffDate } = argv;
153+
154+
const client = new MongoClient(uri, { useUnifiedTopology: true });
155+
try {
156+
await client.connect();
157+
const failedCollections = [];
158+
const errors = [];
159+
const sourceDBObj = client.db(db);
160+
const promises = defaultCollections.map(async (collection) => {
161+
try {
162+
await offloadToCollection(
163+
sourceDBObj,
164+
collection,
165+
targetDB || db,
166+
cutoffDate
167+
);
168+
} catch (error) {
169+
failedCollections.push(collection);
170+
errors.push(error);
89171
}
90-
} finally {
91-
client.close();
172+
});
173+
await Promise.all(promises);
174+
175+
if (failedCollections.length) {
176+
throw new Error(
177+
`failed to offload from collections: ${failedCollections.join(
178+
", "
179+
)}. ${errors.join(", ")}`
180+
);
92181
}
93-
},
94-
})
182+
} finally {
183+
client.close();
184+
}
185+
},
186+
});
95187

96-
module.exports = command;
188+
module.exports = command;

0 commit comments

Comments
 (0)