Skip to content

Commit 2804065

Browse files
rotem-codefreshrotem-codefresh
authored
fixed bug, added env variables (#712)
* fixed bug, added env variables * deleted json files
1 parent ce6a8d5 commit 2804065

File tree

5 files changed

+97
-62
lines changed

5 files changed

+97
-62
lines changed

‎lib/interface/cli/commands/offline-logs/base.cmd.js

Lines changed: 28 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,33 @@
1-
const yargs = require('yargs');
2-
const Command = require('../../Command');
1+
const yargs = require("yargs");
2+
const Command = require("../../Command");
33

44
const command = new Command({
5-
command: 'offline-logs',
6-
root: true,
7-
description: 'Manages offline logs',
8-
webDocs: {
9-
category: 'Logs category',
10-
subCategory: 'Logs sub category',
11-
title: 'Archives old logs to file or collection.',
12-
},
13-
builder: (yargs) => {
14-
// set options which are used in both sub-commands
15-
return yargs
16-
.option('uri', {
17-
describe: "Mongodb URI",
18-
demandOption: true,
19-
type: "string",
20-
})
21-
.option('db', {
22-
describe: "Database name",
23-
demandOption: true,
24-
type: "string",
25-
})
26-
.option('collections', {
27-
alias: "c",
28-
describe: "Source collections names",
29-
default: ["logs", "metadata"],
30-
array: true,
31-
type: "string",
32-
})
33-
.option('cutoffDate', {
34-
alias: "cod",
35-
describe:
36-
"The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
37-
demandOption: true,
38-
type: "string",
39-
});
40-
},
41-
handler: () => {
42-
yargs.showHelp();
43-
},
5+
command: "offline-logs",
6+
root: true,
7+
description: "Manages offline logs",
8+
webDocs: {
9+
category: "Logs category",
10+
subCategory: "Logs sub category",
11+
title: "Archives old logs to file or collection.",
12+
},
13+
builder: (yargs) => {
14+
// set options which are used in both sub-commands
15+
return yargs
16+
.env("RUNTIME_MONGO")
17+
.option("uri", {
18+
describe:
19+
"Mongodb URI. If not provided, will be parsed from environment variables.",
20+
type: "string",
21+
})
22+
.option("db", {
23+
describe:
24+
"Database name. If not provided, will be parsed from environment variables.",
25+
type: "string",
26+
});
27+
},
28+
handler: () => {
29+
yargs.showHelp();
30+
},
4431
});
4532

4633
module.exports = command;

‎lib/interface/cli/commands/offline-logs/offload-to-collection.cmd.js

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,21 @@ const { MongoClient, ObjectId } = require("mongodb");
22
const moment = require('moment')
33
const Command = require('../../Command');
44
const cmd = require('./base.cmd');
5-
const { objectIdFromDate } = require('./utils')
5+
const { objectIdFromDate, defaultCollections } = require('./utils')
66

77
const offloadToCollection = async function(sourceDBObj, collection, targetDB, cutoffDate) {
88
const sourceCollectionObj = sourceDBObj.collection(collection)
99
const targetCollection = `archive-${collection}`
1010

1111
const cutoffDateObj = moment(cutoffDate)
1212
.add(1, 'days')
13-
.startOf("day").toDate()
13+
.startOf("day")
1414

1515
if(!cutoffDateObj.isValid()){
1616
throw new Error('please enter a valid date in ISO 8601 format')
1717
}
1818

19-
const cutoffDateId = objectIdFromDate(cutoffDateObj)
19+
const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate())
2020

2121
var result = sourceCollectionObj.aggregate([
2222
{ $match: { _id: { $lte: ObjectId(cutoffDateId) } } },
@@ -31,6 +31,7 @@ const offloadToCollection = async function(sourceDBObj, collection, targetDB, cu
3131
await result.toArray()
3232

3333
if (!result.cursorState.killed){
34+
console.info(`logs from '${collection}' were archived to '${targetCollection}'`)
3435
await sourceCollectionObj.deleteMany({_id: {$lte: ObjectId(cutoffDateId)}})
3536
}
3637
else {
@@ -53,21 +54,28 @@ const command = new Command({
5354
describe: "Target database name, if none inserted, db will be defined as target.",
5455
type: "string",
5556
})
56-
.example('codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --c logs foo --cod "2021-07-08" '),
57+
.option('cutoffDate', {
58+
alias: "cod",
59+
describe:
60+
"The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
61+
demandOption: true,
62+
type: "string",
63+
})
64+
.example('codefresh offline-logs offload-to-collection --uri "mongodb://192.168.99.100:27017" --db logs --c logs foo --cod "2021-07-08" '),
5765
handler: async (argv) => {
5866
const {
5967
uri,
6068
db,
61-
collections,
6269
targetDB,
6370
cutoffDate,
6471
} = argv
72+
6573
const client = new MongoClient(uri);
6674
try{
6775
await client.connect()
6876
const failedCollections = [];
6977
const sourceDBObj = client.db(db);
70-
const promises = collections.map( async (collection) => {
78+
const promises = defaultCollections.map( async (collection) => {
7179
try{
7280
await offloadToCollection(sourceDBObj, collection, targetDB || db, cutoffDate);
7381
} catch (error) {

‎lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@ const { MongoClient, ObjectId } = require("mongodb");
33
const moment = require('moment')
44
const Command = require('../../Command');
55
const cmd = require('./base.cmd');
6-
const {objectIdFromDate, writeLogsToFile, checkRemnant} = require('./utils')
7-
const { boolean } = require("yargs");
6+
const {objectIdFromDate, writeLogsToFile, checkRemnant, defaultCollections} = require('./utils');
87

98
const offloadToFile = async function(database, collection, chunkDuration, cutoffDate, path) {
109
const collectionObj = database.collection(collection);
@@ -93,12 +92,34 @@ const command = new Command({
9392
default: 1,
9493
type: "number",
9594
})
95+
.option('cutoffDate', {
96+
alias: "cod",
97+
describe:
98+
"The date in ISO format (yyyy-MM-dd) from which logs will be archived (going backwards, including logs from that day).",
99+
demandOption: true,
100+
type: "string",
101+
})
96102
.option('path', {
97103
describe: "Directory path to which archive files will be saved.",
98104
default: ".",
99105
type: "string",
100106
})
101-
.example('codefresh offline-logs offload-to-file --uri "mongodb://192.168.99.100:27017" --db logs --collections logs foo --cod "2021-07-08" --chdur 3 --path "./" '),
107+
.option('collections', {
108+
alias: 'c',
109+
describe: "Source collections names",
110+
default: defaultCollections,
111+
array: true,
112+
type: "string",
113+
coerce(arg) {
114+
for (const collection of arg) {
115+
if (!collection.includes('logs') && !collection.includes('metadata')) {
116+
throw new Error ('invalid collection name')
117+
}
118+
return arg
119+
}
120+
}
121+
})
122+
.example('codefresh offline-logs offload-to-file --uri "mongodb://192.168.99.100:27017" --db logs --collections archive-logs --cod "2021-07-08" --chdur 3 --path "./" '),
102123
handler: async (argv) => {
103124
const {
104125
uri,

‎lib/interface/cli/commands/offline-logs/utils.js

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,43 @@
1-
const moment = require('moment');
1+
const readline = require("readline");
2+
const moment = require("moment");
23

3-
const { join } = require('path');
4-
const fs = require('fs');
4+
const { join } = require("path");
5+
const fs = require("fs");
6+
7+
const defaultCollections = ["logs", "metadata"];
58

69
// converts date to objectId for filter purposes
710
const objectIdFromDate = function (date) {
8-
return Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
11+
return Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
912
};
1013

11-
const writeLogsToFile = function(upperBound, lowerBound, collection, logsToArchive, path) {
12-
const date = moment(upperBound).subtract(1, "days")
13-
const fileDateRange = `${moment(lowerBound).format('YYYY-MM-DD')}-${date.format('YYYY-MM-DD')}`
14+
const writeLogsToFile = function (
15+
upperBound,
16+
lowerBound,
17+
collection,
18+
logsToArchive,
19+
path
20+
) {
21+
const date = moment(upperBound).subtract(1, "days");
22+
const fileDateRange = `${moment(lowerBound).format(
23+
"YYYY-MM-DD"
24+
)}-${date.format("YYYY-MM-DD")}`;
1425
const fileName = `${fileDateRange}-${collection}.json`;
1526
const absPath = join(path, fileName);
1627
const data = JSON.stringify(logsToArchive);
1728
fs.writeFileSync(absPath, data);
29+
console.info(`logs from collection '${collection}' were offloaded to '${absPath}'`)
30+
};
31+
32+
const checkRemnant = function (lowerBound, cutoffDateObj) {
33+
return Math.ceil(cutoffDateObj.diff(lowerBound, "days"));
1834
};
1935

20-
const checkRemnant = function(lowerBound, cutoffDateObj) {
21-
return Math.ceil(cutoffDateObj.diff(lowerBound , "days" ));
22-
}
2336

24-
module.exports = {objectIdFromDate, writeLogsToFile, checkRemnant};
37+
38+
module.exports = {
39+
objectIdFromDate,
40+
writeLogsToFile,
41+
checkRemnant,
42+
defaultCollections,
43+
};

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "codefresh",
3-
"version": "0.76.0",
3+
"version": "0.77.0",
44
"description": "Codefresh command line utility",
55
"main": "index.js",
66
"preferGlobal": true,

0 commit comments

Comments
 (0)