Skip to content

Commit 8424c67

Browse files
rotem-codefreshrotem-codefresh
authored
CR-12330 offline logs offload to file memory issue (#750)
* offload to file with cursor and buffer * wip * count deleted * some fixes + lint fixes * remove files * lint * bump * tests
1 parent a2ed0a8 commit 8424c67

File tree

4 files changed

+254
-194
lines changed

4 files changed

+254
-194
lines changed

‎lib/interface/cli/commands/offline-logs/offload-to-file.cmd.js

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* eslint-disable */
12
const { MongoClient, ObjectId } = require("mongodb");
23
const moment = require("moment");
34
const Command = require("../../Command");
@@ -34,7 +35,7 @@ const offloadToFile = async function (
3435
}
3536
const cutoffDateId = objectIdFromDate(cutoffDateObj.toDate());
3637

37-
const minLog = await findMinLog(collectionObj, cutoffDateId, collection);
38+
const minLog = await findMinLog(collectionObj, cutoffDateId);
3839

3940
if (!minLog) {
4041
console.info(
@@ -61,25 +62,22 @@ const offloadToFile = async function (
6162
const lowerBoundId = objectIdFromDate(lowerBound.toDate());
6263
const upperBoundId = objectIdFromDate(upperBound.toDate());
6364

64-
const logsToArchive = await collectionObj
65+
const logsCursor = await collectionObj
6566
.find({
6667
_id: {
6768
$gte: ObjectId(lowerBoundId),
6869
$lt: ObjectId(upperBoundId),
6970
},
70-
})
71-
.toArray();
71+
});
7272

73-
if (logsToArchive.length > 0) {
74-
writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path);
73+
await writeLogsToFile(upperBound, lowerBound, collection, logsCursor, path);
7574

76-
await collectionObj.deleteMany({
77-
_id: {
78-
$gte: ObjectId(lowerBoundId),
79-
$lt: ObjectId(upperBoundId),
80-
},
81-
});
82-
}
75+
await collectionObj.deleteMany({
76+
_id: {
77+
$gte: ObjectId(lowerBoundId),
78+
$lt: ObjectId(upperBoundId),
79+
},
80+
});
8381
}
8482
};
8583

@@ -152,7 +150,7 @@ const command = new Command({
152150
path
153151
);
154152
} catch (e) {
155-
console.log(e);
153+
console.error(e);
156154
failedCollections.push(collection);
157155
errors.push(error);
158156
}

‎lib/interface/cli/commands/offline-logs/offload-to-file.spec.js

Lines changed: 110 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,116 @@
1-
const moment = require('moment')
1+
/* eslint-disable no-unused-vars */
2+
const moment = require('moment');
23
const { writeLogsToFile, checkRemnant } = require('./utils');
3-
const fs = require('fs')
4+
const fs = require('fs');
45

56
describe('offload-to-file', () => {
6-
describe('writeLogsToFile', () => {
7-
it('writes the input data to file', () => {
8-
fs.writeFileSync = jest.fn((path, data) => {
9-
return {path, data};
10-
});
11-
const upperBound = '2021-08-10';
12-
const lowerBound = '2021-08-01';
13-
const collection = 'logs';
14-
const logsToArchive = [1, 2, 3, 4];
15-
const expectedStr = JSON.stringify(logsToArchive)
16-
const path = 'some/path';
17-
writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path);
18-
expect(fs.writeFileSync.mock.calls.length).toBe(1);
19-
expect(fs.writeFileSync.mock.calls[0][0]).toContain(path);
20-
expect(fs.writeFileSync.mock.calls[0][1]).toBe(expectedStr);
21-
});
22-
it('creates the correct file name', () => {
23-
fs.writeFileSync = jest.fn((path, data) => {
24-
return {path, data};
25-
});
26-
const upperBound = '2021-08-10';
27-
const lowerBound = '2021-08-01';
28-
const collection = 'logs';
29-
const logsToArchive = [1, 2, 3, 4];
30-
const expectedPath = 'some/path/2021-08-01-2021-08-09-logs.json'
31-
const path = 'some/path';
32-
writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path);
33-
expect(fs.writeFileSync.mock.calls.length).toBe(1);
34-
expect(fs.writeFileSync.mock.calls[0][0]).toBe(expectedPath);
35-
});
36-
it('checks leap year date calculation', () => {
37-
fs.writeFileSync = jest.fn((path, data) => {
38-
return {path, data};
39-
});
40-
const upperBound = '2020-03-01';
41-
const lowerBound = '2020-02-01';
42-
const collection = 'logs';
43-
const logsToArchive = [1, 2, 3, 4];
44-
const expectedPath = 'some/path/2020-02-01-2020-02-29-logs.json'
45-
const path = 'some/path';
46-
writeLogsToFile(upperBound, lowerBound, collection, logsToArchive, path);
47-
expect(fs.writeFileSync.mock.calls.length).toBe(1);
48-
expect(fs.writeFileSync.mock.calls[0][0]).toBe(expectedPath);
7+
describe('writeLogsToFile', () => {
8+
it('writes the input data to file', async () => {
9+
const upperBound = '2021-08-10';
10+
const lowerBound = '2021-08-01';
11+
const collection = 'logs';
12+
const pathMock = 'some/path';
13+
let writerResult = '';
14+
const expectedWrite = '[\n{"key1":"val"},\n{"key2":"val"},\n{"key3":"val"},\n{"key4":"val"}\n]';
15+
class LogsCursor {
16+
constructor() {
17+
this.logs = [{ key1: 'val' }, { key2: 'val' }, { key3: 'val' }, { key4: 'val' }];
18+
this.pointer = 0;
19+
this.next = jest.fn(() => {
20+
let currentDoc = null;
21+
if (this.pointer < this.logs.length) {
22+
currentDoc = this.logs[this.pointer];
23+
}
24+
this.pointer += 1;
25+
return currentDoc;
26+
});
27+
this.isClosed = jest.fn(() => this.pointer > this.logs.length);
28+
}
29+
}
30+
const writerMock = {
31+
write: jest.fn((text) => {
32+
writerResult = `${writerResult}${text}`;
33+
}),
34+
close: jest.fn(),
35+
};
36+
37+
fs.createWriteStream = jest.fn(path => writerMock);
38+
await writeLogsToFile(upperBound, lowerBound, collection, new LogsCursor(), pathMock);
39+
expect(fs.createWriteStream.mock.calls.length).toBe(1);
40+
expect(fs.createWriteStream.mock.calls[0][0]).toContain(pathMock);
41+
expect(writerMock.write.mock.calls.length).toBe(6);
42+
expect(writerResult).toBe(expectedWrite);
43+
expect(writerMock.close.mock.calls.length).toBe(1);
44+
});
45+
it('creates the correct file name', async () => {
46+
const upperBound = '2021-08-10';
47+
const lowerBound = '2021-08-01';
48+
const collection = 'logs';
49+
const expectedPath = 'some/path/2021-08-01-2021-08-09-logs.json';
50+
const somePath = 'some/path';
51+
class LogsCursor {
52+
constructor() {
53+
this.logs = [{ key1: 'val' }, { key2: 'val' }, { key3: 'val' }, { key4: 'val' }];
54+
this.pointer = 0;
55+
this.next = jest.fn(() => {
56+
let currentDoc = null;
57+
if (this.pointer < this.logs.length) {
58+
currentDoc = this.logs[this.pointer];
59+
}
60+
this.pointer += 1;
61+
return currentDoc;
62+
});
63+
this.isClosed = jest.fn(() => this.pointer > this.logs.length);
64+
}
65+
}
66+
const writerMock = {
67+
write: jest.fn(text => text),
68+
close: jest.fn(),
69+
};
70+
71+
fs.createWriteStream = jest.fn(path => writerMock);
72+
await writeLogsToFile(upperBound, lowerBound, collection, new LogsCursor(), somePath);
73+
expect(fs.createWriteStream.mock.calls.length).toBe(1);
74+
expect(fs.createWriteStream.mock.calls[0][0]).toBe(expectedPath);
75+
});
76+
it('checks leap year date calculation', async () => {
77+
const upperBound = '2020-03-01';
78+
const lowerBound = '2020-02-01';
79+
const collection = 'logs';
80+
const expectedPath = 'some/path/2020-02-01-2020-02-29-logs.json';
81+
const somePath = 'some/path';
82+
class LogsCursor {
83+
constructor() {
84+
this.logs = [{ key1: 'val' }, { key2: 'val' }, { key3: 'val' }, { key4: 'val' }];
85+
this.pointer = 0;
86+
this.next = jest.fn(() => {
87+
let currentDoc = null;
88+
if (this.pointer < this.logs.length) {
89+
currentDoc = this.logs[this.pointer];
90+
}
91+
this.pointer += 1;
92+
return currentDoc;
93+
});
94+
this.isClosed = jest.fn(() => this.pointer > this.logs.length);
95+
}
96+
}
97+
const writerMock = {
98+
write: jest.fn(text => text),
99+
close: jest.fn(),
100+
};
101+
102+
fs.createWriteStream = jest.fn(path => writerMock);
103+
await writeLogsToFile(upperBound, lowerBound, collection, new LogsCursor(), somePath);
104+
expect(fs.createWriteStream.mock.calls.length).toBe(1);
105+
expect(fs.createWriteStream.mock.calls[0][0]).toBe(expectedPath);
106+
});
49107
});
50-
});
51-
describe('checkRemnant', () => {
52-
it('checkes days remnant to make sure time range is not going off boundaries', () => {
53-
const lowerBound = '2020-08-01';
54-
const cutoffDateObj = moment('2020-08-01').add(1, 'days').startOf("day");
55-
const result = checkRemnant(lowerBound, cutoffDateObj);
56-
expect(result).toBe(1);
108+
describe('checkRemnant', () => {
109+
it('checkes days remnant to make sure time range is not going off boundaries', () => {
110+
const lowerBound = '2020-08-01';
111+
const cutoffDateObj = moment('2020-08-01').add(1, 'days').startOf('day');
112+
const result = checkRemnant(lowerBound, cutoffDateObj);
113+
expect(result).toBe(1);
114+
});
57115
});
58-
});
59116
});

0 commit comments

Comments
 (0)