Generic tasklogger (#13)

Author: itai-codefresh

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:8.1.4-alpine
+FROM node:11.10.0-alpine
 
 WORKDIR /root/cf-runtime
 

Dockerfile.aarch64

@@ -1,7 +1,7 @@
 FROM multiarch/alpine:aarch64-v3.6
 
 ENV NPM_CONFIG_LOGLEVEL info
-ENV NODE_VERSION 8.1.4
+ENV NODE_VERSION 11.10.0
 
 RUN addgroup -g 1000 node \
     && adduser -u 1000 -G node -s /bin/sh -D node \

codefresh.yml

@@ -5,24 +5,23 @@ steps:
 
   install_dependencies:
     title: 'Installing testing dependencies'
-    image: codefresh/node-tester-image:8.1.4
+    image: node:11.10.0
     commands:
         - yarn install --frozen-lockfile
 
-
   eslint:
     title: 'Running linting logic'
-    image: codefresh/node-tester-image:8.1.4
+    image: node:11.10.0
     commands:
         - yarn lint
         - yarn no-onlys
 
   unit_test_step:
-    image: codefresh/node-tester-image:8.1.4
+    image: node:11.10.0
     environment:
         - COVERALLS_TOKEN=${{COVERALLS_TOKEN}}
     commands:
-        - yarn test:ci
+        - yarn test
 
   build_step:
     title: "Building production image"

gulpfile.js

@@ -19,7 +19,7 @@ gulp.task('no.onlys', function (callback) {
 
 
 gulp.task('lint', ['clean'], function () {
-	return gulp.src(['**/*.js', '!**/node_modules/**', '!**/server/migration/**', '!coverage/**/*.js'])
+	return gulp.src(['**/*.js', '!**/node_modules/**', '!**/server/migration/**', '!coverage/**/*.js', '!test/logger.unit.spec.js'])
 		.pipe(jshint({lookup: true}))
 		.pipe(jshint.reporter('default'))
 		.pipe(jshint.reporter('fail'));

lib/ContainerLogger.js

@@ -11,19 +11,15 @@ class ContainerLogger extends EventEmitter {
     constructor({
         containerId,
         containerInterface,
-        firebaseLogger,
-        firebaseLastUpdate,
-        firebaseMetricsLogs,
+        stepLogger,
         logSizeLimit,
         isWorkflowLogSizeExceeded, // eslint-disable-line
         loggerStrategy
     }) {
         super();
         this.containerId               = containerId;
         this.containerInterface        = containerInterface;
-        this.firebaseLogger            = firebaseLogger;
-        this.firebaseLastUpdate        = firebaseLastUpdate;
-        this.firebaseMetricsLogs       = firebaseMetricsLogs;
+        this.stepLogger                = stepLogger;
         this.loggerStrategy            = loggerStrategy;
         this.tty                       = false;
         this.logSizeLimit              = logSizeLimit;
@@ -107,7 +103,7 @@ class ContainerLogger extends EventEmitter {
         stream.on('data', (chunk) => {
             const buf     = new Buffer(chunk);
             const message = buf.toString('utf8');
-            this._logMessageToFirebase(message, isError);
+            this._logMessage(message, isError);
         });
         logger.info(`Listening on stream 'data' event for container: ${this.containerId}`);
     }
@@ -120,7 +116,7 @@ class ContainerLogger extends EventEmitter {
                 if (payload === null) {
                     break;
                 }
-                this._logMessageToFirebase(new Buffer(payload).toString('utf8'), isError);
+                this._logMessage(new Buffer(payload).toString('utf8'), isError);
                 header = stream.read(8);
             }
         });
@@ -131,7 +127,7 @@ class ContainerLogger extends EventEmitter {
         return this.logSize > this.logSizeLimit;
     }
 
-    _logMessageToFirebase(message, isError) {
+    _logMessage(message, isError) {
         if (this.logSizeLimit && (this._stepLogSizeExceeded() || this.isWorkflowLogSizeExceeded()) && !isError) {
             if (!this.logExceededLimitsNotified) {
                 this.logExceededLimitsNotified = true;
@@ -145,12 +141,11 @@ class ContainerLogger extends EventEmitter {
             message = `\x1B[31m${message}\x1B[0m`;
         }
 
-        this.firebaseLogger.push(message);
-        this.firebaseLastUpdate.set(new Date().getTime());
+        this.stepLogger.write(message);
 
         if (this.logSizeLimit) {
             this.logSize += Buffer.byteLength(message);
-            this.firebaseMetricsLogs.child('total').set(this.logSize);
+            this.stepLogger.setLogSize(this.logSize);
         }
         this.emit('message.logged');
     }

lib/index.js

@@ -13,10 +13,8 @@ const Logger = require('./logger');
 
 const logger = new Logger({
     loggerId: process.env.LOGGER_ID,
-    firebaseAuthUrl: process.env.FIREBASE_AUTH_URL,
-    firebaseSecret: process.env.FIREBASE_SECRET,
+    taskLoggerConfig: JSON.parse(process.env.TASK_LOGGER_CONFIG),
     findExistingContainers: process.env.LISTEN_ON_EXISTING,
-    firebaseMetricsLogsUrl: process.env.FIREBASE_METRICS_LOGS_URL,
     logSizeLimit: process.env.LOG_SIZE_LIMIT ? (parseInt(process.env.LOG_SIZE_LIMIT) * 1000000) : undefined,
 });
 

lib/logger.js

@@ -2,7 +2,6 @@
 
 const fs                      = require('fs');
 const _                       = require('lodash');
-const Firebase                = require('firebase');
 const Docker                  = require('dockerode');
 const DockerEvents            = require('docker-events');
 const CFError                 = require('cf-errors');
@@ -11,26 +10,24 @@ const ContainerStatus         = require('./enums').ContainerStatus;
 const LoggerStrategy          = require('./enums').LoggerStrategy;
 const ContainerHandlingStatus = require('./enums').ContainerHandlingStatus;
 const ContainerLogger         = require('./ContainerLogger');
+const { TaskLogger }          = require('@codefresh-io/task-logger');
 
 class Logger {
 
     constructor({
         loggerId,
-        firebaseAuthUrl,
-        firebaseSecret,
+        taskLoggerConfig,
         findExistingContainers,
-        firebaseMetricsLogsUrl,
         logSizeLimit
     }) {
         this.state                  = { status: 'init' };
-        this.firebaseAuthUrl        = firebaseAuthUrl;
-        this.firebaseSecret         = firebaseSecret;
+        this.taskLoggerConfig       = taskLoggerConfig;
         this.loggerId               = loggerId;
         this.findExistingContainers = findExistingContainers === 'true';
-        this.firebaseMetricsLogsUrl = firebaseMetricsLogsUrl;
         this.logSizeLimit           = logSizeLimit;
         this.containerLoggers       = [];
         this.logSize                = 0;
+        this.taskLogger             = undefined;
 
         let dockerSockPath;
         if (fs.existsSync('/var/run/codefresh/docker.sock')) {
@@ -51,11 +48,8 @@ class Logger {
      * @returns {*}
      */
     validate() {
-        if (!this.firebaseAuthUrl) {
-            return this._error(new CFError('firebase auth url is missing'));
-        }
-        if (!this.firebaseSecret) {
-            return this._error(new CFError('firebase secret is missing'));
+        if (!this.taskLoggerConfig) {
+            return this._error(new CFError('taskLogger configuration is missing'));
         }
         if (!this.loggerId) {
             return this._error(new CFError('logger id is missing'));
@@ -67,38 +61,36 @@ class Logger {
      * will attach it self to all created containers that their ids in their labels equals loggerId
      * will attach it self to all existing containers if requested
      * the container label should be 'io.codefresh.loggerId'
-     * the path to write the containers logs will be passed through 'io.codefresh.firebaseUrl' label
-     * @param loggerId
-     * @param firebaseAuthUrl
-     * @param firebaseSecret
      */
     start() {
 
         logger.info(`Logging container created for logger id: ${this.loggerId}`);
 
-        const authRef = new Firebase(this.firebaseAuthUrl);
-        authRef.authWithCustomToken(this.firebaseSecret, (err) => {
-            if (err) {
-                this._error(new CFError({
-                    cause: err,
-                    message: `Failed to authenticate to firebase url ${this.firebaseAuthUrl}`
-                }));
-                return;
-            }
-            logger.info(`Authenticated to firebase url: ${this.firebaseAuthUrl}`);
-            this.firebaseMetricsLogs = new Firebase(this.firebaseMetricsLogsUrl);
-
-            this._listenForNewContainers();
+        TaskLogger(this.taskLoggerConfig.task, this.taskLoggerConfig.opts)
+            .then((taskLogger) => {
+                taskLogger.on('error', (err) => {
+                    logger.error(err.stack);
+                });
 
-            this.state.status = 'ready';
-            this._writeNewState();
+                this.taskLogger = taskLogger;
+                logger.info(`taskLogger successfully created`);
 
-            if (this.findExistingContainers) {
-                this._listenForExistingContainers();
-            }
+                this._listenForNewContainers();
 
-        });
+                this.state.status = 'ready';
+                this._writeNewState();
 
+                if (this.findExistingContainers) {
+                    this._listenForExistingContainers();
+                }
+            })
+            .catch((err) => {
+                this._error(new CFError({
+                    cause: err,
+                    message: `Failed to create taskLogger`
+                }));
+                return;
+            });
     }
 
     /**
@@ -149,19 +141,12 @@ class Logger {
      * @param docker
      * @param newContainer
      */
-    _handleContainer(container) {
+    async _handleContainer(container) { // jshint ignore:line
         const containerId                   = container.Id || container.id;
         const containerStatus               = container.Status || container.status;
         const receivedLoggerId              = _.get(container, 'Labels', _.get(container, 'Actor.Attributes'))['io.codefresh.logger.id'];
-        const receivedFirebaseLogsUrl       = _.get(container,
-            'Labels',
-            _.get(container, 'Actor.Attributes'))['io.codefresh.logger.firebase.logsUrl'];
-        const receivedFirebaseLastUpdateUrl = _.get(container,
-            'Labels',
-            _.get(container, 'Actor.Attributes'))['io.codefresh.logger.firebase.lastUpdateUrl'];
-        const receivedFirebaseMetricsLogsUrl = _.get(container,
-            'Labels',
-            _.get(container, 'Actor.Attributes'))['io.codefresh.logger.firebase.metricsLogs'];
+        const runCreationLogic              = _.get(container, 'Labels', _.get(container, 'Actor.Attributes'))['io.codefresh.runCreationLogic'];
+        const stepName                      = _.get(container, 'Labels', _.get(container, 'Actor.Attributes'))['io.codefresh.logger.stepName'];
         const receivedLogSizeLimit = _.get(container,
             'Labels',
             _.get(container, 'Actor.Attributes'))['io.codefresh.logger.logSizeLimit'];
@@ -188,23 +173,13 @@ class Logger {
             return;
         }
 
-        if (!receivedFirebaseLogsUrl) {
-            logger.error(`Container: ${containerId} does contain a firebaseUrl label`);
-            return;
-        }
-
-        if (!receivedFirebaseLastUpdateUrl) {
-            logger.error(`Container: ${containerId} does contain a loggerFirebaseLastUpdateUrl label`);
-            return;
-        }
-
-        if (!receivedFirebaseMetricsLogsUrl) {
-            logger.error(`Container: ${containerId} does contain a loggerFirebaseMetricsLogsUrl label`);
+        if (!stepName) {
+            logger.error(`Container: ${containerId} does not contain a stepName label`);
             return;
         }
 
         if (!loggerStrategy) {
-            logger.error(`Container: ${containerId} does contain a loggerStrategy label`);
+            logger.error(`Container: ${containerId} does not contain a loggerStrategy label`);
             return;
         }
 
@@ -222,37 +197,13 @@ class Logger {
 
         this.state[containerId] = { status: ContainerHandlingStatus.INITIALIZING };
         logger.info(`Handling container: ${containerId}, status: '${containerStatus}'`);
-        let firebaseLogger;
-        try {
-            firebaseLogger = new Firebase(receivedFirebaseLogsUrl);
-        } catch (err) {
-            const error = new CFError({
-                cause: err,
-                message: `Failed to create a new firebase logger ref`
-            });
-            logger.error(error.toString());
-            return;
-        }
-
-        let firebaseLastUpdate;
-        try {
-            firebaseLastUpdate = new Firebase(receivedFirebaseLastUpdateUrl);
-        } catch (err) {
-            const error = new CFError({
-                cause: err,
-                message: `Failed to create a new firebase lastUpdate ref`
-            });
-            logger.error(error.toString());
-            return;
-        }
-
-        let firebaseMetricsLogs;
+        const stepLogger = this.taskLogger.create(stepName, undefined, undefined, runCreationLogic);
         try {
-            firebaseMetricsLogs = new Firebase(receivedFirebaseMetricsLogsUrl);
+            await stepLogger.restore(); // jshint ignore:line
         } catch (err) {
             const error = new CFError({
                 cause: err,
-                message: `Failed to create a new firebase metricsLogs ref`
+                message: `Failed to restore step`
             });
             logger.error(error.toString());
             return;
@@ -264,9 +215,7 @@ class Logger {
         const containerLogger    = new ContainerLogger({
             containerId,
             containerInterface,
-            firebaseLogger,
-            firebaseLastUpdate,
-            firebaseMetricsLogs,
+            stepLogger,
             logSizeLimit,
             isWorkflowLogSizeExceeded: this.logLimitExceeded.bind(this),
             loggerStrategy
@@ -290,7 +239,7 @@ class Logger {
 
     _updateTotalLogSize() {
         this.logSize = this._getTotalLogSize();
-        this.firebaseMetricsLogs.child('total').set(this.logSize);
+        this.taskLogger.setLogSize(this.logSize);
     }
 
     /**

package.json

@@ -1,22 +1,22 @@
 {
   "name": "cf-container-logger",
   "version": "0.0.4",
-  "description": "codefresh container logger to firebase",
+  "description": "codefresh container logger",
   "keywords": [
     "cf-container-logger"
   ],
   "dependencies": {
+    "@codefresh-io/task-logger": "^0.0.1",
     "cf-errors": "^0.1.11",
     "cf-logs": "git+https://github.com/codefresh-io/cf-logs.git#ceba4f309e52a077747a0c6bf9c3ad02e762dc4b",
     "docker-events": "0.0.2",
     "dockerode": "^2.3.0",
-    "firebase": "git+https://github.com/codefresh-io/firebase.git#80b2ed883ff281cd67b53bd0f6a0bbd6f330fed5",
     "forever": "^0.15.3",
     "lodash": "^4.15.0",
     "q": "^1.4.1"
   },
   "devDependencies": {
-    "chai": "^3.0.0",
+    "chai": "^4.2.0",
     "gulp": "^3.9.1",
     "gulp-coveralls": "^0.1.4",
     "gulp-env": "^0.2.0",
@@ -29,16 +29,16 @@
     "proxyquire": "^1.7.4",
     "run-sequence": "^1.1.0",
     "shelljs": "^0.6.0",
-    "sinon": "^1.17.4",
-    "sinon-chai": "^2.8.0"
+    "sinon": "^7.2.4",
+    "sinon-chai": "^3.3.0"
   },
   "engines": {
-    "node": "8.1.4"
+    "node": ">8.8.0"
   },
   "scripts": {
     "lint": "gulp lint",
     "no-onlys": "gulp no.onlys",
-    "test": "gulp unit_test",
+    "test": "mocha './{,!(node_modules)/**/}*.spec.js'",
     "test:integration": "gulp integ_test",
     "test:ci": "gulp unit_test && gulp coveralls && gulp clean",
     "coveralls": "gulp coveralls",