deepExtend only deep extends Object, it doesn't deep merge Arrays.

Author: firecow

package-lock.json

@@ -15,10 +15,9 @@
   "dependencies": {
     "ansi-colors": "4.x",
     "dateformat": "3.x",
-    "dot-prop": "5.x",
+    "deep-extend": "0.x",
     "glob": "7.x",
     "js-yaml": "3.x",
-    "lodash.merge": "4.x",
     "pretty-hrtime": "1.x",
     "shelljs": "0.x",
     "winston": "3.x",
@@ -32,8 +31,8 @@
   "author": "Mads Jon Nielsen <madsjon@gmail.com>",
   "license": "ISC",
   "devDependencies": {
+    "@types/deep-extend": "0.x",
     "@types/js-yaml": "3.x",
-    "@types/lodash.merge": "4.x",
     "@types/pretty-hrtime": "1.x",
     "@types/shelljs": "0.x",
     "@types/yaml": "1.x",

package.json

@@ -3,10 +3,6 @@ import * as winston from "winston";
 import yargs = require("yargs");
 import {Parser} from "./parser";
 
-export interface IKeyValue {
-     [key: string]: string | undefined;
-}
-
 const colorizer = winston.format.colorize();
 
 const logger: winston.Logger = winston.createLogger({

src/globals.ts

@@ -1,16 +1,13 @@
 import c = require("ansi-colors");
-import * as dotProp from "dot-prop";
 import prettyHrtime = require("pretty-hrtime");
 import * as shelljs from "shelljs";
-import {Globals} from "./globals";
-import {IKeyValue} from "./index";
 
 const shell = process.env.EXEPATH ? `${process.env.EXEPATH}/bash.exe` : "/bin/bash";
 
 export class Job {
 
     private static getScriptLikesFromData(jobData: any, keyname: string): string[] | null {
-        const sc = dotProp.get<string | string[] | undefined>(jobData, keyname);
+        const sc = jobData[keyname];
         if (sc) {
             let scripts: string[] = [];
             scripts = scripts.concat(sc);
@@ -23,9 +20,9 @@ export class Job {
     public readonly name: string;
 
     private readonly cwd: any;
+    private readonly globals: any;
 
-    private readonly globals: Globals;
-    private readonly variables: IKeyValue;
+    private readonly variables: {[key: string]: string};
 
     private readonly allowFailure: boolean;
 
@@ -35,17 +32,18 @@ export class Job {
 
     private running: boolean = false;
 
-    constructor(jobData: any, name: string, cwd: any, globals: Globals) {
+    constructor(jobData: any, name: string, cwd: any, globals: any) {
         this.name = name;
         this.cwd = cwd;
         this.globals = globals;
 
-        this.stage = dotProp.get<string>(jobData, "stage") || ".pre";
-        this.scripts = Job.getScriptLikesFromData(jobData, "script") || [];
-        this.beforeScripts = Job.getScriptLikesFromData(jobData, "before_script") || globals.beforeScripts || [];
-        this.afterScripts = Job.getScriptLikesFromData(jobData, "after_script") || globals.afterScripts || [];
-        this.allowFailure = dotProp.get<boolean>(jobData, "allow_failure") || false;
-        this.variables = dotProp.get<IKeyValue>(jobData, "variables") || {};
+        this.stage = jobData.stage || ".pre";
+        this.scripts = [].concat(jobData.script || []);
+
+        this.beforeScripts = [].concat(jobData.before_script || globals.before_script || []);
+        this.afterScripts = [].concat(jobData.after_script || globals.after_script || []);
+        this.allowFailure = jobData.allow_failure || false;
+        this.variables = jobData.variables || {};
     }
 
     public async start(): Promise<void> {
@@ -112,8 +110,8 @@ export class Job {
         return `${c.blueBright(`${this.name}`)} ${mistakeStr}`;
     }
 
-    private getEnvs(): IKeyValue {
-        return {...this.globals.variables, ...this.variables, ...process.env};
+    private getEnvs(): {[key: string]: string} {
+        return {...this.globals.variables || {}, ...this.variables, ...process.env};
     }
 
     private async exec(script: string): Promise<number> {

src/index.ts

@@ -1,10 +1,8 @@
 import c = require("ansi-colors");
-import * as dotProp from "dot-prop";
+import deepExtend  = require("deep-extend");
 import fs = require("fs");
 import yaml = require("js-yaml");
-import merge = require("lodash.merge");
 import * as winston from "winston";
-import {IKeyValue} from "./index";
 import {Job} from "./job";
 import {Stage} from "./stage";
 
@@ -19,34 +17,51 @@ export class Parser {
     private readonly stages: Map<string, Stage> = new Map();
 
     constructor(cwd: any, logger: winston.Logger) {
-        // Parse .gitlab-ci.yml
+        // Fail if .gitlab-ci.yml missing
         const gitlabCiYmlPath = `${cwd}/.gitlab-ci.yml`;
         if (!fs.existsSync(gitlabCiYmlPath)) {
             logger.error(`Could not find ${gitlabCiYmlPath}`);
             process.exit(1);
         }
-        const gitlabCiContent = fs.readFileSync(gitlabCiYmlPath, "utf8");
-        const gitlabCiData = yaml.safeLoad(gitlabCiContent);
-        const globalVariables = dotProp.get<IKeyValue>(gitlabCiData, "variables") || {};
 
-        // Parse .gitlab-local.yml
+        // Fail if .gitlab-ci.local.yml missing
         const gitlabCiLocalYmlPath = `${cwd}/.gitlab-ci.local.yml`;
         if (!fs.existsSync(gitlabCiLocalYmlPath)) {
             logger.error(`Could not find ${gitlabCiLocalYmlPath}`);
             process.exit(1);
         }
-        const gitlabCiLocalContent = fs.readFileSync(gitlabCiLocalYmlPath, "utf8");
-        const gitlabLocalData = yaml.safeLoad(gitlabCiLocalContent);
-        const globalLocalVariables = dotProp.get<IKeyValue>(gitlabLocalData, "variables") || {};
 
-        const gitlabData = merge(gitlabCiData, gitlabLocalData);
-        gitlabData.variables = {...globalVariables, ...globalLocalVariables};
+        const orderedVariables = [];
+        const orderedYml = [];
+
+        // Parse .gitlab-ci.yml
+        orderedYml.push(yaml.safeLoad(fs.readFileSync(gitlabCiYmlPath, "utf8")));
+        orderedVariables.push(orderedYml[orderedYml.length - 1].variables || {});
+
+        // Parse .gitlab-ci.local.yml
+        orderedYml.push(yaml.safeLoad(fs.readFileSync(gitlabCiLocalYmlPath, "utf8")));
+        orderedVariables.push(orderedYml[orderedYml.length - 1].variables || {});
+
+        // Parse yamls included by other ci files.
+        const includes = deepExtend.apply(this, orderedYml).include || [];
+        for (const value of includes) {
+            if (!value.local) {
+                continue;
+            }
+
+            orderedYml.unshift(yaml.safeLoad(fs.readFileSync(`${cwd}/${value.local}`, "utf8")));
+            orderedVariables.unshift(orderedYml[0].variables || {});
+        }
+
+        // Setup variables and "merged" yml
+        const gitlabData = deepExtend.apply(this, orderedYml);
 
-        for (const value of gitlabCiData.stages) {
+        // Generate stages
+        for (const value of gitlabData.stages) {
             this.stages.set(value, new Stage(value));
         }
 
-        // Generate all jobs specified in final gitlabData
+        // Generate jobs and put them into stages
         for (const [key, value] of Object.entries(gitlabData)) {
             if (this.illigalJobNames.includes(key) || key[0] === ".") {
                 continue;

src/job.ts

@@ -1,5 +1,6 @@
 variables:
-  INCLUDED_FROM_ANOTHER_FILE: "Its correct i'm a var from another file"
+  PROJECT_NAME: "Trying to override PROJECT_NAME, not going to happen, .gitlab-ci.yml have higher priority than me"
+  INCLUDED_FROM_ANOTHER_FILE: "Its correct i'm a var from another file" # 'New' var gets recognized
 
 echo_project_name:
   tags: [some_runner_tag]
@@ -20,4 +21,5 @@ local_only_job:
 new_job_from_include:
   tags: [some_runner_tag]
   stage: startup
-  - echo "I get run, because i'm not specified anywhere else"
+  script:
+    - echo "I get run, because i'm not specified anywhere else"

src/parser.ts

@@ -29,6 +29,12 @@ error_test:
     - echo "Throwing an error, but its ok, allow_failure equals true"
     - exit 1
 
+var_in_another_file:
+  <<: *tags
+  stage: startup
+  script:
+    - echo "${INCLUDED_FROM_ANOTHER_FILE}"
+
 after_script_test:
   <<: *tags
   stage: last