Implented needs tag

firecow · firecow · commit 9953a15c37ad · 2020-02-22T21:01:46.000+01:00
diff --git a/src/index.ts b/src/index.ts
@@ -36,48 +36,73 @@ if (firstArg === "manual") {
 const parser = new Parser(cwd);
 
 const runJobs = async () => {
-    const stages = parser.getStages();
-    for (const [stageName, stage] of stages) {
-        const promises: Array<Promise<any>> = [];
-        const jobs = stage.getJobs();
-
-        if (stage.getJobs().length === 0) {
-            console.log(`=> ${c.yellow(`${stageName}`)} has no jobs`);
-            console.log("");
-            continue;
+    const jobs = Array.from(parser.getJobs().values());
+    const stages = Array.from(parser.getStages().values());
+
+    let stage = stages.shift();
+    while (stage !== undefined) {
+        const jobsInStage = stage.getJobs();
+        const stageName = stage.name;
+
+        if (!stage.isRunning()) {
+            console.log(`=> ${c.yellow(`${stageName}`)} <=`);
+            if (jobsInStage.length === 0 && !stage.isRunning()) {
+                console.log(`=> ${c.yellow(`${stageName}`)} has no jobs`);
+            }
         }
 
-        const jobNames = `${jobs.map((j) => j.name).join(" ")}`;
-        console.log(`=> ${c.yellow(`${stageName}`)} > ${c.blueBright(`${jobNames}`)} ${c.magentaBright("starting")}...`);
-        for (const job of jobs) {
-            if (job.isManual() && !manualArgs.includes(job.name)) {
+        for (const job of jobsInStage) {
+            if (job.isManual() && !manualArgs.includes(job.name) && !job.isFinished()) {
                 console.log(`${job.getJobNameString()} skipped. when:manual`);
+                job.setFinished(true);
                 continue;
             }
 
-            if (job.isNever()) {
+            if (job.isNever() && !job.isFinished()) {
                 console.log(`${job.getJobNameString()} skipped. when:never`);
+                job.setFinished(true);
                 continue;
             }
 
-            const jobPromise = job.start();
-            promises.push(jobPromise);
+            if (!job.isRunning() && !job.isFinished()) {
+                /* tslint:disable */
+                // noinspection ES6MissingAwait
+                job.start();
+                /* tslint:enabled */
+            }
         }
 
-        try {
-            await Promise.all(promises);
-            console.log("");
-        } catch (e) {
-            if (e !== "") {
-                console.error(e);
+        // Find jobs that can be started, because their needed jobs have finished
+        for (const job of jobs) {
+            if (job.isRunning() || job.isFinished() || job.needs.length === 0) {
+                continue;
+            }
+
+            const finishedJobNames = jobs.filter((e) => e.isFinished()).map((j) => j.name);
+            const needsConditionMet = job.needs.every((v) => (finishedJobNames.indexOf(v) >= 0));
+            if (needsConditionMet) {
+                /* tslint:disable */
+                // noinspection ES6MissingAwait
+                job.start();
+                /* tslint:enabled */
             }
-            process.exit(1);
+        }
+
+        await new Promise((r) => setTimeout(r, 50));
+
+        if (stage.isFinished()) {
+            if (!stage.isSuccess()) {
+                process.exit(2);
+            }
+            console.log("");
+            stage = stages.shift();
         }
     }
+
 };
 
 const runExecJobs = async () => {
-    const promises: Array<Promise<any>> = [];
+    const jobs = [];
     for (let i = 1; i < argv._.length; i += 1) {
         const jobName = argv._[i];
         const job = parser.getJobs().get(argv._[i]);
@@ -86,24 +111,27 @@ const runExecJobs = async () => {
             process.exit(1);
         }
 
-        const jobPromise = job.start();
-        promises.push(jobPromise);
+        jobs.push(job);
+
+        /* tslint:disable */
+        // noinspection ES6MissingAwait
+        job.start();
+        /* tslint:enabled */
     }
 
-    try {
-        await Promise.all(promises);
-        console.log("");
-    } catch (e) {
-        if (e !== "") {
-            console.error(e);
-        }
-        process.exit(1);
+    while (jobs.filter((j) => j.isRunning()).length > 0) {
+        await new Promise((r) => setTimeout(r, 50));
+    }
+
+    if (jobs.filter((j) => j.isSuccess()).length !== jobs.length) {
+        process.exit(2);
     }
 };
 
 process.on("uncaughtException", (err) => {
     // Handle the error safely
     console.log(err);
+    process.exit(5);
 });
 
 // Ensure gitlab-ci-local working directory and assets.
@@ -114,15 +142,16 @@ if (!pipelinesState) {
     pipelinesState = {};
 }
 pipelinesState.pipelineId = pipelinesState.pipelineId !== undefined ? Number(pipelinesState.pipelineId) : 0;
-console.log(firstArg);
 if (["pipeline", "manual"].includes(firstArg)) {
     pipelinesState.pipelineId += 1;
 }
 fs.writeFileSync(pipelinesStatePath, yaml.safeDump(pipelinesState), {});
 process.env.CI_PIPELINE_ID = pipelinesState.pipelineId;
 
 if (["pipeline", "manual"].includes(firstArg)) {
-    runJobs().catch();
+    // noinspection JSIgnoredPromiseFromCall
+    runJobs();
 } else if (firstArg === "exec") {
-    runExecJobs().catch();
+    // noinspection JSIgnoredPromiseFromCall
+    runExecJobs();
 }
diff --git a/src/job.ts b/src/job.ts
@@ -2,7 +2,6 @@ import * as c from "ansi-colors";
 import * as deepExtend from "deep-extend";
 import * as fs from "fs-extra";
 import * as glob from "glob";
-import * as path from "path";
 import * as prettyHrtime from "pretty-hrtime";
 import * as shelljs from "shelljs";
 
@@ -18,16 +17,19 @@ if (process.env.EXEPATH) {
 
 export class Job {
     public readonly name: string;
+    public readonly needs: string[];
     public readonly stage: string;
 
     private readonly afterScripts: string[] = [];
     private readonly allowFailure: boolean;
     private readonly beforeScripts: string[] = [];
     private readonly cwd: any;
+    private finished = false;
     private readonly globals: any;
     private readonly maxJobNameLength: number;
     private running = false;
     private readonly scripts: string[] = [];
+    private success = true;
     private readonly variables: { [key: string]: string };
     private readonly when: string;
 
@@ -57,20 +59,36 @@ export class Job {
         this.stage = jobData.stage || ".pre";
         this.scripts = [].concat(jobData.script || []);
 
+        const jobNameStr = this.getJobNameString();
+
+        if (this.scripts.length === 0) {
+            console.error(`${jobNameStr} ${c.red("must have script specified")}`);
+            process.exit(1);
+        }
+
         const ciDefault = globals.default || {};
         this.when = jobData.when || "on_success";
         this.beforeScripts = [].concat(jobData.before_script || ciDefault.before_script || globals.before_script || []);
         this.afterScripts = [].concat(jobData.after_script || ciDefault.after_script || globals.after_script || []);
         this.allowFailure = jobData.allow_failure || false;
         this.variables = jobData.variables || {};
+        if (this.needs && this.needs.length === 0) {
+            console.error(`${jobNameStr} ${c.red("'needs' cannot be empty array")}`);
+            process.exit(1);
+        }
+        this.needs = jobData.needs || [];
     }
 
     public getJobNameString() {
         return `${c.blueBright(`${this.name.padEnd(this.maxJobNameLength + 1)}`)}`;
     }
 
     public getOutputFilesPath() {
-        return `${this.cwd}/.gitlab-ci-local/output/${this.getEnvs().CI_PIPELINE_ID}/${this.name}.log`;
+        return `${this.cwd}/.gitlab-ci-local/output/${this.name}.log`;
+    }
+
+    public isFinished() {
+        return this.finished;
     }
 
     public isManual(): boolean {
@@ -81,38 +99,50 @@ export class Job {
         return this.when === "never";
     }
 
-    public async start(): Promise<void> {
-        const jobNameStr = this.getJobNameString();
+    public isRunning() {
+        return this.running;
+    }
 
-        this.running = true;
+    public isSuccess() {
+        return this.success;
+    }
 
-        if (this.scripts.length === 0) {
-            console.error(`${jobNameStr} ${c.red("must have script specified")}`);
-            process.exit(1);
-        }
+    public setFinished(finished: boolean) {
+        this.finished = finished;
+    }
+
+    public async start(): Promise<void> {
+        fs.ensureFileSync(this.getOutputFilesPath());
+        fs.truncateSync(this.getOutputFilesPath());
+        console.log(this.getStartingString());
+        this.running = true;
 
         const startTime = process.hrtime();
         const prescripts = this.beforeScripts.concat(this.scripts);
         const prescriptsExitCode = await this.exec(prescripts.join(" && "));
-
         if (this.afterScripts.length === 0 && prescriptsExitCode > 0 && !this.allowFailure) {
-            throw this.getExitedString(prescriptsExitCode, false);
+            console.error(this.getExitedString(startTime, prescriptsExitCode, false));
+            this.running = false;
+            this.finished = true;
+            this.success = false;
+
+            return;
         }
 
         if (this.afterScripts.length === 0 && prescriptsExitCode > 0 && this.allowFailure) {
-            console.error(this.getExitedString(prescriptsExitCode, true));
-            console.log(this.getFinishedString(startTime));
+            console.error(this.getExitedString(startTime, prescriptsExitCode, true));
             this.running = false;
+            this.finished = true;
 
             return;
         }
 
         if (prescriptsExitCode > 0 && this.allowFailure) {
-            console.error(this.getExitedString(prescriptsExitCode, true));
+            console.error(this.getExitedString(startTime, prescriptsExitCode, true));
         }
 
         if (prescriptsExitCode > 0 && !this.allowFailure) {
-            console.error(this.getExitedString(prescriptsExitCode, false));
+            console.error(this.getExitedString(startTime, prescriptsExitCode, false));
         }
 
         let afterScriptsCode = 0;
@@ -121,15 +151,17 @@ export class Job {
         }
 
         if (afterScriptsCode > 0) {
-            console.error(this.getExitedString(prescriptsExitCode, true, " (after_script)"));
+            console.error(this.getExitedString(startTime, prescriptsExitCode, true, " (after_script)"));
         }
 
         if (prescriptsExitCode > 0 && !this.allowFailure) {
-            throw "";
+            this.success = false;
         }
 
         console.log(this.getFinishedString(startTime));
+
         this.running = false;
+        this.finished = true;
 
         return;
     }
@@ -139,9 +171,6 @@ export class Job {
     }
 
     private async exec(script: string): Promise<number> {
-        fs.ensureFileSync(this.getOutputFilesPath());
-        fs.truncateSync(this.getOutputFilesPath());
-
         return new Promise<any>((resolve, reject) => {
             const jobNameStr = this.getJobNameString();
             const outputFilesPath = this.getOutputFilesPath();
@@ -176,14 +205,13 @@ export class Job {
         return {...this.globals.variables || {}, ...this.variables, ...process.env};
     }
 
-    private getExitedString(code: number, warning: boolean = false, prependString: string = "") {
-        const seeLogStr = `See ${path.resolve(this.getOutputFilesPath())}`;
-        const jobNameStr = this.getJobNameString();
+    private getExitedString(startTime: [number, number], code: number, warning: boolean = false, prependString: string = "") {
+        const finishedStr = this.getFinishedString(startTime);
         if (warning) {
-            return `${jobNameStr} ${c.yellowBright(`warning with code ${code}`)} ${prependString} ${seeLogStr}`;
+            return `${finishedStr} ${c.yellowBright(`warning with code ${code}`)} ${prependString}`;
         }
 
-        return `${jobNameStr} ${c.red(`exited with code ${code}`)} ${prependString} ${seeLogStr}`;
+        return `${finishedStr} ${c.red(`exited with code ${code}`)} ${prependString}`;
     }
 
     private getFinishedString(startTime: [number, number]) {
@@ -193,4 +221,10 @@ export class Job {
 
         return `${jobNameStr} ${c.magentaBright("finished")} in ${c.magenta(`${timeStr}`)}`;
     }
+
+    private getStartingString() {
+        const jobNameStr = this.getJobNameString();
+
+        return `${jobNameStr} ${c.magentaBright("starting")}...`;
+    }
 }
diff --git a/src/stage.ts b/src/stage.ts
@@ -16,4 +16,16 @@ export class Stage {
     public getJobs(): ReadonlyArray<Job> {
         return this.jobs;
     }
+
+    public isFinished(): boolean {
+        return this.jobs.filter((j) => j.isFinished()).length === this.jobs.length;
+    }
+
+    public isRunning(): boolean {
+        return this.jobs.filter((j) => j.isRunning()).length > 0;
+    }
+
+    public isSuccess(): boolean {
+        return this.jobs.filter((j) => j.isSuccess()).length === this.jobs.length;
+    }
 }
diff --git a/tests/test-yml-spec/.ci-include.yml b/tests/test-yml-spec/.ci-include.yml
@@ -23,3 +23,4 @@ new_job_from_include:
   stage: startup
   script:
     - echo "I get run, because i'm not specified anywhere else. This is the CI_PIPELINE_ID ${CI_PIPELINE_ID}"
+    - sleep 4
diff --git a/tests/test-yml-spec/.gitlab-ci.yml b/tests/test-yml-spec/.gitlab-ci.yml
@@ -31,7 +31,7 @@ echo_project_name:
   <<: *tags
   stage: startup
   script:
-    - sleep 1
+    - sleep 2
     - echo "Project Name is '${PROJECT_NAME}'"
 
 project_context:
@@ -56,7 +56,7 @@ var_in_another_file:
 after_script_test:
   <<: *tags
   stage: last
-  needs: [var_in_another_file]
+  needs: [var_in_another_file, echo_project_name]
   script:
     - echo "Throwing an error"
     - exit 1
