Skip to content

Commit f6eeaa1

Browse files
authored
Fix broken cloud deploys by using depot ephemeral registry, skip the registry proxy (#1637)
1 parent eaf46ba commit f6eeaa1

14 files changed

+351
-195
lines changed

.changeset/pink-mice-battle.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"trigger.dev": patch
3+
"@trigger.dev/core": patch
4+
---
5+
6+
Fix broken cloud deploys by using depot ephemeral registry

apps/webapp/app/env.server.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,10 @@ const EnvironmentSchema = z.object({
147147
CONTAINER_REGISTRY_ORIGIN: z.string().optional(),
148148
CONTAINER_REGISTRY_USERNAME: z.string().optional(),
149149
CONTAINER_REGISTRY_PASSWORD: z.string().optional(),
150+
ENABLE_REGISTRY_PROXY: z.string().optional(),
150151
DEPLOY_REGISTRY_HOST: z.string().optional(),
152+
DEPLOY_REGISTRY_USERNAME: z.string().optional(),
153+
DEPLOY_REGISTRY_PASSWORD: z.string().optional(),
151154
DEPLOY_REGISTRY_NAMESPACE: z.string().default("trigger"),
152155
DEPLOY_TIMEOUT_MS: z.coerce
153156
.number()

apps/webapp/app/routes/api.v1.deployments.$deploymentId.start-indexing.ts renamed to apps/webapp/app/routes/api.v2.deployments.$deploymentId.finalize.ts

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import { ActionFunctionArgs, json } from "@remix-run/server-runtime";
2-
import { StartDeploymentIndexingRequestBody } from "@trigger.dev/core/v3";
2+
import { FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3";
33
import { z } from "zod";
44
import { authenticateApiRequest } from "~/services/apiAuth.server";
55
import { logger } from "~/services/logger.server";
6-
import { StartDeploymentIndexing } from "~/v3/services/startDeploymentIndexing.server";
6+
import { ServiceValidationError } from "~/v3/services/baseService.server";
7+
import { FinalizeDeploymentV2Service } from "~/v3/services/finalizeDeploymentV2";
78

89
const ParamsSchema = z.object({
910
deploymentId: z.string(),
@@ -34,21 +35,31 @@ export async function action({ request, params }: ActionFunctionArgs) {
3435
const { deploymentId } = parsedParams.data;
3536

3637
const rawBody = await request.json();
37-
const body = StartDeploymentIndexingRequestBody.safeParse(rawBody);
38+
const body = FinalizeDeploymentRequestBody.safeParse(rawBody);
3839

3940
if (!body.success) {
4041
return json({ error: "Invalid body", issues: body.error.issues }, { status: 400 });
4142
}
4243

43-
const service = new StartDeploymentIndexing();
44+
try {
45+
const service = new FinalizeDeploymentV2Service();
46+
await service.call(authenticatedEnv, deploymentId, body.data);
4447

45-
const deployment = await service.call(authenticatedEnv, deploymentId, body.data);
46-
47-
return json(
48-
{
49-
id: deployment.friendlyId,
50-
contentHash: deployment.contentHash,
51-
},
52-
{ status: 200 }
53-
);
48+
return json(
49+
{
50+
id: deploymentId,
51+
},
52+
{ status: 200 }
53+
);
54+
} catch (error) {
55+
if (error instanceof ServiceValidationError) {
56+
return json({ error: error.message }, { status: 400 });
57+
} else if (error instanceof Error) {
58+
logger.error("Error finalizing deployment", { error: error.message });
59+
return json({ error: `Internal server error: ${error.message}` }, { status: 500 });
60+
} else {
61+
logger.error("Error finalizing deployment", { error: String(error) });
62+
return json({ error: "Internal server error" }, { status: 500 });
63+
}
64+
}
5465
}

apps/webapp/app/v3/registryProxy.server.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,11 @@ function initializeProxy() {
435435
return;
436436
}
437437

438+
if (!env.ENABLE_REGISTRY_PROXY || env.ENABLE_REGISTRY_PROXY === "false") {
439+
logger.info("Registry proxy is disabled");
440+
return;
441+
}
442+
438443
return new RegistryProxy({
439444
origin: env.CONTAINER_REGISTRY_ORIGIN,
440445
auth: {

apps/webapp/app/v3/services/finalizeDeployment.server.ts

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ export class FinalizeDeploymentService extends BaseService {
1616
id: string,
1717
body: FinalizeDeploymentRequestBody
1818
) {
19-
const deployment = await this._prisma.workerDeployment.findUnique({
19+
const deployment = await this._prisma.workerDeployment.findFirst({
2020
where: {
2121
friendlyId: id,
2222
environmentId: authenticatedEnv.id,
@@ -48,6 +48,12 @@ export class FinalizeDeploymentService extends BaseService {
4848
throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
4949
}
5050

51+
let imageReference = body.imageReference;
52+
53+
if (registryProxy && body.selfHosted !== true && body.skipRegistryProxy !== true) {
54+
imageReference = registryProxy.rewriteImageReference(body.imageReference);
55+
}
56+
5157
// Link the deployment with the background worker
5258
const finalizedDeployment = await this._prisma.workerDeployment.update({
5359
where: {
@@ -56,10 +62,7 @@ export class FinalizeDeploymentService extends BaseService {
5662
data: {
5763
status: "DEPLOYED",
5864
deployedAt: new Date(),
59-
imageReference:
60-
registryProxy && body.selfHosted !== true
61-
? registryProxy.rewriteImageReference(body.imageReference)
62-
: body.imageReference,
65+
imageReference: imageReference,
6366
},
6467
});
6568

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
import { ExternalBuildData, FinalizeDeploymentRequestBody } from "@trigger.dev/core/v3/schemas";
2+
import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
3+
import { logger } from "~/services/logger.server";
4+
import { BaseService, ServiceValidationError } from "./baseService.server";
5+
import { join } from "node:path";
6+
import { tmpdir } from "node:os";
7+
import { mkdtemp, writeFile } from "node:fs/promises";
8+
import { env } from "~/env.server";
9+
import { depot as execDepot } from "@depot/cli";
10+
import { FinalizeDeploymentService } from "./finalizeDeployment.server";
11+
12+
export class FinalizeDeploymentV2Service extends BaseService {
13+
public async call(
14+
authenticatedEnv: AuthenticatedEnvironment,
15+
id: string,
16+
body: FinalizeDeploymentRequestBody
17+
) {
18+
// if it's self hosted, lets just use the v1 finalize deployment service
19+
if (body.selfHosted) {
20+
const finalizeService = new FinalizeDeploymentService();
21+
22+
return finalizeService.call(authenticatedEnv, id, body);
23+
}
24+
25+
const deployment = await this._prisma.workerDeployment.findFirst({
26+
where: {
27+
friendlyId: id,
28+
environmentId: authenticatedEnv.id,
29+
},
30+
include: {
31+
environment: true,
32+
worker: {
33+
include: {
34+
tasks: true,
35+
project: true,
36+
},
37+
},
38+
},
39+
});
40+
41+
if (!deployment) {
42+
logger.error("Worker deployment not found", { id });
43+
return;
44+
}
45+
46+
if (!deployment.worker) {
47+
logger.error("Worker deployment does not have a worker", { id });
48+
throw new ServiceValidationError("Worker deployment does not have a worker");
49+
}
50+
51+
if (deployment.status !== "DEPLOYING") {
52+
logger.error("Worker deployment is not in DEPLOYING status", { id });
53+
throw new ServiceValidationError("Worker deployment is not in DEPLOYING status");
54+
}
55+
56+
const externalBuildData = deployment.externalBuildData
57+
? ExternalBuildData.safeParse(deployment.externalBuildData)
58+
: undefined;
59+
60+
if (!externalBuildData) {
61+
throw new ServiceValidationError("External build data is missing");
62+
}
63+
64+
if (!externalBuildData.success) {
65+
throw new ServiceValidationError("External build data is invalid");
66+
}
67+
68+
if (
69+
!env.DEPLOY_REGISTRY_HOST ||
70+
!env.DEPLOY_REGISTRY_USERNAME ||
71+
!env.DEPLOY_REGISTRY_PASSWORD
72+
) {
73+
throw new ServiceValidationError("Missing deployment registry credentials");
74+
}
75+
76+
if (!env.DEPOT_TOKEN) {
77+
throw new ServiceValidationError("Missing depot token");
78+
}
79+
80+
const pushResult = await executePushToRegistry({
81+
depot: {
82+
buildId: externalBuildData.data.buildId,
83+
orgToken: env.DEPOT_TOKEN,
84+
projectId: externalBuildData.data.projectId,
85+
},
86+
registry: {
87+
host: env.DEPLOY_REGISTRY_HOST,
88+
namespace: env.DEPLOY_REGISTRY_NAMESPACE,
89+
username: env.DEPLOY_REGISTRY_USERNAME,
90+
password: env.DEPLOY_REGISTRY_PASSWORD,
91+
},
92+
deployment: {
93+
version: deployment.version,
94+
environmentSlug: deployment.environment.slug,
95+
projectExternalRef: deployment.worker.project.externalRef,
96+
},
97+
});
98+
99+
if (!pushResult.ok) {
100+
throw new ServiceValidationError(pushResult.error);
101+
}
102+
103+
const finalizeService = new FinalizeDeploymentService();
104+
105+
const finalizedDeployment = await finalizeService.call(authenticatedEnv, id, {
106+
imageReference: pushResult.image,
107+
skipRegistryProxy: true,
108+
});
109+
110+
return finalizedDeployment;
111+
}
112+
}
113+
114+
type ExecutePushToRegistryOptions = {
115+
depot: {
116+
buildId: string;
117+
orgToken: string;
118+
projectId: string;
119+
};
120+
registry: {
121+
host: string;
122+
namespace: string;
123+
username: string;
124+
password: string;
125+
};
126+
deployment: {
127+
version: string;
128+
environmentSlug: string;
129+
projectExternalRef: string;
130+
};
131+
};
132+
133+
type ExecutePushResult =
134+
| {
135+
ok: true;
136+
image: string;
137+
logs: string;
138+
}
139+
| {
140+
ok: false;
141+
error: string;
142+
logs: string;
143+
};
144+
145+
async function executePushToRegistry({
146+
depot,
147+
registry,
148+
deployment,
149+
}: ExecutePushToRegistryOptions): Promise<ExecutePushResult> {
150+
// Step 1: We need to "login" to the digital ocean registry
151+
const configDir = await ensureLoggedIntoDockerRegistry(registry.host, {
152+
username: registry.username,
153+
password: registry.password,
154+
});
155+
156+
const imageTag = `${registry.host}/${registry.namespace}/${deployment.projectExternalRef}:${deployment.version}.${deployment.environmentSlug}`;
157+
158+
// Step 2: We need to run the depot push command
159+
// DEPOT_TOKEN="<org token>" DEPOT_PROJECT_ID="<project id>" depot push <build id> -t registry.digitalocean.com/trigger-failover/proj_bzhdaqhlymtuhlrcgbqy:20250124.54.prod
160+
// Step 4: Build and push the image
161+
const childProcess = execDepot(["push", depot.buildId, "-t", imageTag, "--progress", "plain"], {
162+
env: {
163+
NODE_ENV: process.env.NODE_ENV,
164+
DEPOT_TOKEN: depot.orgToken,
165+
DEPOT_PROJECT_ID: depot.projectId,
166+
DEPOT_NO_SUMMARY_LINK: "1",
167+
DEPOT_NO_UPDATE_NOTIFIER: "1",
168+
DOCKER_CONFIG: configDir,
169+
},
170+
});
171+
172+
const errors: string[] = [];
173+
174+
try {
175+
const processCode = await new Promise<number | null>((res, rej) => {
176+
// For some reason everything is output on stderr, not stdout
177+
childProcess.stderr?.on("data", (data: Buffer) => {
178+
const text = data.toString();
179+
180+
// Emitted data chunks can contain multiple lines. Remove empty lines.
181+
const lines = text.split("\n").filter(Boolean);
182+
183+
errors.push(...lines);
184+
logger.debug(text, {
185+
imageTag,
186+
deployment,
187+
});
188+
});
189+
190+
childProcess.on("error", (e) => rej(e));
191+
childProcess.on("close", (code) => res(code));
192+
});
193+
194+
const logs = extractLogs(errors);
195+
196+
if (processCode !== 0) {
197+
return {
198+
ok: false as const,
199+
error: `Error pushing image`,
200+
logs,
201+
};
202+
}
203+
204+
return {
205+
ok: true as const,
206+
image: imageTag,
207+
logs,
208+
};
209+
} catch (e) {
210+
return {
211+
ok: false as const,
212+
error: e instanceof Error ? e.message : JSON.stringify(e),
213+
logs: extractLogs(errors),
214+
};
215+
}
216+
}
217+
218+
async function ensureLoggedIntoDockerRegistry(
219+
registryHost: string,
220+
auth: { username: string; password: string }
221+
) {
222+
const tmpDir = await createTempDir();
223+
// Read the current docker config
224+
const dockerConfigPath = join(tmpDir, "config.json");
225+
226+
await writeJSONFile(dockerConfigPath, {
227+
auths: {
228+
[registryHost]: {
229+
auth: Buffer.from(`${auth.username}:${auth.password}`).toString("base64"),
230+
},
231+
},
232+
});
233+
234+
logger.debug(`Writing docker config to ${dockerConfigPath}`);
235+
236+
return tmpDir;
237+
}
238+
239+
// Create a temporary directory within the OS's temp directory
240+
async function createTempDir(): Promise<string> {
241+
// Generate a unique temp directory path
242+
const tempDirPath: string = join(tmpdir(), "trigger-");
243+
244+
// Create the temp directory synchronously and return the path
245+
const directory = await mkdtemp(tempDirPath);
246+
247+
return directory;
248+
}
249+
250+
async function writeJSONFile(path: string, json: any, pretty = false) {
251+
await writeFile(path, JSON.stringify(json, undefined, pretty ? 2 : undefined), "utf8");
252+
}
253+
254+
function extractLogs(outputs: string[]) {
255+
// Remove empty lines
256+
const cleanedOutputs = outputs.map((line) => line.trim()).filter((line) => line !== "");
257+
258+
return cleanedOutputs.map((line) => line.trim()).join("\n");
259+
}

0 commit comments

Comments
 (0)