[Tests] E2E File upload/download

package.json

@@ -70,7 +70,7 @@
     "fastest-levenshtein": "^1.0.16",
     "file-type": "^18.6.0",
     "glob": "^10.3.10",
-    "got": "^13.0.0",
+    "got": "^14.3.0",
     "graphql": "^16.8.1",
     "graphql-parse-resolve-info": "^4.14.0",
     "graphql-scalars": "^1.22.4",

src/components/file/bucket/local-bucket.ts

@@ -8,6 +8,7 @@ import { Command } from '@smithy/smithy-client';
 import { pickBy } from 'lodash';
 import { DateTime, Duration } from 'luxon';
 import { URL } from 'node:url';
+import { firstValueFrom, Observable } from 'rxjs';
 import { Readable } from 'stream';
 import { assert } from 'ts-essentials';
 import {
@@ -19,7 +20,7 @@ import {
 } from './file-bucket';
 
 export interface LocalBucketOptions {
-  baseUrl: URL;
+  baseUrl: Observable<URL>;
 }
 
 export type FakeAwsFile = Required<Pick<GetObjectOutput, 'ContentType'>> &
@@ -96,7 +97,8 @@ export abstract class LocalBucket<
           .toMillis(),
       },
     });
-    const url = new URL(this.options.baseUrl.toString());
+    const baseUrl = await firstValueFrom(this.options.baseUrl);
+    const url = new URL(baseUrl.toString());
     url.searchParams.set('signed', signed);
     return url.toString();
   }

src/components/file/file.service.ts

@@ -144,7 +144,7 @@ export class FileService {
 
   async getUrl(node: FileNode, download: boolean) {
     const url = withAddedPath(
-      this.config.hostUrl,
+      this.config.hostUrl$.value,
       FileUrl.path,
       isFile(node) ? node.latestVersionId : node.id,
       encodeURIComponent(node.name),

src/components/file/files-bucket.factory.ts

@@ -1,6 +1,7 @@
 import { S3 } from '@aws-sdk/client-s3';
 import { FactoryProvider } from '@nestjs/common';
 import { resolve } from 'path';
+import { map } from 'rxjs/operators';
 import { withAddedPath } from '~/common/url.util';
 import { ConfigService } from '~/core';
 import {
@@ -21,7 +22,9 @@ export const FilesBucketFactory: FactoryProvider = {
   useFactory: (s3: S3, config: ConfigService) => {
     const { sources } = config.files;
 
-    const baseUrl = withAddedPath(config.hostUrl, LocalBucketController.path);
+    const baseUrl = config.hostUrl$.pipe(
+      map((hostUrl) => withAddedPath(hostUrl, LocalBucketController.path)),
+    );
 
     const files: FileFactory = ({ path }) =>
       new FilesystemBucket({

src/components/file/local-bucket.controller.ts

@@ -5,6 +5,7 @@ import {
   Put,
   Request,
   Response,
+  StreamableFile,
 } from '@nestjs/common';
 import { Request as IRequest, Response as IResponse } from 'express';
 import { DateTime } from 'luxon';
@@ -49,7 +50,10 @@ export class LocalBucketController {
   }
 
   @Get()
-  async download(@Request() req: IRequest, @Response() res: IResponse) {
+  async download(
+    @Request() req: IRequest,
+    @Response({ passthrough: true }) res: IResponse,
+  ) {
     const url = new URL(`https://localhost${req.url}`);
     const { Key, operation, ...rest } = await this.bucket.parseSignedUrl(url);
     if (operation !== 'GetObject') {
@@ -84,6 +88,6 @@ export class LocalBucketController {
       }
     }
 
-    out.Body.pipe(res);
+    return new StreamableFile(out.Body);
   }
 }

src/core/config/config.service.ts

@@ -10,6 +10,7 @@ import LRUCache from 'lru-cache';
 import { Duration, DurationLike } from 'luxon';
 import { nanoid } from 'nanoid';
 import { Config as Neo4JDriverConfig } from 'neo4j-driver';
+import { BehaviorSubject } from 'rxjs';
 import { keys as keysOf } from 'ts-transformer-keys';
 import { Class, Merge, ReadonlyDeep } from 'type-fest';
 import { ID } from '~/common';
@@ -35,9 +36,9 @@ export const makeConfig = (env: EnvironmentService) =>
     port = env.number('port').optional(3000);
     // The port where the app is being hosted. i.e. a docker bound port
     publicPort = env.number('public_port').optional(this.port);
-    hostUrl = env
-      .url('host_url')
-      .optional(`http://localhost:${this.publicPort}`);
+    hostUrl$ = new BehaviorSubject(
+      env.url('host_url').optional(`http://localhost:${this.publicPort}`),
+    );
 
     graphQL = {
       persistedQueries: {

src/core/tracing/tracing.module.ts

@@ -47,12 +47,15 @@ export class TracingModule implements OnModuleInit, NestModule {
       XRay.middleware.disableCentralizedSampling();
     }
 
-    XRay.SegmentUtils.setServiceData({
-      name: this.config.hostUrl.toString(),
-      version: (await this.version.version).toString(),
-      runtime: process.release?.name ?? 'unknown',
-      // eslint-disable-next-line @typescript-eslint/naming-convention
-      runtime_version: process.version,
+    const version = await this.version.version;
+    this.config.hostUrl$.subscribe((hostUrl) => {
+      XRay.SegmentUtils.setServiceData({
+        name: hostUrl.toString(),
+        version: version.toString(),
+        runtime: process.release?.name ?? 'unknown',
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        runtime_version: process.version,
+      });
     });
 
     if (this.config.xray.daemonAddress) {

src/core/tracing/xray.middleware.ts

@@ -36,7 +36,8 @@ export class XRayMiddleware implements NestMiddleware, NestInterceptor {
     root.addIncomingRequestData(reqData);
     // Use public DNS as url instead of specific IP
     // @ts-expect-error xray library types suck
-    root.http.request.url = this.config.hostUrl + req.originalUrl.slice(1);
+    root.http.request.url =
+      this.config.hostUrl$.value + req.originalUrl.slice(1);
 
     // Add to segment so interceptor can access without having to calculate again.
     Object.defineProperty(reqData, 'traceData', {

src/main.ts

@@ -23,13 +23,13 @@ async function bootstrap() {
   app.enableCors(config.cors as CorsOptions); // typecast to undo deep readonly
   app.use(cookieParser());
 
-  app.setGlobalPrefix(config.hostUrl.pathname.slice(1));
+  app.setGlobalPrefix(config.hostUrl$.value.pathname.slice(1));
 
   config.applyTimeouts(app.getHttpServer(), config.httpTimeouts);
 
   app.enableShutdownHooks();
   await app.listen(config.port, () => {
-    app.get(Logger).log(`Listening at ${config.hostUrl}graphql`);
+    app.get(Logger).log(`Listening at ${config.hostUrl$.value}graphql`);
   });
 }
 bootstrap().catch((err) => {

test/file.e2e-spec.ts

@@ -1,5 +1,5 @@
 import { faker } from '@faker-js/faker';
-import { bufferFromStream } from '@seedcompany/common';
+import got from 'got';
 import { startCase, times } from 'lodash';
 import {
   DateTime,
@@ -104,13 +104,18 @@ function resetNow() {
 }
 
 const expectEqualContent = async (
-  bucket: LocalBucket,
+  app: TestApp,
   url: string,
   expected: FakeFile,
 ) => {
-  const actualFile = await bucket.download(url);
-  const contents = await bufferFromStream(actualFile.Body);
-  expect(contents.toString()).toEqual(expected.content.toString());
+  const expectedContents = expected.content;
+  const actualContents = await got(url, {
+    headers: {
+      Authorization: `Bearer ${app.graphql.authToken}`,
+    },
+    enableUnixSockets: true,
+  }).buffer();
+  expect(expectedContents).toEqual(actualContents);
 };
 
 describe('File e2e', () => {
@@ -156,7 +161,7 @@ describe('File e2e', () => {
       expect(modifiedAt.diffNow().as('seconds')).toBeGreaterThan(-30);
       const createdAt = DateTime.fromISO(file.createdAt);
       expect(createdAt.diffNow().as('seconds')).toBeGreaterThan(-30);
-      await expectEqualContent(bucket, file.downloadUrl, fakeFile);
+      await expectEqualContent(app, file.url, fakeFile);
       expect(file.parents[0].id).toEqual(root.id);
     }
   });
@@ -177,7 +182,7 @@ describe('File e2e', () => {
     expect(version.createdBy.id).toEqual(me.id);
     const createdAt = DateTime.fromISO(version.createdAt);
     expect(createdAt.diffNow().as('seconds')).toBeGreaterThan(-30);
-    await expectEqualContent(bucket, file.downloadUrl, fakeFile);
+    await expectEqualContent(app, file.url, fakeFile);
     expect(version.parents[0].id).toEqual(file.id);
   });
 
@@ -216,7 +221,7 @@ describe('File e2e', () => {
     input: FakeFile,
   ) {
     expect(updated.id).toEqual(initial.id);
-    await expectEqualContent(bucket, updated.downloadUrl, input);
+    await expectEqualContent(app, updated.url, input);
     expect(updated.size).toEqual(input.size);
     expect(updated.mimeType).toEqual(input.mimeType);
     const createdAt = DateTime.fromISO(updated.createdAt);

test/utility/create-file.ts

@@ -1,7 +1,7 @@
 import { faker } from '@faker-js/faker';
-import { assert, MarkOptional } from 'ts-essentials';
+import got from 'got';
+import { MarkOptional } from 'ts-essentials';
 import { ID } from '~/common';
-import { FileBucket, LocalBucket } from '../../src/components/file/bucket';
 import {
   CreateFileVersionInput,
   FileListInput,
@@ -13,15 +13,18 @@ import { RawFile, RawFileNode, RawFileNodeChildren } from './fragments';
 import * as fragments from './fragments';
 import { gql } from './gql-tag';
 
-export const generateFakeFile = () => ({
-  name: faker.system.fileName(),
-  content: Buffer.from(
+export const generateFakeFile = () => {
+  const content = Buffer.from(
     faker.image.dataUri({ width: 200, height: 200 }).split(',')[1],
     'base64',
-  ),
-  size: faker.number.int(1_000_000),
-  mimeType: faker.helpers.arrayElement(mimeTypes).name,
-});
+  );
+  return {
+    name: faker.system.fileName(),
+    content: content,
+    size: content.length,
+    mimeType: faker.helpers.arrayElement(mimeTypes).name,
+  };
+};
 
 export type FakeFile = ReturnType<typeof generateFakeFile>;
 
@@ -49,18 +52,14 @@ export const uploadFileContents = async (
     ...generateFakeFile(),
     ...input,
   };
-  const {
-    content: Body,
-    mimeType: ContentType,
-    size: ContentLength,
-  } = completeInput;
-
-  const bucket = app.get(FileBucket);
-  assert(bucket instanceof LocalBucket);
-  await bucket.upload(url, {
-    Body,
-    ContentType,
-    ContentLength,
+  const { content, mimeType } = completeInput;
+
+  await got.put(url, {
+    headers: {
+      'Content-Type': mimeType,
+    },
+    body: content,
+    enableUnixSockets: true,
   });
 
   return completeInput;

test/utility/create-graphql-client.ts

@@ -7,6 +7,7 @@ import {
   print,
 } from 'graphql';
 import { Merge } from 'type-fest';
+import { ConfigService } from '~/core';
 // eslint-disable-next-line import/no-duplicates
 import { ErrorExpectations } from './expect-gql-error';
 // eslint-disable-next-line import/no-duplicates -- ensures runtime execution
@@ -29,6 +30,7 @@ export const createGraphqlClient = async (
 ): Promise<GraphQLTestClient> => {
   await app.listen(0);
   const url = await app.getUrl();
+  app.get(ConfigService).hostUrl$.next(new URL(url) as URL & string);
 
   let authToken = '';
 

test/utility/fragments.ts

@@ -394,12 +394,12 @@ export const fileNode = gql`
     ... on FileVersion {
       mimeType
       size
-      downloadUrl
+      url
     }
     ... on File {
       mimeType
       size
-      downloadUrl
+      url
       modifiedAt
       modifiedBy {
         ...user
@@ -421,7 +421,7 @@ export type RawBaseFileNode = RawNode<
 >;
 export type RawDirectory = RawBaseFileNode;
 export type RawFileVersion = RawBaseFileNode &
-  RawNode<FileVersion, keyof IFileNode, { downloadUrl: string }>;
+  RawNode<FileVersion, keyof IFileNode, { url: string }>;
 export type RawFile = RawFileVersion &
   RawNode<
     File,