Make changes to support new features in v1.25

ci/compose.sh

@@ -21,5 +21,5 @@ function compose_down_all {
 }
 
 function all_weaviate_ports {
-  echo "8079 8080 8081 8082 8083 8085 8086 8087 8088"
+  echo "8079 8080 8081 8082 8083 8085 8086 8087 8088 8089 8090"
 }

ci/docker-compose-backup.yml

@@ -0,0 +1,19 @@
+---
+version: '3.4'
+services:
+  weaviate-backup:
+    image: semitechnologies/weaviate:${WEAVIATE_VERSION}
+    restart: on-failure:0
+    ports:
+      - 8090:8080
+      - 50061:50051
+    environment:
+      QUERY_DEFAULTS_LIMIT: 20
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+      PERSISTENCE_DATA_PATH: "./weaviate-data"
+      BACKUP_FILESYSTEM_PATH: "/tmp/backups"
+      ENABLE_MODULES: backup-filesystem
+      CLUSTER_GOSSIP_BIND_PORT: "7100"
+      CLUSTER_DATA_BIND_PORT: "7101"
+      DISABLE_TELEMETRY: 'true'
+...

src/backup/index.ts

@@ -6,6 +6,7 @@ import Connection from '../connection/index.js';
 
 export type Backend = 'filesystem' | 's3' | 'gcs' | 'azure';
 export type BackupStatus = 'STARTED' | 'TRANSFERRING' | 'TRANSFERRED' | 'SUCCESS' | 'FAILED';
+export type BackupCompressionLevel = 'DefaultCompression' | 'BestSpeed' | 'BestCompression';
 
 export interface Backup {
   creator: () => BackupCreator;

src/collections/backup/client.ts

@@ -1,5 +1,6 @@
 import {
   Backend,
+  BackupCompressionLevel,
   BackupCreateStatusGetter,
   BackupCreator,
   BackupRestoreStatusGetter,
@@ -11,11 +12,28 @@ import { WeaviateBackupFailed, WeaviateDeserializationError } from '../../errors
 import {
   BackupCreateResponse,
   BackupCreateStatusResponse,
+  BackupRestoreResponse,
   BackupRestoreStatusResponse,
 } from '../../openapi/types.js';
 
+/** Configuration options available when creating a backup */
+export type BackupConfigCreate = {
+  /** The size of the chunks to use for the backup. */
+  chunkSize?: number;
+  /** The standard of compression to use for the backup. */
+  compressionLevel?: BackupCompressionLevel;
+  /** The percentage of CPU to use for the backup creation job. */
+  cpuPercentage?: number;
+};
+
+/** Configuration options available when restoring a backup */
+export type BackupConfigRestore = {
+  /** The percentage of CPU to use for the backuop restoration job. */
+  cpuPercentage?: number;
+};
+
 /** The arguments required to create and restore backups. */
-export interface BackupArgs {
+export type BackupArgs<C extends BackupConfigCreate | BackupConfigRestore> = {
   /** The ID of the backup. */
   backupId: string;
   /** The backend to use for the backup. */
@@ -26,21 +44,16 @@ export interface BackupArgs {
   excludeCollections?: string[];
   /** Whether to wait for the backup to complete. */
   waitForCompletion?: boolean;
-}
+  /** The configuration options for the backup. */
+  config?: C;
+};
 
 /** The arguments required to get the status of a backup. */
-export interface BackupStatusArgs {
+export type BackupStatusArgs = {
   /** The ID of the backup. */
   backupId: string;
   /** The backend to use for the backup. */
   backend: Backend;
-}
-
-/** The response from a backup creation request. */
-export type BackupReturn = {
-  collections: string[];
-  status: BackupStatus;
-  path: string;
 };
 
 export const backup = (connection: Connection) => {
@@ -57,7 +70,7 @@ export const backup = (connection: Connection) => {
       .do();
   };
   return {
-    create: async (args: BackupArgs): Promise<BackupCreateResponse> => {
+    create: async (args: BackupArgs<BackupConfigCreate>): Promise<BackupCreateResponse> => {
       let builder = new BackupCreator(connection, new BackupCreateStatusGetter(connection))
         .withBackupId(args.backupId)
         .withBackend(args.backend);
@@ -67,27 +80,42 @@ export const backup = (connection: Connection) => {
       if (args.excludeCollections) {
         builder = builder.withExcludeClassNames(...args.excludeCollections);
       }
-      const res = builder.do();
+      if (args.config) {
+        builder = builder.withConfig({
+          ChunkSize: args.config.chunkSize,
+          CompressionLevel: args.config.compressionLevel,
+          CPUPercentage: args.config.cpuPercentage,
+        });
+      }
+      let res: BackupCreateResponse;
+      try {
+        res = await builder.do();
+      } catch (err) {
+        throw new Error(`Backup creation failed: ${err}`);
+      }
+      if (res.status === 'FAILED') {
+        throw new Error(`Backup creation failed: ${res.error}`);
+      }
+      let status: BackupCreateStatusResponse | undefined;
       if (args.waitForCompletion) {
         let wait = true;
         while (wait) {
           const res = await getCreateStatus(args); // eslint-disable-line no-await-in-loop
           if (res.status === 'SUCCESS') {
             wait = false;
+            status = res;
           }
           if (res.status === 'FAILED') {
             throw new WeaviateBackupFailed(res.error ? res.error : '<unknown>', 'creation');
           }
           await new Promise((resolve) => setTimeout(resolve, 1000)); // eslint-disable-line no-await-in-loop
         }
       }
-      return res.then(() =>
-        new BackupCreateStatusGetter(connection).withBackupId(args.backupId).withBackend(args.backend).do()
-      );
+      return status ? { ...status, classes: res.classes } : res;
     },
     getCreateStatus: getCreateStatus,
     getRestoreStatus: getRestoreStatus,
-    restore: async (args: BackupArgs): Promise<BackupRestoreStatusResponse> => {
+    restore: async (args: BackupArgs<BackupConfigRestore>): Promise<BackupRestoreResponse> => {
       let builder = new BackupRestorer(connection, new BackupRestoreStatusGetter(connection))
         .withBackupId(args.backupId)
         .withBackend(args.backend);
@@ -97,23 +125,41 @@ export const backup = (connection: Connection) => {
       if (args.excludeCollections) {
         builder = builder.withExcludeClassNames(...args.excludeCollections);
       }
-      const res = builder.do();
+      if (args.config) {
+        builder = builder.withConfig({
+          CPUPercentage: args.config.cpuPercentage,
+        });
+      }
+      let res: BackupRestoreResponse;
+      try {
+        res = await builder.do();
+      } catch (err) {
+        throw new Error(`Backup restoration failed: ${err}`);
+      }
+      if (res.status === 'FAILED') {
+        throw new Error(`Backup restoration failed: ${res.error}`);
+      }
+      let status: BackupRestoreStatusResponse | undefined;
       if (args.waitForCompletion) {
         let wait = true;
         while (wait) {
           const res = await getRestoreStatus(args); // eslint-disable-line no-await-in-loop
           if (res.status === 'SUCCESS') {
             wait = false;
+            status = res;
           }
           if (res.status === 'FAILED') {
             throw new WeaviateBackupFailed(res.error ? res.error : '<unknown>', 'restoration');
           }
           await new Promise((resolve) => setTimeout(resolve, 1000)); // eslint-disable-line no-await-in-loop
         }
       }
-      return res.then(() =>
-        new BackupRestoreStatusGetter(connection).withBackupId(args.backupId).withBackend(args.backend).do()
-      );
+      return status
+        ? {
+            ...status,
+            classes: res.classes,
+          }
+        : res;
     },
   };
 };
@@ -125,26 +171,26 @@ export interface Backup {
    * @param {BackupArgs} args The arguments for the request.
    * @returns {Promise<BackupCreateResponse>} The response from Weaviate.
    */
-  create(args: BackupArgs): Promise<BackupCreateResponse>;
+  create(args: BackupArgs<BackupConfigCreate>): Promise<BackupCreateResponse>;
   /**
    * Get the status of a backup creation.
    *
    * @param {BackupStatusArgs} args The arguments for the request.
-   * @returns {Promise<BackupStatus>} The status of the backup creation.
+   * @returns {Promise<BackupCreateStatusResponse>} The status of the backup creation.
    */
   getCreateStatus(args: BackupStatusArgs): Promise<BackupCreateStatusResponse>;
   /**
    * Get the status of a backup restore.
    *
    * @param {BackupStatusArgs} args The arguments for the request.
-   * @returns {Promise<BackupStatus>} The status of the backup restore.
+   * @returns {Promise<BackupRestoreStatusResponse>} The status of the backup restore.
    */
   getRestoreStatus(args: BackupStatusArgs): Promise<BackupRestoreStatusResponse>;
   /**
    * Restore a backup of the database.
    *
    * @param {BackupArgs} args The arguments for the request.
-   * @returns {Promise<BackupRestoreStatusResponse>} The response from Weaviate.
+   * @returns {Promise<BackupRestoreResponse>} The response from Weaviate.
    */
-  restore(args: BackupArgs): Promise<BackupRestoreStatusResponse>;
+  restore(args: BackupArgs<BackupConfigRestore>): Promise<BackupRestoreResponse>;
 }

src/collections/backup/collection.ts

@@ -3,19 +3,20 @@ import Connection from '../../connection/index.js';
 import {
   BackupCreateResponse,
   BackupCreateStatusResponse,
+  BackupRestoreResponse,
   BackupRestoreStatusResponse,
 } from '../../openapi/types.js';
 import { BackupStatusArgs, backup } from './client.js';
 
 /** The arguments required to create and restore backups. */
-export interface BackupCollectionArgs {
+export type BackupCollectionArgs = {
   /** The ID of the backup. */
   backupId: string;
   /** The backend to use for the backup. */
   backend: Backend;
   /** The collections to include in the backup. */
   waitForCompletion?: boolean;
-}
+};
 
 export const backupCollection = (connection: Connection, name: string) => {
   const handler = backup(connection);
@@ -47,21 +48,21 @@ export interface BackupCollection {
    * Get the status of a backup.
    *
    * @param {BackupStatusArgs} args The arguments for the request.
-   * @returns {Promise<BackupStatus>} The status of the backup.
+   * @returns {Promise<BackupCreateStatusResponse>} The status of the backup.
    */
   getCreateStatus(args: BackupStatusArgs): Promise<BackupCreateStatusResponse>;
   /**
    * Get the status of a restore.
    *
    * @param {BackupStatusArgs} args The arguments for the request.
-   * @returns {Promise<BackupStatus>} The status of the restore.
+   * @returns {Promise<BackupRestoreStatusResponse>} The status of the restore.
    */
   getRestoreStatus(args: BackupStatusArgs): Promise<BackupRestoreStatusResponse>;
   /**
    * Restore a backup of this collection.
    *
    * @param {BackupArgs} args The arguments for the request.
-   * @returns {Promise<BackupRestoreStatusResponse>} The response from Weaviate.
+   * @returns {Promise<BackupRestoreResponse>} The response from Weaviate.
    */
-  restore(args: BackupCollectionArgs): Promise<BackupRestoreStatusResponse>;
+  restore(args: BackupCollectionArgs): Promise<BackupRestoreResponse>;
 }

src/collections/backup/integration.test.ts

@@ -0,0 +1,105 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+/* eslint-disable @typescript-eslint/no-non-null-asserted-optional-chain */
+/* eslint-disable no-await-in-loop */
+import { Backend } from '../../backup/index.js';
+import weaviate, { Collection, WeaviateClient } from '../../index.js';
+
+// These must run sequentially because Weaviate is not capable of running multiple backups at the same time
+describe('Integration testing of backups', () => {
+  const clientPromise = weaviate.connectToLocal({
+    port: 8090,
+    grpcPort: 50061,
+  });
+
+  const getCollection = (client: WeaviateClient) => client.collections.get('TestBackupCollection');
+
+  beforeAll(() =>
+    clientPromise.then((client) =>
+      Promise.all([
+        client.collections.create({ name: 'TestBackupClient' }).then((col) => col.data.insert()),
+        client.collections.create({ name: 'TestBackupCollection' }).then((col) => col.data.insert()),
+      ])
+    )
+  );
+
+  afterAll(() => clientPromise.then((client) => client.collections.deleteAll()));
+
+  const testClientWaitForCompletion = async (client: WeaviateClient) => {
+    const res = await client.backup.create({
+      backupId: `test-backup-${randomBackupId()}`,
+      backend: 'filesystem',
+      waitForCompletion: true,
+    });
+    expect(res.status).toBe('SUCCESS');
+    return client;
+  };
+
+  const testClientNoWaitForCompletion = async (client: WeaviateClient) => {
+    const res = await client.backup.create({
+      backupId: `test-backup-${randomBackupId()}`,
+      backend: 'filesystem',
+    });
+    expect(res.status).toBe('STARTED');
+    const status = await client.backup.getCreateStatus({
+      backupId: res.id as string,
+      backend: res.backend as 'filesystem',
+    });
+    expect(status).not.toBe('SUCCESS'); // can be 'STARTED' or 'TRANSFERRING' depending on the speed of the test machine
+
+    // wait to complete so that other tests can run without colliding with Weaviate's lack of simultaneous backups
+    let wait = true;
+    while (wait) {
+      const { status, error } = await client.backup.getCreateStatus({
+        backupId: res.id as string,
+        backend: res.backend as Backend,
+      });
+      if (status === 'SUCCESS') {
+        wait = false;
+      }
+      if (status === 'FAILED') {
+        throw new Error(`Backup creation failed: ${error}`);
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+    }
+
+    return client;
+  };
+
+  const testCollectionWaitForCompletion = async (collection: Collection) => {
+    const res = await collection.backup.create({
+      backupId: `test-backup-${randomBackupId()}`,
+      backend: 'filesystem',
+      waitForCompletion: true,
+    });
+    expect(res.status).toBe('SUCCESS');
+    expect(res.classes).toEqual(['TestBackupCollection']);
+    return collection;
+  };
+
+  const testCollectionNoWaitForCompletion = async (collection: Collection) => {
+    const res = await collection.backup.create({
+      backupId: `test-backup-${randomBackupId()}`,
+      backend: 'filesystem',
+    });
+    expect(res.status).toBe('STARTED');
+    expect(res.classes).toEqual(['TestBackupCollection']);
+    const status = await collection.backup.getCreateStatus({
+      backupId: res.id as string,
+      backend: res.backend as 'filesystem',
+    });
+    expect(status).not.toBe('SUCCESS'); // can be 'STARTED' or 'TRANSFERRING' depending on the speed of the test machine
+    return collection;
+  };
+
+  it('run', () =>
+    clientPromise
+      .then(testClientWaitForCompletion)
+      .then(testClientNoWaitForCompletion)
+      .then(getCollection)
+      .then(testCollectionWaitForCompletion)
+      .then(testCollectionNoWaitForCompletion));
+});
+
+function randomBackupId() {
+  return 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+}