diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b52b0192..e749da5c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -8,7 +8,7 @@ on: env: - WEAVIATE_VERSION: 1.24.1 + WEAVIATE_VERSION: 1.25.1 jobs: checks: @@ -27,15 +27,23 @@ jobs: tests: needs: checks runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + versions: [ + { node: "18.x", weaviate: $WEAVIATE_VERSION}, + { node: "20.x", weaviate: $WEAVIATE_VERSION}, + { node: "22.x", weaviate: $WEAVIATE_VERSION} + ] steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: '18.x' + node-version: ${{ matrix.versions.node }} - name: "Install dependencies" run: | npm install - ci/run_dependencies.sh ${{ env.WEAVIATE_VERSION }} + ci/run_dependencies.sh ${{ matrix.versions.weaviate }} - name: "Run tests with authentication tests" if: ${{ !github.event.pull_request.head.repo.fork }} env: @@ -52,7 +60,7 @@ jobs: npm test npm run build - name: "Stop Weaviate" - run: ci/stop_dependencies.sh ${{ env.WEAVIATE_VERSION }} + run: ci/stop_dependencies.sh ${{ matrix.versions.weaviate }} publish: needs: tests diff --git a/ci/docker-compose-cluster.yml b/ci/docker-compose-cluster.yml index 56d04ac3..2d22c48d 100644 --- a/ci/docker-compose-cluster.yml +++ b/ci/docker-compose-cluster.yml @@ -6,16 +6,22 @@ services: restart: on-failure:0 ports: - "8087:8080" + - "50058:50051" environment: - CONTEXTIONARY_URL: contextionary:9999 QUERY_DEFAULTS_LIMIT: 20 AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' PERSISTENCE_DATA_PATH: "./weaviate-node-1" - DEFAULT_VECTORIZER_MODULE: text2vec-contextionary - ENABLE_MODULES: text2vec-contextionary + CLUSTER_HOSTNAME: "node1" CLUSTER_GOSSIP_BIND_PORT: "7110" CLUSTER_DATA_BIND_PORT: "7111" + RAFT_PORT: '8300' + RAFT_INTERNAL_RPC_PORT: "8301" + RAFT_JOIN: "node1:8300,node2:8300,node3:8300" + RAFT_BOOTSTRAP_EXPECT: "3" DISABLE_TELEMETRY: 'true' + CONTEXTIONARY_URL: contextionary:9999 + DEFAULT_VECTORIZER_MODULE: text2vec-contextionary + ENABLE_MODULES: text2vec-contextionary weaviate-node-2: init: true @@ -29,19 +35,55 @@ services: image: semitechnologies/weaviate:${WEAVIATE_VERSION} ports: - 8088:8080 - - 6061:6060 + - "50059:50051" restart: on-failure:0 environment: - CONTEXTIONARY_URL: contextionary:9999 LOG_LEVEL: 'debug' QUERY_DEFAULTS_LIMIT: 20 AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' PERSISTENCE_DATA_PATH: './weaviate-node-2' + CLUSTER_HOSTNAME: 'node2' + CLUSTER_GOSSIP_BIND_PORT: '7110' + CLUSTER_DATA_BIND_PORT: '7111' + CLUSTER_JOIN: 'weaviate-node-1:7110' + RAFT_PORT: '8300' + RAFT_INTERNAL_RPC_PORT: "8301" + RAFT_JOIN: "node1:8300,node2:8300,node3:8300" + RAFT_BOOTSTRAP_EXPECT: "3" + DISABLE_TELEMETRY: 'true' + CONTEXTIONARY_URL: contextionary:9999 DEFAULT_VECTORIZER_MODULE: text2vec-contextionary ENABLE_MODULES: text2vec-contextionary - CLUSTER_HOSTNAME: 'node2' - CLUSTER_GOSSIP_BIND_PORT: '7112' - CLUSTER_DATA_BIND_PORT: '7113' + + weaviate-node-3: + init: true + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: semitechnologies/weaviate:${WEAVIATE_VERSION} + ports: + - 8089:8080 + - "50060:50051" + restart: on-failure:0 + environment: + LOG_LEVEL: 'debug' + QUERY_DEFAULTS_LIMIT: 20 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: './weaviate-node-2' + CLUSTER_HOSTNAME: 'node3' + CLUSTER_GOSSIP_BIND_PORT: '7110' + CLUSTER_DATA_BIND_PORT: '7111' CLUSTER_JOIN: 'weaviate-node-1:7110' + RAFT_PORT: '8300' + RAFT_INTERNAL_RPC_PORT: "8301" + RAFT_JOIN: "node1:8300,node2:8300,node3:8300" + RAFT_BOOTSTRAP_EXPECT: "3" DISABLE_TELEMETRY: 'true' -... + CONTEXTIONARY_URL: contextionary:9999 + DEFAULT_VECTORIZER_MODULE: text2vec-contextionary + ENABLE_MODULES: text2vec-contextionary +... \ No newline at end of file diff --git a/src/connection/httpClient.ts b/src/connection/httpClient.ts index 1c43934a..2a03666f 100644 --- a/src/connection/httpClient.ts +++ b/src/connection/httpClient.ts @@ -149,8 +149,8 @@ const makeCheckStatus = (expectResponseBody: boolean) => (res: Response) => { }; const handleHeadResponse = (expectResponseBody: boolean) => (res: Response) => { - if (res.status == 204 || res.status == 404) { - return res.status == 204; + if (res.status == 200 || res.status == 204 || res.status == 404) { + return res.status == 200 || res.status == 204; } return makeCheckStatus(expectResponseBody)(res); }; diff --git a/src/graphql/getter.test.ts b/src/graphql/getter.test.ts index 8d820cff..15a64f82 100644 --- a/src/graphql/getter.test.ts +++ b/src/graphql/getter.test.ts @@ -1289,6 +1289,21 @@ describe('bm25 valid searchers', () => { expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); }); + + test('query and groupby', () => { + const expectedQuery = `{Get{Person(bm25:{query:"accountant"},groupBy:{path:["employer"],groups:2,objectsPerGroup:3}){name}}}`; + + new Getter(mockClient) + .withClassName('Person') + .withFields('name') + .withBm25({ + query: 'accountant', + }) + .withGroupBy({ path: ['employer'], groups: 2, objectsPerGroup: 3 }) + .do(); + + expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); + }); }); describe('hybrid valid searchers', () => { @@ -1383,6 +1398,50 @@ describe('hybrid valid searchers', () => { expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); }); + + test('query and groupby', () => { + const expectedQuery = `{Get{Person(hybrid:{query:"accountant"},groupBy:{path:["employer"],groups:2,objectsPerGroup:3}){name}}}`; + + new Getter(mockClient) + .withClassName('Person') + .withFields('name') + .withHybrid({ + query: 'accountant', + }) + .withGroupBy({ path: ['employer'], groups: 2, objectsPerGroup: 3 }) + .do(); + + expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); + }); + + test('query and subsearches', () => { + const subQuery = `searches:[{nearVector:{vector:[1,2,3],certainty:0.8,targetVectors:["employer"]}},{nearText:{concepts:["accountant"],distance:0.3,moveTo:{concepts:["foo"],objects:[{id:"uuid"}],force:0.8}}}]`; + const expectedQuery = `{Get{Person(hybrid:{query:"accountant",${subQuery}}){name}}}`; + + new Getter(mockClient) + .withClassName('Person') + .withFields('name') + .withHybrid({ + query: 'accountant', + searches: [ + { nearVector: { certainty: 0.8, targetVectors: ['employer'], vector: [1, 2, 3] } }, + { + nearText: { + concepts: ['accountant'], + distance: 0.3, + moveTo: { + concepts: ['foo'], + objects: [{ id: 'uuid' }], + force: 0.8, + }, + }, + }, + ], + }) + .do(); + + expect(mockClient.query).toHaveBeenCalledWith(expectedQuery); + }); }); describe('generative search', () => { diff --git a/src/graphql/hybrid.ts b/src/graphql/hybrid.ts index a7ace9b4..0e021739 100644 --- a/src/graphql/hybrid.ts +++ b/src/graphql/hybrid.ts @@ -1,3 +1,5 @@ +import { Move, parseMove } from './nearText'; + export interface HybridArgs { alpha?: number; query: string; @@ -5,6 +7,27 @@ export interface HybridArgs { properties?: string[]; targetVectors?: string[]; fusionType?: FusionType; + searches?: HybridSubSearch[]; +} + +export interface NearTextSubSearch { + concepts: string[]; + certainty?: number; + distance?: number; + moveAwayFrom?: Move; + moveTo?: Move; +} + +export interface NearVectorSubSearch { + vector: number[]; + certainty?: number; + distance?: number; + targetVectors?: string[]; +} + +export interface HybridSubSearch { + nearText?: NearTextSubSearch; + nearVector?: NearVectorSubSearch; } export enum FusionType { @@ -12,6 +35,50 @@ export enum FusionType { relativeScoreFusion = 'relativeScoreFusion', } +class GraphQLHybridSubSearch { + private nearText?: NearTextSubSearch; + private nearVector?: NearVectorSubSearch; + + constructor(args: HybridSubSearch) { + this.nearText = args.nearText; + this.nearVector = args.nearVector; + } + + toString(): string { + let outer: string[] = []; + if (this.nearText !== undefined) { + let inner = [`concepts:${JSON.stringify(this.nearText.concepts)}`]; + if (this.nearText.certainty) { + inner = [...inner, `certainty:${this.nearText.certainty}`]; + } + if (this.nearText.distance) { + inner = [...inner, `distance:${this.nearText.distance}`]; + } + if (this.nearText.moveTo) { + inner = [...inner, parseMove('moveTo', this.nearText.moveTo)]; + } + if (this.nearText.moveAwayFrom) { + inner = [...inner, parseMove('moveAwayFrom', this.nearText.moveAwayFrom)]; + } + outer = [...outer, `nearText:{${inner.join(',')}}`]; + } + if (this.nearVector !== undefined) { + let inner = [`vector:${JSON.stringify(this.nearVector.vector)}`]; + if (this.nearVector.certainty) { + inner = [...inner, `certainty:${this.nearVector.certainty}`]; + } + if (this.nearVector.distance) { + inner = [...inner, `distance:${this.nearVector.distance}`]; + } + if (this.nearVector.targetVectors && this.nearVector.targetVectors.length > 0) { + inner = [...inner, `targetVectors:${JSON.stringify(this.nearVector.targetVectors)}`]; + } + outer = [...outer, `nearVector:{${inner.join(',')}}`]; + } + return `{${outer.join(',')}}`; + } +} + export default class GraphQLHybrid { private alpha?: number; private query: string; @@ -19,6 +86,7 @@ export default class GraphQLHybrid { private properties?: string[]; private targetVectors?: string[]; private fusionType?: FusionType; + private searches?: GraphQLHybridSubSearch[]; constructor(args: HybridArgs) { this.alpha = args.alpha; @@ -27,6 +95,7 @@ export default class GraphQLHybrid { this.properties = args.properties; this.targetVectors = args.targetVectors; this.fusionType = args.fusionType; + this.searches = args.searches?.map((search) => new GraphQLHybridSubSearch(search)); } toString() { @@ -52,6 +121,10 @@ export default class GraphQLHybrid { args = [...args, `fusionType:${this.fusionType}`]; } + if (this.searches !== undefined) { + args = [...args, `searches:[${this.searches.map((search) => search.toString()).join(',')}]`]; + } + return `{${args.join(',')}}`; } } diff --git a/src/graphql/journey.test.ts b/src/graphql/journey.test.ts index 65536614..b467c4d0 100644 --- a/src/graphql/journey.test.ts +++ b/src/graphql/journey.test.ts @@ -311,6 +311,26 @@ describe('the graphql journey', () => { }); }); + test('graphql get bm25 with query and groupby', () => { + return client.graphql + .get() + .withClassName('Article') + .withBm25({ query: 'Apple' }) + .withGroupBy({ + path: ['title'], + objectsPerGroup: 1, + groups: 1, + }) + .withFields('_additional { id }') + .do() + .then((res: any) => { + expect(res.data.Get.Article.length).toBe(1); + }) + .catch((e: any) => { + throw new Error('it should not have errord' + e); + }); + }); + test('graphql get nearText with autocut', () => { return client.graphql .get() @@ -424,6 +444,117 @@ describe('the graphql journey', () => { }); }); + test('graphql get hybrid with query and groupby', () => { + return client.graphql + .get() + .withClassName('Article') + .withHybrid({ query: 'Apple', properties: ['title'], alpha: 0 }) + .withGroupBy({ + path: ['title'], + objectsPerGroup: 1, + groups: 1, + }) + .withFields('_additional { id }') + .do() + .then((res: any) => { + expect(res.data.Get.Article.length).toBe(1); + }) + .catch((e: any) => { + throw new Error('it should not have errord' + e); + }); + }); + + test('graphql get hybrid with query and nearText subsearch', () => { + return client.graphql + .get() + .withClassName('Article') + .withHybrid({ + query: '', + searches: [ + { + nearText: { + concepts: ['Article'], + certainty: 0.7, + }, + }, + ], + }) + .withFields('_additional { id }') + .do() + .then((res: any) => { + expect(res.data.Get.Article.length).toBe(3); + }) + .catch((e: any) => { + throw new Error('it should not have errord' + e); + }); + }); + + test('graphql get hybrid with query and nearVector subsearch', () => { + const searchVec = [ + -0.15047126, 0.061322376, -0.17812507, 0.12811552, 0.36847013, -0.50840724, -0.10406531, 0.11413283, + 0.2997712, 0.7039331, 0.22155242, 0.1413957, 0.025396502, 0.14802167, 0.26640236, 0.15965445, + -0.45570126, -0.5215438, 0.14628491, 0.10946681, 0.0040095793, 0.017442623, -0.1988451, -0.05362646, + 0.104278944, -0.2506941, 0.2667653, 0.36438593, -0.44370207, 0.07204353, 0.077371456, 0.14557181, + 0.6026817, 0.45073593, 0.09438019, 0.03936342, -0.20441438, 0.12333719, -0.20247602, 0.5078446, + -0.06079732, -0.02166342, 0.02165861, -0.11712191, 0.0493167, -0.012123002, 0.26458082, -0.10784768, + -0.26852348, 0.049759883, -0.39999008, -0.08977922, 0.003169497, -0.36184034, -0.069065355, 0.18940343, + 0.5684866, -0.24626277, -0.2326087, 0.090373255, 0.33161184, -1.0541122, -0.039116446, -0.17496277, + -0.16834813, -0.0765323, -0.16189013, -0.062876746, -0.19826415, 0.07437007, -0.018362755, 0.23634757, + -0.19062655, -0.26524994, 0.33691254, -0.1926698, 0.018848037, 0.1735524, 0.34301907, -0.014238952, + -0.07596742, -0.61302894, -0.044652265, 0.1545376, 0.67256856, 0.08630557, 0.50236076, 0.23438522, + 0.27686095, 0.13633616, -0.27525797, 0.04282576, 0.18319897, -0.008353968, -0.27330264, 0.12624736, + -0.17051372, -0.35854533, -0.008455927, 0.154786, -0.20306401, -0.09021733, 0.80594194, 0.036562894, + -0.48894945, -0.27981675, -0.5001396, -0.3581464, -0.057082724, -0.0051904973, -0.3209166, 0.057098284, + 0.111587055, -0.09097725, -0.213181, -0.5038173, -0.024070809, -0.05350453, 0.13345918, -0.42136985, + 0.24050911, -0.2556207, 0.03156968, 0.4381214, 0.053237516, -0.20783865, 1.885739, 0.28429136, + -0.12231187, -0.30934808, 0.032250155, -0.32959512, 0.08670603, -0.60112613, -0.43010503, 0.70870006, + 0.3548015, -0.010406012, 0.036294986, 0.0030629474, -0.017579105, 0.28948352, -0.48063236, -0.39739868, + 0.17860937, 0.5099417, -0.24304488, -0.12671146, -0.018249692, -0.32057074, -0.08146134, 0.3572229, + -0.47601065, 0.35100546, -0.19663939, 0.34194613, -0.04653828, 0.47278664, -0.8723091, -0.19756387, + -0.5890681, 0.16688067, -0.23709822, -0.26478595, -0.18792373, 0.2204168, 0.030987943, 0.15885714, + -0.38817936, -0.4194334, -0.3287098, 0.15394142, -0.09496768, 0.6561987, -0.39340565, -0.5479265, + -0.22363484, -0.1193662, 0.2014849, 0.31138006, -0.45485613, -0.9879565, 0.3708223, 0.17318928, + 0.21229307, 0.042776756, -0.077399045, 0.42621315, -0.09917796, 0.34220153, 0.06380378, 0.14129028, + -0.14563583, -0.07081333, 0.026335392, 0.10566285, -0.28074324, -0.059861198, -0.24855351, 0.13623764, + -0.8228192, -0.15095113, 0.16250934, 0.031107651, -0.1504525, 0.20840737, 0.12919411, -0.0926323, + 0.30937102, 0.16636328, -0.36754072, 0.035581365, -0.2799259, 0.1446048, -0.11680267, 0.13226685, + 0.175023, -0.18840964, 0.27609056, -0.09350581, 0.08284562, 0.45897093, 0.13188471, -0.07115303, + 0.18009436, 0.16689545, -0.6991295, 0.26496106, -0.29619592, -0.19242188, -0.6362671, -0.16330126, + 0.2474778, 0.37738156, -0.12921557, -0.07843309, 0.28509396, 0.5658691, 0.16096894, 0.095068075, + 0.02419672, -0.30691084, 0.21180221, 0.21670066, 0.0027263877, 0.30853105, -0.16187873, 0.20786561, + 0.22136153, -0.008828387, -0.011165021, 0.60076475, 0.0089871045, 0.6179727, -0.38049766, -0.08179336, + -0.15306218, -0.13186441, -0.5360041, -0.06123339, -0.06399122, 0.21292226, -0.18383273, -0.21540102, + 0.28566808, -0.29953584, -0.36946672, 0.03341637, -0.08435299, -0.5381947, -0.28651953, 0.08704594, + -0.25493965, 0.0019178925, -0.7242109, 0.3578676, -0.55617595, -0.01930952, 0.32922924, 0.14903364, + 0.21613406, -0.11927183, 0.15165499, -0.10101261, 0.2499076, -0.18526322, -0.057230365, 0.10008554, + 0.16178907, 0.39356324, -0.03106238, 0.09375929, 0.17185533, 0.10400415, -0.36850816, 0.18424486, + -0.081376314, 0.23645392, 0.05198973, 0.09471436, + ]; + + return client.graphql + .get() + .withClassName('Article') + .withHybrid({ + query: '', + searches: [ + { + nearVector: { + vector: searchVec, + certainty: 0.7, + }, + }, + ], + }) + .withFields('_additional { id }') + .do() + .then((res: any) => { + expect(res.data.Get.Article.length).toBe(3); + }) + .catch((e: any) => { + throw new Error('it should not have errord' + e); + }); + }); + test('graphql get with nearText (with certainty)', () => { return client.graphql .get() diff --git a/src/graphql/nearText.ts b/src/graphql/nearText.ts index 596f1d76..4edef311 100644 --- a/src/graphql/nearText.ts +++ b/src/graphql/nearText.ts @@ -56,34 +56,11 @@ export default class GraphQLNearText { } if (this.moveTo) { - let moveToArgs: string[] = []; - if (this.moveTo.concepts) { - moveToArgs = [...moveToArgs, `concepts:${JSON.stringify(this.moveTo.concepts)}`]; - } - if (this.moveTo.objects) { - moveToArgs = [...moveToArgs, `objects:${this.parseMoveObjects('moveTo', this.moveTo.objects)}`]; - } - if (this.moveTo.force) { - moveToArgs = [...moveToArgs, `force:${this.moveTo.force}`]; - } - args = [...args, `moveTo:{${moveToArgs.join(',')}}`]; + args = [...args, parseMove('moveTo', this.moveTo)]; } if (this.moveAwayFrom) { - let moveAwayFromArgs: string[] = []; - if (this.moveAwayFrom.concepts) { - moveAwayFromArgs = [...moveAwayFromArgs, `concepts:${JSON.stringify(this.moveAwayFrom.concepts)}`]; - } - if (this.moveAwayFrom.objects) { - moveAwayFromArgs = [ - ...moveAwayFromArgs, - `objects:${this.parseMoveObjects('moveAwayFrom', this.moveAwayFrom.objects)}`, - ]; - } - if (this.moveAwayFrom.force) { - moveAwayFromArgs = [...moveAwayFromArgs, `force:${this.moveAwayFrom.force}`]; - } - args = [...args, `moveAwayFrom:{${moveAwayFromArgs.join(',')}}`]; + args = [...args, parseMove('moveAwayFrom', this.moveAwayFrom)]; } if (this.autocorrect !== undefined) { @@ -112,24 +89,38 @@ export default class GraphQLNearText { } } } +} - parseMoveObjects(move: MoveType, objects: MoveObject[]): string { - const moveObjects: string[] = []; - for (const i in objects) { - if (!objects[i].id && !objects[i].beacon) { - throw new Error(`nearText: ${move}.objects[${i}].id or ${move}.objects[${i}].beacon must be present`); - } - const objs = []; - if (objects[i].id) { - objs.push(`id:"${objects[i].id}"`); - } - if (objects[i].beacon) { - objs.push(`beacon:"${objects[i].beacon}"`); - } - moveObjects.push(`{${objs.join(',')}}`); +type MoveType = 'moveTo' | 'moveAwayFrom'; + +export function parseMoveObjects(move: MoveType, objects: MoveObject[]): string { + const moveObjects: string[] = []; + for (const i in objects) { + if (!objects[i].id && !objects[i].beacon) { + throw new Error(`nearText: ${move}.objects[${i}].id or ${move}.objects[${i}].beacon must be present`); + } + const objs = []; + if (objects[i].id) { + objs.push(`id:"${objects[i].id}"`); } - return `[${moveObjects.join(',')}]`; + if (objects[i].beacon) { + objs.push(`beacon:"${objects[i].beacon}"`); + } + moveObjects.push(`{${objs.join(',')}}`); } + return `[${moveObjects.join(',')}]`; } -type MoveType = 'moveTo' | 'moveAwayFrom'; +export function parseMove(move: MoveType, args: Move): string { + let moveArgs: string[] = []; + if (args.concepts) { + moveArgs = [...moveArgs, `concepts:${JSON.stringify(args.concepts)}`]; + } + if (args.objects) { + moveArgs = [...moveArgs, `objects:${parseMoveObjects(move, args.objects)}`]; + } + if (args.force) { + moveArgs = [...moveArgs, `force:${args.force}`]; + } + return `${move}:{${moveArgs.join(',')}}`; +} diff --git a/src/openapi/schema.ts b/src/openapi/schema.ts index cf0bd1e8..a17803d5 100644 --- a/src/openapi/schema.ts +++ b/src/openapi/schema.ts @@ -112,9 +112,6 @@ export interface paths { /** Gives meta information about the server and can be used to provide information to another Weaviate instance that wants to interact with the current instance. */ get: operations['meta.get']; }; - '/schema/cluster-status': { - get: operations['schema.cluster.status']; - }; '/schema': { get: operations['schema.dump']; post: operations['schema.objects.create']; @@ -145,6 +142,10 @@ export interface paths { /** delete tenants from a specific class */ delete: operations['tenants.delete']; }; + '/schema/{className}/tenants/{tenantName}': { + /** Check if a tenant exists for a specific class */ + head: operations['tenant.exists']; + }; '/backups/{backend}': { /** Starts a process of creating a backup for a set of classes */ post: operations['backups.create']; @@ -159,6 +160,10 @@ export interface paths { /** Starts a process of restoring a backup for a set of classes */ post: operations['backups.restore']; }; + '/cluster/statistics': { + /** Returns Raft cluster statistics of Weaviate DB. */ + get: operations['cluster.get.statistics']; + }; '/nodes': { /** Returns status of Weaviate DB. */ get: operations['nodes.get']; @@ -364,6 +369,8 @@ export interface definitions { MultiTenancyConfig: { /** @description Whether or not multi-tenancy is enabled for this class */ enabled?: boolean; + /** @description Nonexistent tenants should (not) be created implicitly */ + autoTenantCreation?: boolean; }; /** @description JSON object value. */ JsonObject: { [key: string]: unknown }; @@ -716,6 +723,8 @@ export interface definitions { * @description The length of the vector indexing queue. */ vectorQueueLength?: number; + /** @description The load status of the shard. */ + loaded?: boolean; }; /** @description The definition of a backup node status response body */ NodeStatus: { @@ -742,6 +751,57 @@ export interface definitions { NodesStatusResponse: { nodes?: definitions['NodeStatus'][]; }; + /** @description The definition of Raft statistics. */ + RaftStatistics: { + appliedIndex?: string; + commitIndex?: string; + fsmPending?: string; + lastContact?: string; + lastLogIndex?: string; + lastLogTerm?: string; + lastSnapshotIndex?: string; + lastSnapshotTerm?: string; + /** @description Weaviate Raft nodes. */ + latestConfiguration?: { [key: string]: unknown }; + latestConfigurationIndex?: string; + numPeers?: string; + protocolVersion?: string; + protocolVersionMax?: string; + protocolVersionMin?: string; + snapshotVersionMax?: string; + snapshotVersionMin?: string; + state?: string; + term?: string; + }; + /** @description The definition of node statistics. */ + Statistics: { + /** @description The name of the node. */ + name?: string; + /** + * @description Node's status. + * @default HEALTHY + * @enum {string} + */ + status?: 'HEALTHY' | 'UNHEALTHY' | 'UNAVAILABLE' | 'TIMEOUT'; + bootstrapped?: boolean; + dbLoaded?: boolean; + /** Format: uint64 */ + initialLastAppliedIndex?: number; + lastAppliedIndex?: number; + isVoter?: boolean; + leaderId?: { [key: string]: unknown }; + leaderAddress?: { [key: string]: unknown }; + open?: boolean; + ready?: boolean; + candidates?: { [key: string]: unknown }; + /** @description Weaviate Raft statistics. */ + raft?: definitions['RaftStatistics']; + }; + /** @description The cluster statistics of all of the Weaviate nodes */ + ClusterStatisticsResponse: { + statistics?: definitions['Statistics'][]; + synchronized?: boolean; + }; /** @description Either set beacon (direct reference) or set class and schema (concept reference) */ SingleRef: { /** @@ -2216,19 +2276,13 @@ export interface operations { }; }; }; - 'schema.cluster.status': { - responses: { - /** The schema in the cluster is in sync. */ - 200: { - schema: definitions['SchemaClusterStatus']; - }; - /** The schema is either out of sync (see response body) or the sync check could not be completed. */ - 500: { - schema: definitions['SchemaClusterStatus']; + 'schema.dump': { + parameters: { + header: { + /** If consistency is true, the request will be proxied to the leader to ensure strong schema consistency */ + consistency?: boolean; }; }; - }; - 'schema.dump': { responses: { /** Successfully dumped the database schema. */ 200: { @@ -2278,6 +2332,10 @@ export interface operations { path: { className: string; }; + header: { + /** If consistency is true, the request will be proxied to the leader to ensure strong schema consistency */ + consistency?: boolean; + }; }; responses: { /** Found the Class, returned as body */ @@ -2460,6 +2518,10 @@ export interface operations { path: { className: string; }; + header: { + /** If consistency is true, the request will be proxied to the leader to ensure strong schema consistency */ + consistency?: boolean; + }; }; responses: { /** tenants from specified class. */ @@ -2573,6 +2635,39 @@ export interface operations { }; }; }; + /** Check if a tenant exists for a specific class */ + 'tenant.exists': { + parameters: { + path: { + className: string; + tenantName: string; + }; + header: { + /** If consistency is true, the request will be proxied to the leader to ensure strong schema consistency */ + consistency?: boolean; + }; + }; + responses: { + /** The tenant exists in the specified class */ + 200: unknown; + /** Unauthorized or invalid credentials. */ + 401: unknown; + /** Forbidden */ + 403: { + schema: definitions['ErrorResponse']; + }; + /** The tenant not found */ + 404: unknown; + /** Invalid Tenant class */ + 422: { + schema: definitions['ErrorResponse']; + }; + /** An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. */ + 500: { + schema: definitions['ErrorResponse']; + }; + }; + }; /** Starts a process of creating a backup for a set of classes */ 'backups.create': { parameters: { @@ -2709,6 +2804,29 @@ export interface operations { }; }; }; + /** Returns Raft cluster statistics of Weaviate DB. */ + 'cluster.get.statistics': { + responses: { + /** Cluster statistics successfully returned */ + 200: { + schema: definitions['ClusterStatisticsResponse']; + }; + /** Unauthorized or invalid credentials. */ + 401: unknown; + /** Forbidden */ + 403: { + schema: definitions['ErrorResponse']; + }; + /** Invalid backup restoration status attempt. */ + 422: { + schema: definitions['ErrorResponse']; + }; + /** An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. */ + 500: { + schema: definitions['ErrorResponse']; + }; + }; + }; /** Returns status of Weaviate DB. */ 'nodes.get': { parameters: { diff --git a/src/schema/index.ts b/src/schema/index.ts index d190a75a..49a27c0c 100644 --- a/src/schema/index.ts +++ b/src/schema/index.ts @@ -14,6 +14,7 @@ import TenantsDeleter from './tenantsDeleter'; import Connection from '../connection'; import deleteAll from './deleteAll'; import { Tenant } from '../openapi/types'; +import TenantsExists from './tenantsExists'; export interface Schema { classCreator: () => ClassCreator; @@ -30,6 +31,7 @@ export interface Schema { tenantsGetter: (className: string) => TenantsGetter; tenantsUpdater: (className: string, tenants: Array) => TenantsUpdater; tenantsDeleter: (className: string, tenants: Array) => TenantsDeleter; + tenantsExists: (className: string, tenant: string) => TenantsExists; } const schema = (client: Connection): Schema => { @@ -51,6 +53,7 @@ const schema = (client: Connection): Schema => { new TenantsUpdater(client, className, tenants), tenantsDeleter: (className: string, tenants: Array) => new TenantsDeleter(client, className, tenants), + tenantsExists: (className: string, tenant: string) => new TenantsExists(client, className, tenant), }; }; @@ -66,3 +69,4 @@ export { default as TenantsCreator } from './tenantsCreator'; export { default as TenantsUpdater } from './tenantsUpdater'; export { default as TenantsGetter } from './tenantsGetter'; export { default as TenantsDeleter } from './tenantsDeleter'; +export { default as TenantsExists } from './tenantsExists'; diff --git a/src/schema/journey.test.ts b/src/schema/journey.test.ts index 11badc51..aba97a0e 100644 --- a/src/schema/journey.test.ts +++ b/src/schema/journey.test.ts @@ -181,6 +181,7 @@ describe('schema', () => { }, }, multiTenancyConfig: { + autoTenantCreation: false, enabled: false, }, shardingConfig: { @@ -587,7 +588,7 @@ describe('property setting defaults and migrations', () => { const errMsg1 = '`indexInverted` is deprecated and can not be set together with `indexFilterable` or `indexSearchable`'; - const errMsg2 = '`indexSearchable` is not allowed for other than text/text[] data types'; + const errMsg2 = '`indexSearchable` is allowed only for text/text[] data types'; test.each([ ['text', false, null, false, errMsg1], ['text', false, null, true, errMsg1], @@ -678,6 +679,7 @@ describe('multi tenancy', () => { vectorIndexType: 'hnsw', vectorizer: 'text2vec-contextionary', multiTenancyConfig: { + autoTenantCreation: true, enabled: true, }, }; @@ -731,6 +733,20 @@ describe('multi tenancy', () => { }); }); + it('successfully finds an existing tenant for MultiTenancy class', () => { + return client.schema + .tenantsExists(classObj.class!, tenants[1].name!) + .do() + .then((res: boolean) => expect(res).toEqual(true)); + }); + + it('successfully fails to find a non-existant tenant for MultiTenancy class', () => { + return client.schema + .tenantsExists(classObj.class!, 'nonExistantTenant') + .do() + .then((res: boolean) => expect(res).toEqual(false)); + }); + it('deletes MultiTenancy class', () => { return deleteClass(client, classObj.class!); }); @@ -826,6 +842,7 @@ function newClassObject(className: string) { }, }, multiTenancyConfig: { + autoTenantCreation: false, enabled: false, }, shardingConfig: { diff --git a/src/schema/tenantsExists.ts b/src/schema/tenantsExists.ts new file mode 100644 index 00000000..9abd98c3 --- /dev/null +++ b/src/schema/tenantsExists.ts @@ -0,0 +1,22 @@ +import Connection from '../connection'; +import { CommandBase } from '../validation/commandBase'; +import { Tenant } from '../openapi/types'; + +export default class TenantsExists extends CommandBase { + private className: string; + private tenant: string; + + constructor(client: Connection, className: string, tenant: string) { + super(client); + this.className = className; + this.tenant = tenant; + } + + validate = () => { + // nothing to validate + }; + + do = (): Promise => { + return this.client.head(`/schema/${this.className}/tenants/${this.tenant}`, undefined); + }; +} diff --git a/tools/refresh_schema.sh b/tools/refresh_schema.sh index 95c411b5..d4f9389b 100755 --- a/tools/refresh_schema.sh +++ b/tools/refresh_schema.sh @@ -2,6 +2,6 @@ set -euo pipefail -branchOrTag="${1:-master}" +branchOrTag="${1:-main}" npx openapi-typescript https://raw.githubusercontent.com/weaviate/weaviate/${branchOrTag}/openapi-specs/schema.json -o ./src/openapi/schema.ts npx prettier --write --no-error-on-unmatched-pattern './src/openapi/schema.ts'