diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8d3ba9ff..f58bee78 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,7 +7,8 @@ on: pull_request: env: - NEW_WEAVIATE_VERSION: 1.24.10 + WEAVIATE_124: 1.24.12 + WEAVIATE_125: preview-merge-stable-v1-25-into-main-ec81815 jobs: checks: @@ -38,9 +39,12 @@ jobs: fail-fast: false matrix: versions: [ - { node: "18.x", weaviate: $NEW_WEAVIATE_VERSION}, - { node: "20.x", weaviate: $NEW_WEAVIATE_VERSION}, - { node: "22.x", weaviate: $NEW_WEAVIATE_VERSION} + { node: "18.x", weaviate: $WEAVIATE_124}, + { node: "20.x", weaviate: $WEAVIATE_124}, + { node: "22.x", weaviate: $WEAVIATE_124}, + { node: "18.x", weaviate: $WEAVIATE_125}, + { node: "20.x", weaviate: $WEAVIATE_125}, + { node: "22.x", weaviate: $WEAVIATE_125} ] steps: - uses: actions/checkout@v3 diff --git a/ci/docker-compose-cluster.yml b/ci/docker-compose-cluster.yml index 56d04ac3..aa44675a 100644 --- a/ci/docker-compose-cluster.yml +++ b/ci/docker-compose-cluster.yml @@ -6,16 +6,22 @@ services: restart: on-failure:0 ports: - "8087:8080" + - "50058:50051" environment: - CONTEXTIONARY_URL: contextionary:9999 QUERY_DEFAULTS_LIMIT: 20 AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' PERSISTENCE_DATA_PATH: "./weaviate-node-1" - DEFAULT_VECTORIZER_MODULE: text2vec-contextionary - ENABLE_MODULES: text2vec-contextionary + CLUSTER_HOSTNAME: "node1" CLUSTER_GOSSIP_BIND_PORT: "7110" CLUSTER_DATA_BIND_PORT: "7111" + RAFT_PORT: '8300' + RAFT_INTERNAL_RPC_PORT: "8301" + RAFT_JOIN: "node1:8300,node2:8300,node3:8300" + RAFT_BOOTSTRAP_EXPECT: "3" DISABLE_TELEMETRY: 'true' + CONTEXTIONARY_URL: contextionary:9999 + DEFAULT_VECTORIZER_MODULE: text2vec-contextionary + ENABLE_MODULES: text2vec-contextionary weaviate-node-2: init: true @@ -29,19 +35,55 @@ services: image: semitechnologies/weaviate:${WEAVIATE_VERSION} ports: - 8088:8080 - - 6061:6060 + - "50059:50051" restart: on-failure:0 environment: - CONTEXTIONARY_URL: contextionary:9999 LOG_LEVEL: 'debug' QUERY_DEFAULTS_LIMIT: 20 AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' PERSISTENCE_DATA_PATH: './weaviate-node-2' + CLUSTER_HOSTNAME: 'node2' + CLUSTER_GOSSIP_BIND_PORT: '7110' + CLUSTER_DATA_BIND_PORT: '7111' + CLUSTER_JOIN: 'weaviate-node-1:7110' + RAFT_PORT: '8300' + RAFT_INTERNAL_RPC_PORT: "8301" + RAFT_JOIN: "node1:8300,node2:8300,node3:8300" + RAFT_BOOTSTRAP_EXPECT: "3" + DISABLE_TELEMETRY: 'true' + CONTEXTIONARY_URL: contextionary:9999 DEFAULT_VECTORIZER_MODULE: text2vec-contextionary ENABLE_MODULES: text2vec-contextionary - CLUSTER_HOSTNAME: 'node2' - CLUSTER_GOSSIP_BIND_PORT: '7112' - CLUSTER_DATA_BIND_PORT: '7113' + + weaviate-node-3: + init: true + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: semitechnologies/weaviate:${WEAVIATE_VERSION} + ports: + - 8089:8080 + - "50060:50051" + restart: on-failure:0 + environment: + LOG_LEVEL: 'debug' + QUERY_DEFAULTS_LIMIT: 20 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: './weaviate-node-2' + CLUSTER_HOSTNAME: 'node3' + CLUSTER_GOSSIP_BIND_PORT: '7110' + CLUSTER_DATA_BIND_PORT: '7111' CLUSTER_JOIN: 'weaviate-node-1:7110' + RAFT_PORT: '8300' + RAFT_INTERNAL_RPC_PORT: "8301" + RAFT_JOIN: "node1:8300,node2:8300,node3:8300" + RAFT_BOOTSTRAP_EXPECT: "3" DISABLE_TELEMETRY: 'true' + CONTEXTIONARY_URL: contextionary:9999 + DEFAULT_VECTORIZER_MODULE: text2vec-contextionary + ENABLE_MODULES: text2vec-contextionary ... diff --git a/src/collections/query/integration.test.ts b/src/collections/query/integration.test.ts index 208ceb5a..60a2f500 100644 --- a/src/collections/query/integration.test.ts +++ b/src/collections/query/integration.test.ts @@ -339,7 +339,7 @@ describe('Testing of the collection.query methods with a collection with a refer it('should query with nearVector returning the referenced object', async () => { const res = await collection.query.fetchObjectById(id2, { includeVector: true }); - const ret = await collection.query.nearVector(res?.vectors.default!, { + const ret = await collection.query.nearVector(res?.vectors.vector!, { returnProperties: ['testProp'], returnReferences: [ { @@ -898,10 +898,10 @@ describe('Testing of the collection.query methods with a multi-tenancy collectio .query.fetchObjectById(id2, { includeVector: true }))!; const obj1 = await collection .withTenant(tenantOne) - .query.nearVector(vecs1.default, { targetVector: 'vector' }); + .query.nearVector(vecs1.vector, { targetVector: 'vector' }); const obj2 = await collection .withTenant(tenantTwo) - .query.nearVector(vecs2.default, { targetVector: 'vector' }); + .query.nearVector(vecs2.vector, { targetVector: 'vector' }); expect(obj1.objects.length).toEqual(1); expect(obj1.objects[0].properties.testProp).toEqual('one'); expect(obj1.objects[0].uuid).toEqual(id1); @@ -1029,7 +1029,7 @@ maybe('Testing of collection.query using rerank functionality', () => { it('should rerank the results in a nearObject query', async () => { const obj = await collection.query.fetchObjectById(id1, { includeVector: true }); - const ret = await collection.query.nearVector(obj?.vectors.default!, { + const ret = await collection.query.nearVector(obj?.vectors.vector!, { rerank: { property: 'text', query: 'another', diff --git a/src/openapi/schema.ts b/src/openapi/schema.ts index bdd2f754..32b96270 100644 --- a/src/openapi/schema.ts +++ b/src/openapi/schema.ts @@ -163,6 +163,10 @@ export interface paths { /** Starts a process of restoring a backup for a set of classes */ post: operations['backups.restore']; }; + '/cluster/statistics': { + /** Returns Raft cluster statistics of Weaviate DB. */ + get: operations['cluster.get.statistics']; + }; '/nodes': { /** Returns status of Weaviate DB. */ get: operations['nodes.get']; @@ -368,6 +372,8 @@ export interface definitions { MultiTenancyConfig: { /** @description Whether or not multi-tenancy is enabled for this class */ enabled?: boolean; + /** @description Nonexistent tenants should (not) be created implicitly */ + autoTenantCreation?: boolean; }; /** @description JSON object value. */ JsonObject: { [key: string]: unknown }; @@ -720,6 +726,8 @@ export interface definitions { * @description The length of the vector indexing queue. */ vectorQueueLength?: number; + /** @description The load status of the shard. */ + loaded?: boolean; }; /** @description The definition of a backup node status response body */ NodeStatus: { @@ -746,6 +754,57 @@ export interface definitions { NodesStatusResponse: { nodes?: definitions['NodeStatus'][]; }; + /** @description The definition of Raft statistics. */ + RaftStatistics: { + appliedIndex?: string; + commitIndex?: string; + fsmPending?: string; + lastContact?: string; + lastLogIndex?: string; + lastLogTerm?: string; + lastSnapshotIndex?: string; + lastSnapshotTerm?: string; + /** @description Weaviate Raft nodes. */ + latestConfiguration?: { [key: string]: unknown }; + latestConfigurationIndex?: string; + numPeers?: string; + protocolVersion?: string; + protocolVersionMax?: string; + protocolVersionMin?: string; + snapshotVersionMax?: string; + snapshotVersionMin?: string; + state?: string; + term?: string; + }; + /** @description The definition of node statistics. */ + Statistics: { + /** @description The name of the node. */ + name?: string; + /** + * @description Node's status. + * @default HEALTHY + * @enum {string} + */ + status?: 'HEALTHY' | 'UNHEALTHY' | 'UNAVAILABLE' | 'TIMEOUT'; + bootstrapped?: boolean; + dbLoaded?: boolean; + /** Format: uint64 */ + initialLastAppliedIndex?: number; + lastAppliedIndex?: number; + isVoter?: boolean; + leaderId?: { [key: string]: unknown }; + leaderAddress?: { [key: string]: unknown }; + open?: boolean; + ready?: boolean; + candidates?: { [key: string]: unknown }; + /** @description Weaviate Raft statistics. */ + raft?: definitions['RaftStatistics']; + }; + /** @description The cluster statistics of all of the Weaviate nodes */ + ClusterStatisticsResponse: { + statistics?: definitions['Statistics'][]; + synchronized?: boolean; + }; /** @description Either set beacon (direct reference) or set class and schema (concept reference) */ SingleRef: { /** @@ -2233,6 +2292,12 @@ export interface operations { }; }; 'schema.dump': { + parameters: { + header: { + /** If consistency is true, the request will be proxied to the leader to ensure strong schema consistency */ + consistency?: boolean; + }; + }; responses: { /** Successfully dumped the database schema. */ 200: { @@ -2282,6 +2347,10 @@ export interface operations { path: { className: string; }; + header: { + /** If consistency is true, the request will be proxied to the leader to ensure strong schema consistency */ + consistency?: boolean; + }; }; responses: { /** Found the Class, returned as body */ @@ -2464,6 +2533,10 @@ export interface operations { path: { className: string; }; + header: { + /** If consistency is true, the request will be proxied to the leader to ensure strong schema consistency */ + consistency?: boolean; + }; }; responses: { /** tenants from specified class. */ @@ -2584,6 +2657,10 @@ export interface operations { className: string; tenantName: string; }; + header: { + /** If consistency is true, the request will be proxied to the leader to ensure strong schema consistency */ + consistency?: boolean; + }; }; responses: { /** The tenant exists in the specified class */ @@ -2742,6 +2819,29 @@ export interface operations { }; }; }; + /** Returns Raft cluster statistics of Weaviate DB. */ + 'cluster.get.statistics': { + responses: { + /** Cluster statistics successfully returned */ + 200: { + schema: definitions['ClusterStatisticsResponse']; + }; + /** Unauthorized or invalid credentials. */ + 401: unknown; + /** Forbidden */ + 403: { + schema: definitions['ErrorResponse']; + }; + /** Invalid backup restoration status attempt. */ + 422: { + schema: definitions['ErrorResponse']; + }; + /** An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. */ + 500: { + schema: definitions['ErrorResponse']; + }; + }; + }; /** Returns status of Weaviate DB. */ 'nodes.get': { parameters: { diff --git a/src/schema/journey.test.ts b/src/schema/journey.test.ts index 0696360c..8fb77ce3 100644 --- a/src/schema/journey.test.ts +++ b/src/schema/journey.test.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ -import weaviate, { WeaviateClient } from '../v2/index.js'; +import weaviate, { MetaGetter, WeaviateClient } from '../v2/index.js'; import { WeaviateClass, Property, @@ -7,17 +7,29 @@ import { ShardStatus, ShardStatusList, Tenant, + Meta, } from '../openapi/types.js'; +const is125 = (client: WeaviateClient) => + client.misc + .metaGetter() + .do() + .then((res: Meta) => res.version) + .then((version: string) => { + const semver = version.split('.').map((v) => parseInt(v, 10)); + return semver[1] >= 25; + }); + describe('schema', () => { const client = weaviate.client({ scheme: 'http', host: 'localhost:8080', }); - const classObj = newClassObject('MyThingClass'); + const classObjPromise = newClassObject('MyThingClass', is125(client)); - it('creates a thing class (implicitly)', () => { + it('creates a thing class (implicitly)', async () => { + const classObj = await classObjPromise; return client.schema .classCreator() .withClass(classObj) @@ -27,7 +39,8 @@ describe('schema', () => { }); }); - it('gets an existing class', () => { + it('gets an existing class', async () => { + const classObj = await classObjPromise; return client.schema .classGetter() .withClassName(classObj.class) @@ -37,7 +50,8 @@ describe('schema', () => { }); }); - it('checks class existence', () => { + it('checks class existence', async () => { + const classObj = await classObjPromise; return client.schema.exists(classObj.class).then((res) => expect(res).toEqual(true)); }); @@ -101,7 +115,7 @@ describe('schema', () => { return client.schema .getter() .do() - .then((res: WeaviateSchema) => { + .then(async (res: WeaviateSchema) => { expect(res).toEqual({ classes: [ { @@ -181,6 +195,7 @@ describe('schema', () => { }, }, multiTenancyConfig: { + autoTenantCreation: (await is125(client)) ? false : undefined, enabled: false, }, shardingConfig: { @@ -202,7 +217,8 @@ describe('schema', () => { }); }); - it('gets the shards of an existing class', () => { + it('gets the shards of an existing class', async () => { + const classObj = await classObjPromise; return client.schema .shardsGetter() .withClassName(classObj.class) @@ -215,6 +231,7 @@ describe('schema', () => { }); it('updates a shard of an existing class to readonly', async () => { + const classObj = await classObjPromise; const shards = await getShards(client, classObj.class); expect(Array.isArray(shards)).toBe(true); expect(shards.length).toEqual(1); @@ -231,6 +248,7 @@ describe('schema', () => { }); it('updates a shard of an existing class to ready', async () => { + const classObj = await classObjPromise; const shards = await getShards(client, classObj.class); expect(Array.isArray(shards)).toBe(true); expect(shards.length).toEqual(1); @@ -246,7 +264,8 @@ describe('schema', () => { }); }); - it('deletes an existing class', () => { + it('deletes an existing class', async () => { + const classObj = await classObjPromise; return client.schema .classDeleter() .withClassName(classObj.class) @@ -258,7 +277,7 @@ describe('schema', () => { it('updates all shards in a class', async () => { const shardCount = 3; - const newClass: any = newClassObject('NewClass'); + const newClass: any = await newClassObject('NewClass', is125(client)); newClass.shardingConfig.desiredCount = shardCount; await client.schema @@ -301,7 +320,7 @@ describe('schema', () => { }); it('has updated values of bm25 config', async () => { - const newClass: any = newClassObject('NewClass'); + const newClass: any = await newClassObject('NewClass', is125(client)); const bm25Config = { k1: 1.13, b: 0.222 }; newClass.invertedIndexConfig.bm25 = bm25Config; @@ -318,7 +337,7 @@ describe('schema', () => { }); it('has updated values of stopwords config', async () => { - const newClass: any = newClassObject('SpaceClass'); + const newClass: any = await newClassObject('SpaceClass', is125(client)); const stopwordConfig: any = { preset: 'en', additions: ['star', 'nebula'], @@ -370,7 +389,7 @@ describe('schema', () => { it('creates a class with explicit replication config', async () => { const replicationFactor = 1; - const newClass: any = newClassObject('SomeClass'); + const newClass: any = await newClassObject('SomeClass', is125(client)); newClass.replicationConfig.factor = replicationFactor; await client.schema @@ -385,7 +404,7 @@ describe('schema', () => { }); it('creates a class with implicit replication config', async () => { - const newClass: any = newClassObject('SomeClass'); + const newClass: any = await newClassObject('SomeClass', is125(client)); delete newClass.replicationConfig; await client.schema @@ -399,9 +418,9 @@ describe('schema', () => { return deleteClass(client, newClass.class); }); - it('delete all data from the schema', () => { - const newClass: any = newClassObject('LetsDeleteThisClass'); - const newClass2: any = newClassObject('LetsDeleteThisClassToo'); + it('delete all data from the schema', async () => { + const newClass: any = await newClassObject('LetsDeleteThisClass', is125(client)); + const newClass2: any = await newClassObject('LetsDeleteThisClassToo', is125(client)); const classNames = [newClass.class, newClass2.class]; Promise.all([ client.schema.classCreator().withClass(newClass).do(), @@ -587,7 +606,7 @@ describe('property setting defaults and migrations', () => { const errMsg1 = '`indexInverted` is deprecated and can not be set together with `indexFilterable` or `indexSearchable`'; - const errMsg2 = '`indexSearchable` is not allowed for other than text/text[] data types'; + const errMsg2 = '"`indexSearchable`'; test.each([ ['text', false, null, false, errMsg1], ['text', false, null, true, errMsg1], @@ -662,6 +681,10 @@ describe('multi tenancy', () => { scheme: 'http', host: 'localhost:8080', }); + const versionPromise: Promise = client.misc + .metaGetter() + .do() + .then((res: Meta) => res.version); const classObj: WeaviateClass = { class: 'MultiTenancy', @@ -678,12 +701,16 @@ describe('multi tenancy', () => { vectorIndexType: 'hnsw', vectorizer: 'text2vec-contextionary', multiTenancyConfig: { + autoTenantCreation: true, enabled: true, }, }; const tenants: Array = [{ name: 'tenantA' }, { name: 'tenantB' }, { name: 'tenantC' }]; - it('creates a MultiTenancy class', () => { + it('creates a MultiTenancy class', async () => { + if (!(await is125(client))) { + delete classObj.multiTenancyConfig?.autoTenantCreation; + } return client.schema .classCreator() .withClass(classObj) @@ -735,33 +762,33 @@ describe('multi tenancy', () => { return deleteClass(client, classObj.class!); }); - const classObjWithoutMultiTenancyConfig = newClassObject('NoMultiTenancy'); + const classObjWithoutMultiTenancyConfig = newClassObject('NoMultiTenancy', is125(client)); - it('creates a NoMultiTenancy class', () => { + it('creates a NoMultiTenancy class', async () => { return client.schema .classCreator() - .withClass(classObjWithoutMultiTenancyConfig) + .withClass(await classObjWithoutMultiTenancyConfig) .do() - .then((res: WeaviateClass) => { - expect(res).toEqual(classObjWithoutMultiTenancyConfig); + .then(async (res: WeaviateClass) => { + expect(res).toEqual(await classObjWithoutMultiTenancyConfig); }); }); - it('fails to define tenants for NoMultiTenancy class', () => { + it('fails to define tenants for NoMultiTenancy class', async () => { return client.schema - .tenantsCreator(classObjWithoutMultiTenancyConfig.class!, tenants) + .tenantsCreator((await classObjWithoutMultiTenancyConfig).class!, tenants) .do() .catch((e: Error) => { expect(e.message).toContain('multi-tenancy is not enabled for class \\"NoMultiTenancy\\"'); }); }); - it('deletes NoMultiTenancy class', () => { - return deleteClass(client, classObjWithoutMultiTenancyConfig.class); + it('deletes NoMultiTenancy class', async () => { + return deleteClass(client, (await classObjWithoutMultiTenancyConfig).class); }); }); -function newClassObject(className: string) { +async function newClassObject(className: string, is125Promise: Promise) { return { class: className, properties: [ @@ -826,6 +853,7 @@ function newClassObject(className: string) { }, }, multiTenancyConfig: { + autoTenantCreation: (await is125Promise) ? false : undefined, enabled: false, }, shardingConfig: { diff --git a/tools/refresh_schema.sh b/tools/refresh_schema.sh index 95c411b5..d4f9389b 100755 --- a/tools/refresh_schema.sh +++ b/tools/refresh_schema.sh @@ -2,6 +2,6 @@ set -euo pipefail -branchOrTag="${1:-master}" +branchOrTag="${1:-main}" npx openapi-typescript https://raw.githubusercontent.com/weaviate/weaviate/${branchOrTag}/openapi-specs/schema.json -o ./src/openapi/schema.ts npx prettier --write --no-error-on-unmatched-pattern './src/openapi/schema.ts'