What is the proper way to search all partitions when enabling partition key isolation?

[guzy0324]

I tried to set filter to "", and got the following error:

"<MilvusException: (code=65535, message=partition key not found in expr or the expr is invalid when validating partition key isolation)>"

[yhmo]

No problem on milvus v2.5.4

import random

from pymilvus import (
MilvusClient,DataType,CollectionSchema,FieldSchema
)

client = MilvusClient(uri="http://localhost:19530")

collection_name = "AAA"
dim = 128
client.drop_collection(collection_name=collection_name)


fields = [
    FieldSchema(name="id", dtype=DataType.INT64, is_primary=True),
    FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=dim),
    FieldSchema(name="flag", dtype=DataType.INT64, is_partition_key=True),
]

schema = CollectionSchema(fields)
client.create_collection(
    collection_name=collection_name,
    schema=schema,
    num_partitions=20,
)
index_params = client.prepare_index_params()
index_params.add_index(field_name="vector", index_type="FLAT", metric_type="COSINE")
client.create_index(collection_name, index_params)
client.load_collection(collection_name=collection_name)
print(collection_name, "created")

for i in range(100):
    print("insert", i)
    data = [{"id": k, "vector": [random.random() for _ in range(dim)], "flag": k%100} for k in range(1000)]
    client.insert(collection_name=collection_name, data=data)



res = client.search(collection_name=collection_name,
                    data=[[random.random() for _ in range(dim)]],
                    limit=3,
                    filter="",
                    anns_field="vector",
                    output_fields=["flag"],
                    consistency_level="Strong",
                    )
for r in res:
    for k in r:
        print(k)

This script can return 3 results successfully.

[guzy0324]

No problem on milvus v2.5.4

import random

from pymilvus import (
MilvusClient,DataType,CollectionSchema,FieldSchema
)

client = MilvusClient(uri="http://localhost:19530")

collection_name = "AAA"
dim = 128
client.drop_collection(collection_name=collection_name)


fields = [
    FieldSchema(name="id", dtype=DataType.INT64, is_primary=True),
    FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=dim),
    FieldSchema(name="flag", dtype=DataType.INT64, is_partition_key=True),
]

schema = CollectionSchema(fields)
client.create_collection(
    collection_name=collection_name,
    schema=schema,
    num_partitions=20,
)
index_params = client.prepare_index_params()
index_params.add_index(field_name="vector", index_type="FLAT", metric_type="COSINE")
client.create_index(collection_name, index_params)
client.load_collection(collection_name=collection_name)
print(collection_name, "created")

for i in range(100):
    print("insert", i)
    data = [{"id": k, "vector": [random.random() for _ in range(dim)], "flag": k%100} for k in range(1000)]
    client.insert(collection_name=collection_name, data=data)



res = client.search(collection_name=collection_name,
                    data=[[random.random() for _ in range(dim)]],
                    limit=3,
                    filter="",
                    anns_field="vector",
                    output_fields=["flag"],
                    consistency_level="Strong",
                    )
for r in res:
    for k in r:
        print(k)

This script can return 3 results successfully.

Your code doesn't enable partition key isolation.
The error can be reproduced by enabling the partition key isolation, e.g.:

import random

from pymilvus import (
MilvusClient,DataType,CollectionSchema,FieldSchema
)

client = MilvusClient(uri="http://localhost:19530")

collection_name = "AAA"
dim = 128
client.drop_collection(collection_name=collection_name)


fields = [
    FieldSchema(name="id", dtype=DataType.INT64, is_primary=True),
    FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=dim),
    FieldSchema(name="flag", dtype=DataType.INT64, is_partition_key=True),
]

schema = CollectionSchema(fields)
client.create_collection(
    collection_name=collection_name,
    schema=schema,
    num_partitions=20,
    properties={"partitionkey.isolation": True},
)
index_params = client.prepare_index_params()
index_params.add_index(field_name="vector", index_type="FLAT", metric_type="COSINE")
client.create_index(collection_name, index_params)
client.load_collection(collection_name=collection_name)
print(collection_name, "created")

for i in range(100):
    print("insert", i)
    data = [{"id": k, "vector": [random.random() for _ in range(dim)], "flag": k%100} for k in range(1000)]
    client.insert(collection_name=collection_name, data=data)



res = client.search(collection_name=collection_name,
                    data=[[random.random() for _ in range(dim)]],
                    limit=3,
                    filter="",
                    anns_field="vector",
                    output_fields=["flag"],
                    consistency_level="Strong",
                    )
for r in res:
    for k in r:
        print(k)

[yhmo]

I just read the source code. It seems with partition key isolation enabled, you must input a partition key in the expression, or it throws this error:

milvus/internal/util/exprutil/expr_checker.go

Line 519 in bc8e02d

return errors.New("partition key not found in expr or the expr is invalid when validating partition key isolation")

I didn't find a workaround to search the entire collection. It may be a bug of feature design.

[chasingegg]

partition key isolation is designed for multi-tenancy scenario, it is restricted to specific partition key scope and not allowed to search across the whole collection, which is described in doc https://milvus.io/docs/use-partition-key.md#Use-Partition-Key-Isolation. So what is your usecase to use partition key isolation? @guzy0324