wangting0128
commented
6 months ago Recurrent
argo task: inverted-corn-1711728000 test case name: test_inverted_locust_varchar_dml_dql_cluster image: 2.4-20240329-32eff9c6e-amd64
server:
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
inverted-corn-128000-5-12-8571-etcd-0 1/1 Running 0 6m32s 10.104.25.71 4am-node30 <none> <none>
inverted-corn-128000-5-12-8571-etcd-1 1/1 Running 0 6m32s 10.104.28.63 4am-node33 <none> <none>
inverted-corn-128000-5-12-8571-etcd-2 1/1 Running 0 6m32s 10.104.33.206 4am-node36 <none> <none>
inverted-corn-128000-5-12-8571-milvus-datacoord-76cddfc5c5hk9n8 1/1 Running 0 6m32s 10.104.14.154 4am-node18 <none> <none>
inverted-corn-128000-5-12-8571-milvus-datanode-5f88d9f47f-dz8d4 1/1 Running 1 (2m1s ago) 6m32s 10.104.14.152 4am-node18 <none> <none>
inverted-corn-128000-5-12-8571-milvus-indexcoord-7cc64bb7btgvhc 1/1 Running 0 6m32s 10.104.27.188 4am-node31 <none> <none>
inverted-corn-128000-5-12-8571-milvus-indexnode-5b6b5ddfb92xld7 1/1 Running 0 6m32s 10.104.15.231 4am-node20 <none> <none>
inverted-corn-128000-5-12-8571-milvus-proxy-fb4f4578b-98l8q 1/1 Running 1 (2m1s ago) 6m32s 10.104.14.153 4am-node18 <none> <none>
inverted-corn-128000-5-12-8571-milvus-querycoord-68896b896spjtn 1/1 Running 1 (2m1s ago) 6m32s 10.104.14.151 4am-node18 <none> <none>
inverted-corn-128000-5-12-8571-milvus-querynode-b8f47f9ff-nxs5g 1/1 Running 0 6m32s 10.104.5.111 4am-node12 <none> <none>
inverted-corn-128000-5-12-8571-milvus-rootcoord-86d5565b7-hbmwh 1/1 Running 1 (2m ago) 6m32s 10.104.25.65 4am-node30 <none> <none>
inverted-corn-128000-5-12-8571-minio-0 1/1 Running 0 6m32s 10.104.25.72 4am-node30 <none> <none>
inverted-corn-128000-5-12-8571-minio-1 1/1 Running 0 6m31s 10.104.28.64 4am-node33 <none> <none>
inverted-corn-128000-5-12-8571-minio-2 1/1 Running 0 6m31s 10.104.27.194 4am-node31 <none> <none>
inverted-corn-128000-5-12-8571-minio-3 1/1 Running 0 6m31s 10.104.17.218 4am-node23 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-bookie-0 1/1 Running 0 6m32s 10.104.25.73 4am-node30 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-bookie-1 1/1 Running 0 6m31s 10.104.28.65 4am-node33 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-bookie-2 1/1 Running 0 6m31s 10.104.17.219 4am-node23 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-bookie-init-wt7wf 0/1 Completed 0 6m32s 10.104.17.212 4am-node23 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-broker-0 1/1 Running 0 6m32s 10.104.27.189 4am-node31 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-proxy-0 1/1 Running 0 6m32s 10.104.33.204 4am-node36 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-pulsar-init-42c74 0/1 Completed 0 6m32s 10.104.25.64 4am-node30 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-recovery-0 1/1 Running 0 6m32s 10.104.17.211 4am-node23 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-zookeeper-0 1/1 Running 0 6m32s 10.104.27.192 4am-node31 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-zookeeper-1 1/1 Running 0 5m50s 10.104.18.105 4am-node25 <none> <none>
inverted-corn-128000-5-12-8571-pulsar-zookeeper-2 1/1 Running 0 4m46s 10.104.23.61 4am-node27 <none> <none>client logs:
test steps:
concurrent test and calculation of RT and QPS
:purpose: `varchar: different max_length`
verify concurrent DML & DQL scenario which has 3 VARCHAR scalars fields and creating INVERTED index
:test steps:
1. create collection with fields:
'float_vector': 3dim,
'varchar_1': max_length=256, varchar_filled=True
'varchar_2': max_length=32768, varchar_filled=True
'varchar_3': max_length=65535, varchar_filled=True
2. build indexes:
IVF_FLAT: 'float_vector'
INVERTED: 'varchar_1', 'varchar_2', 'varchar_3'
3. insert 300k data
4. flush collection
5. build indexes again using the same params
6. load collection <- raise error
7. concurrent request:
- insert
- delete
- flush
- load
- search
- hybrid_search
- queryserver config:
{
"queryNode": {
"resources": {
"limits": {
"cpu": "8",
"memory": "32Gi"
},
"requests": {
"cpu": "8",
"memory": "32Gi"
}
},
"replicas": 1
},
"indexNode": {
"resources": {
"limits": {
"cpu": "4.0",
"memory": "16Gi"
},
"requests": {
"cpu": "3.0",
"memory": "9Gi"
}
},
"replicas": 1
},
"dataNode": {
"resources": {
"limits": {
"cpu": "2.0",
"memory": "4Gi"
},
"requests": {
"cpu": "2.0",
"memory": "3Gi"
}
}
},
"cluster": {
"enabled": true
},
"pulsar": {},
"kafka": {},
"minio": {
"metrics": {
"podMonitor": {
"enabled": true
}
}
},
"etcd": {
"metrics": {
"enabled": true,
"podMonitor": {
"enabled": true
}
}
},
"metrics": {
"serviceMonitor": {
"enabled": true
}
},
"log": {
"level": "debug"
},
"image": {
"all": {
"repository": "harbor.milvus.io/milvus/milvus",
"tag": "2.4-20240329-32eff9c6e-amd64"
}
}
}client config:
{
"dataset_params": {
"metric_type": "L2",
"dim": 3,
"scalars_index": {
"varchar_1": {
"index_type": "INVERTED"
},
"varchar_2": {
"index_type": "INVERTED"
},
"varchar_3": {
"index_type": "INVERTED"
}
},
"scalars_params": {
"varchar_1": {
"params": {
"max_length": 256
},
"other_params": {
"varchar_filled": true
}
},
"varchar_2": {
"params": {
"max_length": 32768
},
"other_params": {
"varchar_filled": true
}
},
"varchar_3": {
"params": {
"max_length": 65535
},
"other_params": {
"varchar_filled": true
}
}
},
"dataset_name": "local",
"dataset_size": 300000,
"ni_per": 50
},
"collection_params": {
"other_fields": [
"varchar_1",
"varchar_2",
"varchar_3"
],
"shards_num": 2
},
"resource_groups_params": {
"reset": false
},
"database_user_params": {
"reset_rbac": false,
"reset_db": false
},
"index_params": {
"index_type": "IVF_FLAT",
"index_param": {
"nlist": 1024
}
},
"concurrent_params": {
"concurrent_number": [
50
],
"during_time": "1h",
"interval": 20
},
"concurrent_tasks": [
{
"type": "insert",
"weight": 1,
"params": {
"nb": 10,
"timeout": 30,
"random_id": true,
"random_vector": true,
"varchar_filled": false,
"start_id": 300000
}
},
{
"type": "delete",
"weight": 1,
"params": {
"expr": "",
"delete_length": 10,
"timeout": 30
}
},
{
"type": "flush",
"weight": 1,
"params": {
"timeout": 600
}
},
{
"type": "load",
"weight": 1,
"params": {
"replica_number": 1,
"timeout": 30
}
},
{
"type": "search",
"weight": 1,
"params": {
"nq": 1000,
"top_k": 1,
"search_param": {
"nprobe": 32
},
"expr": "varchar_1 like \"a%\" && varchar_2 like \"A%\" && varchar_3 like \"0%\" && id > 0",
"guarantee_timestamp": null,
"partition_names": null,
"output_fields": null,
"ignore_growing": false,
"group_by_field": null,
"timeout": 60,
"random_data": true
}
},
{
"type": "hybrid_search",
"weight": 1,
"params": {
"nq": 1,
"top_k": 10,
"reqs": [
{
"search_param": {
"nprobe": 16
},
"anns_field": "float_vector",
"expr": "varchar_1 like \"0%\"",
"top_k": 2000
},
{
"search_param": {
"nprobe": 128
},
"anns_field": "float_vector",
"expr": "varchar_2 like \"9%\""
}
],
"rerank": {
"WeightedRanker": [
0.5,
0.5
]
},
"output_fields": [
"*"
],
"ignore_growing": false,
"guarantee_timestamp": null,
"partition_names": null,
"timeout": 60,
"random_data": true
}
},
{
"type": "query",
"weight": 1,
"params": {
"ids": null,
"expr": "varchar_3 like \"a%\" && ",
"output_fields": [
"*"
],
"offset": null,
"limit": null,
"ignore_growing": false,
"partition_names": null,
"timeout": 60,
"random_data": true,
"random_count": 20,
"random_range": [
0,
150000
],
"field_name": "id",
"field_type": "int64"
}
}
]
}
longjiquan
stale[bot]
Is there an existing issue for this?
Environment
Current Behavior
argo task: inverted-corn-1711209600 test case name: test_inverted_locust_varchar_dql_cluster
server:
The actual memory used by queryNode is about 1G, and the predicted memory usage is about 10G.
client pod name: inverted-corn-1711209600-971096786
Expected Behavior
No response
Steps To Reproduce
Milvus Log
No response
Anything else?
test result: