I want to run Milvus on my local cluster.
I thought to use docker compose file and docker stack/docker swarm, but when I run my compose file something is not working. I can see all services started on the target nodes but for some reason milvus-proxy listening port is not available.
[ofarjon@rock01 vdb]$ docker ps | grep milvus_milvus-proxy
854486cdb68a milvusdb/milvus:v2.2.10 "/tini -- milvus run…" 51 seconds ago Up 49 seconds **NO PORT HERE** milvus_milvus-proxy.1.jzkt5tq8ck1fq34lx9odgr2jp
minio & etcd listening ports are shown in the docker ps output
These are the steps I’ve made:
I defined a master node and joined a worker using docker swarm
docker swarm init --advertise-addr <master-ip>
docker swarm join --token <token> <master-ip>
Then I defined the compose file:
services:
etcd:
image: quay.io/coreos/etcd:v3.5.5
container_name: etcd
environment:
- ETCD_AUTO_COMPACTION_RETENTION=1
- ETCD_QUOTA_BACKEND_BYTES=4294967296
- ALLOW_NONE_AUTHENTICATION=yes
- ETCDCTL_API=3
ports:
- "2379:2379"
- "2380:2380"
deploy:
placement:
constraints: [node.hostname == master_node]
minio:
image: minio/minio
container_name: minio
environment:
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
command: minio server /minio_data
volumes:
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
ports:
- "9000:9000"
deploy:
placement:
constraints: [node.hostname == master_node]
milvus-proxy:
image: milvusdb/milvus:v2.2.10
container_name: milvus-proxy
command: ["milvus", "run", "proxy"]
environment:
ETCD_ENDPOINTS: <master_node_ip>:2379
MINIO_ADDRESS: <master_node_ip>:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
CUDA_VISIBLE_DEVICES: 0
ports:
- "<master_node_ip>:19530:19530"
deploy:
placement:
constraints: [node.hostname == master_node]
milvus-rootcoord:
image: milvusdb/milvus:v2.2.10
container_name: milvus-rootcoord
command: ["milvus", "run", "rootcoord"]
environment:
ETCD_ENDPOINTS: <master_node_ip>:2379
MINIO_ADDRESS: <master_node_ip>:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
CUDA_VISIBLE_DEVICES: 0
deploy:
placement:
constraints: [node.hostname == master_node]
milvus-datacoord:
image: milvusdb/milvus:v2.2.10
container_name: milvus-datacoord
command: ["milvus", "run", "datacoord"]
environment:
ETCD_ENDPOINTS: <master_node_ip>:2379
MINIO_ADDRESS: <master_node_ip>:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
CUDA_VISIBLE_DEVICES: 0
deploy:
placement:
constraints: [node.hostname == master_node]
milvus-indexcoord:
image: milvusdb/milvus:v2.2.10
container_name: milvus-indexcoord
command: ["milvus", "run", "indexcoord"]
environment:
ETCD_ENDPOINTS: <master_node_ip>:2379
MINIO_ADDRESS: <master_node_ip>:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
CUDA_VISIBLE_DEVICES: 0
deploy:
placement:
constraints: [node.hostname == master_node]
milvus-querycoord:
image: milvusdb/milvus:v2.2.10
container_name: milvus-querycoord
command: ["milvus", "run", "querycoord"]
environment:
ETCD_ENDPOINTS: <master_node_ip>:2379
MINIO_ADDRESS: <master_node_ip>:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
CUDA_VISIBLE_DEVICES: 0
deploy:
placement:
constraints: [node.hostname == master_node]
milvus-datanode-1:
image: milvusdb/milvus:v2.2.10
container_name: milvus-datanode-1
command: ["milvus", "run", "datanode"]
environment:
ETCD_ENDPOINTS: <master_node_ip>:2379
MINIO_ADDRESS: <master_node_ip>:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
CUDA_VISIBLE_DEVICES: 0
deploy:
placement:
constraints: [node.hostname == master_node]
milvus-datanode-2:
--same as milvus-datanode-1 config --
constraints: [node.hostname == worker_node]
milvus-indexnode-1:
image: milvusdb/milvus:v2.2.10
container_name: milvus-indexnode-1
command: ["milvus", "run", "indexnode"]
environment:
ETCD_ENDPOINTS: <master_node_ip>:2379
MINIO_ADDRESS: <master_node_ip>:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
CUDA_VISIBLE_DEVICES: 0
deploy:
placement:
constraints: [node.hostname == master_node]
milvus-indexnode-2:
--same as milvus-indexnode-1 config --
deploy:
placement:
constraints: [node.hostname == worker_node]
milvus-querynode-1:
image: milvusdb/milvus:v2.2.10
container_name: milvus-querynode-1
command: ["milvus", "run", "querynode"]
environment:
ETCD_ENDPOINTS: <master_node_ip>:2379
MINIO_ADDRESS: <master_node_ip>:9000
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
CUDA_VISIBLE_DEVICES: 0
deploy:
placement:
constraints: [node.hostname == master_node]
milvus-querynode-2:
--same as milvus-querynode-1 config --
deploy:
placement:
constraints: [node.hostname == worker_node]
volumes:
minio-data:
And these are the milvus-proxy logs:
["Proxy internal server listen on tcp"] [port=19529]
[2024/07/11 15:35:05.996 +00:00] [INFO] [proxy/service.go:263] ["Proxy internal server already listen on tcp"] [port=19529]
[2024/07/11 15:35:05.996 +00:00] [INFO] [proxy/service.go:281] ["create Proxy internal grpc server"] ["enforcement policy"="{"MinTime":5000000000,"PermitWithoutStream":true}"] ["server parameters"="{"MaxConnectionIdle":0,"MaxConnectionAge":0,"MaxConnectionAgeGrace":0,"Time":60000000000,"Timeout":10000000000}"]
[2024/07/11 15:35:05.996 +00:00] [INFO] [proxy/service.go:155] ["Proxy server listen on tcp"] [port=19530]
[2024/07/11 15:35:05.996 +00:00] [INFO] [proxy/service.go:162] ["Proxy server already listen on tcp"] [port=19530]
[2024/07/11 15:35:05.996 +00:00] [INFO] [proxy/service.go:170] ["Get proxy rate limiter done"] [port=19530]
[2024/07/11 15:35:05.996 +00:00] [INFO] [proxy/service.go:233] ["create Proxy grpc server"] ["enforcement policy"="{"MinTime":5000000000,"PermitWithoutStream":true}"] ["server parameters"="{"MaxConnectionIdle":0,"MaxConnectionAge":0,"MaxConnectionAgeGrace":0,"Time":60000000000,"Timeout":10000000000}"]
[2024/07/11 15:35:05.996 +00:00] [INFO] [proxy/service.go:363] ["register http server of proxy"]