milvus向量数据库

版本介绍Milvus LiteMilvus Lite是一个 Python 库可导入到您的应用程序中。作为 Milvus 的轻量级版本它非常适合在 Jupyter 笔记本或资源有限的智能设备上运行快速原型。Milvus Lite 支持与 Milvus 其他部署相同的 API。与 Milvus Lite 交互的客户端代码也能与其他部署模式下的 Milvus 实例协同工作。要将 Milvus Lite 集成到应用程序中请运行pip install pymilvus进行安装并使用MilvusClient(./demo.db)语句实例化一个带有本地文件的向量数据库以持久化所有数据。更多详情请参阅运行 Milvus Lite。Milvus Lite 目前支持以下环境Ubuntu 20.04x86_64 和 arm64MacOS 11.0苹果硅 M1/M2 和 x86_64Milvus 单机版Milvus Standalone 是单机服务器部署。Milvus Standalone 的所有组件都打包到一个Docker 镜像中部署起来非常方便。如果你有生产工作负载但又不想使用 Kubernetes那么在内存充足的单机上运行 Milvus Standalone 是一个不错的选择。此外Milvus Standalone 通过主从复制支持高可用性。分布式 MilvusMilvus Distributed 可部署在Kubernetes集群上。这种部署采用云原生架构摄取负载和搜索查询分别由独立节点处理允许关键组件冗余。它具有最高的可扩展性和可用性并能灵活定制每个组件中分配的资源。Milvus Distributed 是在生产中运行大规模向量搜索系统的企业用户的首选。Milvus 单机版部署使用Docker Compose安装 Milvus standalone即单机版进行一个快速milvus的体验。前提条件1.系统可以使用centos或者ubuntu,这里使用的是Ubuntu 20.04.6 LTS2.系统已经安装docker和docker-compose3.milvus版本这里使用2.3.12启动etcd、minio、milvus由于milvus依赖etcd和minio因此需要先启动这2个组件。同样也使用docker进行启动。etcd用来存储milvus的元数据。minio用来存储milvus的向量数据和索引数据。下载milvus-standalone-docker-compose.yml 文件保存为docker-compose.yml:wgethttps://github.com/milvus-io/milvus/releases/download/v2.3.12/milvusstandalone-docker-compose.yml-Odocker-compose.yml这里经过了一定修改让其更加方便使用。 这个yml文件里面定义了etcd、minio、milvus的启动参数。 修改后的docker-compose.yml文件内容如下version:3.5services: etcd: container_name: milvus-etcd image: quay.io/coreos/etcd:v3.5.5 environment: -ETCD_AUTO_COMPACTION_MODErevision -ETCD_AUTO_COMPACTION_RETENTION1000-ETCD_QUOTA_BACKEND_BYTES4294967296-ETCD_SNAPSHOT_COUNT50000volumes: -${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd ports: -2379:2379command: etcd -advertise-client-urlshttp://127.0.0.1:2379 -listen-client-urlshttp://0.0.0.0:2379 --data-dir /etcd healthcheck: test:[CMD,etcdctl,endpoint,health]interval: 5s timeout: 3s retries:10minio: container_name: milvus-minio image: minio/minio:RELEASE.2023-03-20T20-16-18Z environment: MINIO_ACCESS_KEY: minioadmin MINIO_SECRET_KEY: minioadmin ports: -9001:9001-9000:9000volumes: -${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data command: minio server /minio_data --console-address:9001healthcheck: test:[CMD,curl,-f,http://localhost:9000/minio/health/live]interval: 5s timeout: 3s retries:10standalone: container_name: milvus-standalone image: milvusdb/milvus:v2.3.12 command:[milvus,run,standalone]security_opt: - seccomp:unconfined environment: ETCD_ENDPOINTS: etcd:2379 MINIO_ADDRESS: minio:9000 volumes: -${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus healthcheck: test:[CMD,curl,-f,http://localhost:9091/healthz]interval: 30s start_period: 90s timeout: 20s retries:3ports: -19530:19530-9091:9091depends_on: - etcd - minio attu: image: zilliz/attu:v2.3.8 container_name: milvus-attu environment: MILVUS_URL: standalone:19530 ports: -8000:3000depends_on: - standalone networks: default: name: milvus启动docker-composeup-d-d代表后台启动 其他命令docker-composeps查看容器docker-composestop 停止容器docker-composestart 启动容器docker-composedown 停止并删除容器(特别注意以免误删容器)启动attu可视化dockerrun-d\--nameattu\-p8000:3000\-eMILVUS_URL101.126.141.93:19530\zilliz/attu:v2.3.9 http://101.126.141.93:8000/整合springbootcrudhttps://gitee.com/zhang-hai-fei/ollama-deep-seek-milvus-test.gitpomdependencygroupIdio.milvus/groupIdartifactIdmilvus-sdk-java/artifactIdversion2.5.4/version/dependencyconfigpackageorg.caizhi.corporation.config;importio.milvus.v2.client.ConnectConfig;importio.milvus.v2.client.MilvusClientV2;importorg.springframework.beans.factory.annotation.Value;importorg.springframework.context.annotation.Bean;importorg.springframework.context.annotation.Configuration;ConfigurationpublicclassMilvusConfig{Value(${milvus.host})privateStringhost;Value(${milvus.port})privateIntegerport;Value(${milvus.db})privateStringDATABASE_NAME;// 指定数据库名称Value(${milvus.username})privateStringusername;Value(${milvus.password})privateStringpassword;BeanpublicMilvusClientV2milvusClientV2(){Stringurihttp://host:port;ConnectConfigconnectConfigConnectConfig.builder().uri(uri).username(username).password(password).dbName(DATABASE_NAME).build();returnnewMilvusClientV2(connectConfig);}}yamlserver:port:8080milvus:host:101.126.141.93port:19530db:testpassword:admin123username:adminservicepackageorg.caizhi.corporation.service;importcom.google.gson.Gson;importcom.google.gson.JsonObject;importio.milvus.v2.client.MilvusClientV2;importio.milvus.v2.common.DataType;importio.milvus.v2.common.IndexParam;importio.milvus.v2.service.collection.request.AddFieldReq;importio.milvus.v2.service.collection.request.CreateCollectionReq;importio.milvus.v2.service.vector.request.GetReq;importio.milvus.v2.service.vector.request.InsertReq;importio.milvus.v2.service.vector.request.SearchReq;importio.milvus.v2.service.vector.request.data.FloatVec;importio.milvus.v2.service.vector.response.GetResp;importio.milvus.v2.service.vector.response.InsertResp;importio.milvus.v2.service.vector.response.SearchResp;importorg.caizhi.corporation.entity.TestRecord;importorg.slf4j.Logger;importorg.slf4j.LoggerFactory;importorg.springframework.stereotype.Component;importjava.util.ArrayList;importjava.util.Collections;importjava.util.List;ComponentpublicclassMilvusDemoService{privatestaticfinalLoggerlogLoggerFactory.getLogger(MilvusDemoService.class);//类似于mysql中的表定义一个名称为collection_01的集合privatestaticfinalStringCOLLECTION_NAMEcollection_test;//为了测试验证方便向量维度定义2privatestaticfinalintVECTOR_DIM2;privatefinalMilvusClientV2client;publicMilvusDemoService(MilvusClientV2client){this.clientclient;}/** * 创建一个Collection */publicvoidcreateCollection(){CreateCollectionReq.CollectionSchemaschemaclient.createSchema();schema.addField(AddFieldReq.builder().fieldName(id).dataType(DataType.VarChar).isPrimaryKey(true).autoID(false).build());schema.addField(AddFieldReq.builder().fieldName(title).dataType(DataType.VarChar).maxLength(10000).build());schema.addField(AddFieldReq.builder().fieldName(title_vector).dataType(DataType.FloatVector).dimension(VECTOR_DIM).build());IndexParamindexParamIndexParam.builder().fieldName(title_vector).metricType(IndexParam.MetricType.COSINE).build();CreateCollectionReqcreateCollectionReqCreateCollectionReq.builder().collectionName(COLLECTION_NAME).collectionSchema(schema).indexParams(Collections.singletonList(indexParam)).build();client.createCollection(createCollectionReq);}/** * 往collection中插入一条数据 */publicvoidinsertRecord(TestRecordrecord){JsonObjectvectornewJsonObject();vector.addProperty(id,record.getId());vector.addProperty(title,record.getTitle());ListFloatvectorListnewArrayList();//为了模拟测试向量写死2个vectorList.add(2.8f);vectorList.add(3.9f);GsongsonnewGson();vector.add(title_vector,gson.toJsonTree(vectorList));InsertReqinsertReqInsertReq.builder().collectionName(COLLECTION_NAME).data(Collections.singletonList(vector)).build();InsertResprespclient.insert(insertReq);}/** * 通过ID获取记录 */publicGetRespgetRecord(Stringid){GetReqgetReqGetReq.builder().collectionName(COLLECTION_NAME).ids(Collections.singletonList(id)).build();GetResprespclient.get(getReq);returnresp;}/** * 按照向量检索找到相似度最近的topK */publicListListSearchResp.SearchResultqueryVector(){SearchRespsearchRclient.search(SearchReq.builder().collectionName(COLLECTION_NAME).data(Collections.singletonList(newFloatVec(newfloat[]{0.9f,2.1f}))).topK(3).outputFields(Collections.singletonList(*)).build());ListListSearchResp.SearchResultsearchResultssearchR.getSearchResults();for(ListSearchResp.SearchResultresults:searchResults){for(SearchResp.SearchResultresult:results){log.info(ID(String)result.getId(),Scoreresult.getScore(),Resultresult.getEntity().toString());}}returnsearchResults;}}controllerpackageorg.caizhi.corporation.controller;importio.milvus.v2.service.vector.response.GetResp;importio.milvus.v2.service.vector.response.SearchResp;importorg.caizhi.corporation.entity.TestRecord;importorg.caizhi.corporation.service.MilvusDemoService;importorg.slf4j.Logger;importorg.slf4j.LoggerFactory;importorg.springframework.beans.factory.annotation.Autowired;importorg.springframework.web.bind.annotation.GetMapping;importorg.springframework.web.bind.annotation.RequestMapping;importorg.springframework.web.bind.annotation.RequestParam;importorg.springframework.web.bind.annotation.RestController;importjava.io.IOException;importjava.util.List;RestControllerRequestMapping(/milvus)publicclassMilvusController{privatestaticfinalLoggerlogLoggerFactory.getLogger(MilvusController.class);AutowiredprivateMilvusDemoServicemilvusDemoService;GetMapping(/createCollection)publicvoidcreateCollection(){milvusDemoService.createCollection();}GetMapping(/insertRecord)publicvoidinsertRecord()throwsIOException{TestRecordrecordnewTestRecord();record.setId(6);record.setTitle(上海不是中国的首都人口有3000多万人);milvusDemoService.insertRecord(record);}GetMapping(/getRecord)publicGetRespgetRecord(RequestParam(nameid)Stringid){GetResprespmilvusDemoService.getRecord(id);log.info(resp resp.getResults);returnresp;}GetMapping(/queryVector)publicListListSearchResp.SearchResultqueryVector(){ListListSearchResp.SearchResultsearchResultsmilvusDemoService.queryVector();returnsearchResults;}}