Skip to content

Commit

Permalink
Refactoring code base (#134)
Browse files Browse the repository at this point in the history
Refactoring code base
  • Loading branch information
shanghaikid authored Apr 14, 2023
1 parent f417413 commit 4193ec5
Show file tree
Hide file tree
Showing 48 changed files with 1,609 additions and 1,583 deletions.
25 changes: 4 additions & 21 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,32 +29,15 @@ The following collection shows Milvus versions and recommended @zilliz/milvus2-s
npm install @zilliz/milvus2-sdk-node
```

## SDK

- [Client](./milvus/Index.ts)
- [Collection](./milvus/Collection.ts)
- [Index](./milvus/MilvusIndex.ts)
- [FLAT](https://github.com/milvus-io/milvus-sdk-node/blob/main/test/Index.spec.ts#L63)
- [IVF_FLAT](https://github.com/milvus-io/milvus-sdk-node/blob/main/test/Index.spec.ts#L76)
- [IVF_SQ8](https://github.com/milvus-io/milvus-sdk-node/blob/main/test/Index.spec.ts#L91)
- [IVF_PQ](https://github.com/milvus-io/milvus-sdk-node/blob/main/test/Index.spec.ts#L106)
- [HNSW](https://github.com/milvus-io/milvus-sdk-node/blob/main/test/Index.spec.ts#L121)
- [ANNOY](https://github.com/milvus-io/milvus-sdk-node/blob/main/test/Index.spec.ts#L136)
- DISKANN (not supported yet)
- [Partition](./milvus/Partition.ts)
- [User](./milvus/User.ts)
- [Utils](./milvus/Utils.ts)
- [Data](./milvus/Data.ts)

More documentation, you can refer [Milvus offical website](https://milvus.io/).

## Example

1. [Hello World](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/HelloMilvus.ts)
2. [How to operate collection](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/Collection.ts)
3. [How to insert data](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/Insert.ts)
4. [Vector similarity search on float field](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/Search.ts)
5. [Vector similarity search on binary field](https://github.com/milvus-io/milvus-sdk-node/blob/main/example/BinarySearch.ts)
2. [Create a collection](https://milvus.io/docs/create_collection.md)
3. [Insert data](https://milvus.io/docs/insert_data.md)
4. [Build index](https://milvus.io/docs/build_index.md)
5. [Do vector search](https://milvus.io/docs/search.md)

## How to contribute

Expand Down
71 changes: 55 additions & 16 deletions example/BinarySearch.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import { MilvusClient } from '../milvus/index';
import { MilvusClient, DataType, InsertReq } from '@zilliz/milvus2-sdk-node';
import { IP } from '../const';
import { DataType } from '../milvus/const/Milvus';
import { InsertReq } from '../milvus/types/Data';
import {
genCollectionParams,
VECTOR_FIELD_NAME,
Expand All @@ -12,29 +10,62 @@ const milvusClient = new MilvusClient(IP);
const COLLECTION_NAME = GENERATE_NAME();

const test = async () => {
let res: any = await milvusClient.collectionManager.createCollection(
genCollectionParams(COLLECTION_NAME, '128', DataType.BinaryVector)
// create new collection
await milvusClient.createCollection(
genCollectionParams(COLLECTION_NAME, '128', DataType.BinaryVector, false)
);
console.log('-----create collection----', res);
// need load collection before search
await milvusClient.collectionManager.loadCollectionSync({
console.info(`collection ${COLLECTION_NAME} created`);

// create index before load
await milvusClient.createIndex({
collection_name: COLLECTION_NAME,
field_name: VECTOR_FIELD_NAME,
extra_params: {
index_type: 'BIN_IVF_FLAT',
metric_type: 'TANIMOTO',
params: JSON.stringify({ nlist: 1024 }),
},
});

console.info(`index created`);

// load collection
await milvusClient.loadCollectionSync({
collection_name: COLLECTION_NAME,
});

console.info(`collection loaded`);

// create schema
const fields = [
{
isVector: true,
dim: 16,
dim: 16, // 128 / 8
name: VECTOR_FIELD_NAME,
},
{
isVector: false,
name: 'age',
},
];
// generate vector data
const vectorsData = generateInsertData(fields, 10);
const params: InsertReq = {
collection_name: COLLECTION_NAME,
fields_data: vectorsData,
};
res = await milvusClient.dataManager.insert(params);
await milvusClient.dataManager.flush({ collection_names: [COLLECTION_NAME] });
const result = await milvusClient.dataManager.search({
// insert data
await milvusClient.insert(params);

console.info(`Vectors inserted`);

// flush data
await milvusClient.flushSync({
collection_names: [COLLECTION_NAME],
});

// execute vector search
const result = await milvusClient.search({
collection_name: COLLECTION_NAME,
// partition_names: [],
expr: '',
Expand All @@ -47,15 +78,23 @@ const test = async () => {
},
vector_type: DataType.BinaryVector,
});
console.log('----search result-----,', result);
const queryRes = await milvusClient.dataManager.query({
console.info(
`Seaching vectors:`,
[4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3]
);
console.info(`Search result-----,`, result);

// query for data based on search result
const queryRes = await milvusClient.query({
collection_name: COLLECTION_NAME,
expr: `age == ${result.results[0].id}`,
output_fields: ['age', 'vector_field'],
});
console.log('----query----', queryRes.data[0].vector_field);
console.info(`Query data: age == ${result.results[0].id}`);
console.info(`Query data returned`, queryRes.data[0].vector_field);

await milvusClient.collectionManager.dropCollection({
// drop collection
await milvusClient.dropCollection({
collection_name: COLLECTION_NAME,
});
};
Expand Down
20 changes: 4 additions & 16 deletions example/Cluster.ts
Original file line number Diff line number Diff line change
@@ -1,27 +1,15 @@
import { MilvusClient } from "../milvus/index";
import { IP } from "../const";
import { MilvusClient } from '@zilliz/milvus2-sdk-node';
import { IP } from '../const';

const milvusClient = new MilvusClient(IP);
const dataManager = milvusClient.dataManager;

const test = async () => {
let res: any = await dataManager.getMetric({
request: { metric_type: "system_info" },
let res: any = await milvusClient.getMetric({
request: { metric_type: 'system_info' },
});
res.response.nodes_info.forEach((v: any) => {
console.log(v.infos);
});

// res = await dataManager.getQuerySegmentInfo({
// collectionName: COLLECTION_NAME,
// });
// console.log("--- seg info ---", res);

// // only work in cluster version
// res = await dataManager.loadBalance({
// src_nodeID: res.infos[0].nodeID,
// });
// console.log("-- load balance ---", res);
};

test();
77 changes: 42 additions & 35 deletions example/Collection.ts
Original file line number Diff line number Diff line change
@@ -1,70 +1,77 @@
import { MilvusClient } from '../milvus/index';
import { MilvusClient } from '@zilliz/milvus2-sdk-node';
import { IP } from '../const';
import { genCollectionParams, GENERATE_NAME } from '../utils/test';
import {
genCollectionParams,
GENERATE_NAME,
VECTOR_FIELD_NAME,
} from '../utils/test';
const milvusClient = new MilvusClient(IP);
const collectionManager = milvusClient.collectionManager;
const COLLECTION_NAME = GENERATE_NAME();

const test = async () => {
const createRes = await collectionManager.createCollection({
// create a new collection with generated parameters
const createRes = await milvusClient.createCollection({
...genCollectionParams(COLLECTION_NAME, '4'),
});
console.log('--- create collection ---', createRes, COLLECTION_NAME);

let res: any = await collectionManager.showCollections();
// show all collections
let res: any = await milvusClient.showCollections();
console.log(res);
await collectionManager.releaseCollection({ collection_name: 'test' });
res = await collectionManager.showCollections({ type: 1 });

// release the collection
await milvusClient.releaseCollection({ collection_name: COLLECTION_NAME });

// show loaded collections
res = await milvusClient.showCollections({ type: 1 });
console.log('----loaded---', res);

res = await collectionManager.hasCollection({
// check if the collection exists
res = await milvusClient.hasCollection({
collection_name: COLLECTION_NAME,
});
console.log(res);
console.log('hasCollection', res);

res = await collectionManager.getCollectionStatistics({
// get statistics for the collection
res = await milvusClient.getCollectionStatistics({
collection_name: COLLECTION_NAME,
});
console.log(res);
console.log('getCollectionStatistics', res);

res = await collectionManager.loadCollectionSync({
// make sure load successful
await milvusClient.createIndex({
collection_name: COLLECTION_NAME,
field_name: VECTOR_FIELD_NAME,
extra_params: {
index_type: 'IVF_FLAT',
metric_type: 'L2',
params: JSON.stringify({ nlist: 1024 }),
},
});
console.log(res);

res = await collectionManager.describeCollection({
// load the collection synchronously
res = await milvusClient.loadCollectionSync({
collection_name: COLLECTION_NAME,
});
console.log(res);
console.log(res.schema.fields);
console.log('loadCollectionSync result', res.schema.fields);

res = await collectionManager.compact({
// describe the collection
res = await milvusClient.describeCollection({
collection_name: COLLECTION_NAME,
});
console.log('--- compact ---', res);
console.log('describeCollection result', res.schema.fields);

const compactionID = res.compactionID;
res = await collectionManager.getCompactionState({
compactionID: compactionID,
});

console.log('--- compact state ---', res);

res = await collectionManager.getCompactionStateWithPlans({
compactionID: compactionID,
});

console.log('--- compact state with plans---', res);

res = await collectionManager.releaseCollection({
// release the collection
res = await milvusClient.releaseCollection({
collection_name: COLLECTION_NAME,
});
console.log(res);
console.log('releaseCollection result', res);

res = await collectionManager.dropCollection({
// drop the collection
res = await milvusClient.dropCollection({
collection_name: COLLECTION_NAME,
});
console.log('delete---', res);
console.log('dropCollection result', res);
};

test();
31 changes: 14 additions & 17 deletions example/HelloMilvus.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
import { MilvusClient } from '@zilliz/milvus2-sdk-node';
import { DataType } from '@zilliz/milvus2-sdk-node/dist/milvus/const/Milvus';
import { InsertReq } from '@zilliz/milvus2-sdk-node/dist/milvus/types/Data';
import { MilvusClient, DataType, InsertReq } from '@zilliz/milvus2-sdk-node';

const milvusClient = new MilvusClient('localhost:19530');
const collectionManager = milvusClient.collectionManager;

const generateInsertData = function generateInsertData(
fields: { isVector: boolean; dim?: number; name: string; isBool?: boolean }[],
Expand All @@ -18,8 +15,8 @@ const generateInsertData = function generateInsertData(
value[name] = isVector
? [...Array(dim)].map(() => Math.random() * 10)
: isBool
? count % 2 === 0
: count;
? count % 2 === 0
: count;
});

value['count'] = count;
Expand All @@ -30,11 +27,11 @@ const generateInsertData = function generateInsertData(
};

const hello_milvus = async () => {
const checkVersion = await milvusClient.checkVersion();
const checkVersion = await milvusClient.getVersion();
console.log('--- check version ---', checkVersion);
const collectionName = 'hello_milvus';
const dim = '4';
const createRes = await collectionManager.createCollection({
const createRes = await milvusClient.createCollection({
collection_name: collectionName,
fields: [
{
Expand Down Expand Up @@ -63,12 +60,12 @@ const hello_milvus = async () => {
});
console.log('--- Create collection ---', createRes, collectionName);

const showCollectionRes = await collectionManager.showCollections();
const showCollectionRes = await milvusClient.showCollections();
console.log('--- Show collections ---', showCollectionRes);

const hasCollectionRes = await collectionManager.hasCollection({
const hasCollectionRes = await milvusClient.hasCollection({
collection_name: collectionName,
});
});
console.log(
'--- Has collection (' + collectionName + ') ---',
hasCollectionRes
Expand All @@ -91,10 +88,10 @@ const hello_milvus = async () => {
collection_name: collectionName,
fields_data: vectorsData,
};
await milvusClient.dataManager.insert(params);
await milvusClient.insert(params);
console.log('--- Insert Data to Collection ---');

await milvusClient.indexManager.createIndex({
await milvusClient.createIndex({
collection_name: collectionName,
field_name: 'float_vector',
extra_params: {
Expand All @@ -106,15 +103,15 @@ const hello_milvus = async () => {
console.log('--- Create Index in Collection ---');

// need load collection before search
const loadCollectionRes = await collectionManager.loadCollectionSync({
const loadCollectionRes = await milvusClient.loadCollectionSync({
collection_name: collectionName,
});
console.log(
'--- Load collection (' + collectionName + ') ---',
loadCollectionRes
);

const result = await milvusClient.dataManager.search({
const result = await milvusClient.search({
collection_name: collectionName,
vectors: [vectorsData[0]['float_vector']],
search_params: {
Expand All @@ -129,12 +126,12 @@ const hello_milvus = async () => {
});
console.log('--- Search collection (' + collectionName + ') ---', result);

const releaseRes = await collectionManager.releaseCollection({
const releaseRes = await milvusClient.releaseCollection({
collection_name: collectionName,
});
console.log('--- Release Collection ---', releaseRes);

const dropRes = await collectionManager.dropCollection({
const dropRes = await milvusClient.dropCollection({
collection_name: collectionName,
});
console.log('--- Drop Collection ---', dropRes);
Expand Down
Loading

0 comments on commit 4193ec5

Please sign in to comment.