From 95eaa1dedfa2eff1ccb69a4e7ef0242f33817792 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 21 Mar 2024 18:49:21 +0900 Subject: [PATCH 01/14] implement test --- rust/Cargo.toml | 2 +- rust/libs/ngt-rs/Cargo.toml | 15 ++++ rust/libs/ngt-rs/build.rs | 18 +++++ rust/libs/ngt-rs/src/input.cpp | 122 +++++++++++++++++++++++++++++ rust/libs/ngt-rs/src/input.h | 40 ++++++++++ rust/libs/ngt-rs/src/lib.rs | 136 +++++++++++++++++++++++++++++++++ 6 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 rust/libs/ngt-rs/Cargo.toml create mode 100644 rust/libs/ngt-rs/build.rs create mode 100644 rust/libs/ngt-rs/src/input.cpp create mode 100644 rust/libs/ngt-rs/src/input.h create mode 100644 rust/libs/ngt-rs/src/lib.rs diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 082e358313..666259b2b2 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -14,4 +14,4 @@ # limitations under the License. # [workspace] -members = ["libs/proto", "libs/ngt", "bin/agent"] +members = ["libs/proto", "libs/ngt", "libs/ngt-rs", "bin/agent"] diff --git a/rust/libs/ngt-rs/Cargo.toml b/rust/libs/ngt-rs/Cargo.toml new file mode 100644 index 0000000000..7527ffe330 --- /dev/null +++ b/rust/libs/ngt-rs/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "ngt-rs" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.81" +cxx = { version = "1.0.117", features = ["c++20"] } + +[build-dependencies] +cxx-build = "1.0.117" +miette = { version = "7.1.0", features = ["fancy"] } + +[dev-dependencies] +rand = "0.8.5" diff --git a/rust/libs/ngt-rs/build.rs b/rust/libs/ngt-rs/build.rs new file mode 100644 index 0000000000..0b2349617c --- /dev/null +++ b/rust/libs/ngt-rs/build.rs @@ -0,0 +1,18 @@ +fn main() -> miette::Result<()> { + let current_dir = std::env::current_dir().unwrap(); + println!("cargo:rustc-link-search=native={}", current_dir.display()); + + cxx_build::bridge("src/lib.rs") + .file("src/input.cpp") + .flag_if_supported("-std=c++20") + .flag_if_supported("-fopenmp") + .flag_if_supported("-DNGT_BFLOAT_DISABLED") + .compile("ngt-rs"); + + println!("cargo:rustc-link-search=native=/usr/local/lib"); + println!("cargo:rustc-link-lib=static=ngt"); + println!("cargo:rustc-link-lib=dylib=gomp"); + println!("cargo:rerun-if-changed=src/*"); + + Ok(()) +} diff --git a/rust/libs/ngt-rs/src/input.cpp b/rust/libs/ngt-rs/src/input.cpp new file mode 100644 index 0000000000..2529437d72 --- /dev/null +++ b/rust/libs/ngt-rs/src/input.cpp @@ -0,0 +1,122 @@ +#include +#include "ngt-rs/src/input.h" +#include "ngt-rs/src/lib.rs.h" + +Property::Property(): p() {} + +void Property::set_dimension(rust::i32 dimension) { + p.dimension = dimension; +} + +rust::i32 Property::get_dimension() { + return p.dimension; +} + +void Property::set_distance_type(const DistanceType t) { + p.distanceType = static_cast(t); +} + +void Property::set_object_type(const ObjectType t) { + p.objectType = static_cast(t); +} + +void Property::set_edge_size_for_creation(rust::i16 edge_size) { + p.edgeSizeForCreation = edge_size; +} + +void Property::set_edge_size_for_search(rust::i16 edge_size) { + p.edgeSizeForSearch = edge_size; +} + +Index::Index(const rust::String& path, bool read_only): index(new NGT::Index(std::string(path), read_only)) {} + +Index::Index(const rust::String& path, Property& p) { + std::string cpath(path); + NGT::Index::createGraphAndTree(cpath, p.p, true); + index = new NGT::Index(cpath); +} + +Index::Index(Property& p): index(new NGT::GraphAndTreeIndex(p.p)) {} + +void Index::search(rust::Slice v, rust::i32 size, rust::f32 epsilon, rust::f32 radius, rust::i32 edge_size, rust::i32 *ids, rust::f32 *distances) { + if (radius < 0.0) { + radius = FLT_MAX; + } + std::vector vquery(v.begin(), v.end()); + NGT::Object* query = index->allocateObject(vquery); + NGT::SearchContainer sc(*query); + NGT::ObjectDistances ngtresults; + sc.setResults(&ngtresults); + sc.setSize(size); + sc.setRadius(radius); + sc.setEpsilon(epsilon); + if (edge_size != INT_MIN) { + sc.setEdgeSize(edge_size); + } + + index->search(sc); + index->deleteObject(query); + + for (int i = 0; i < size; i++) { + ids[i] = ngtresults[i].id; + distances[i] = ngtresults[i].distance; + } +} + +void Index::linear_search(rust::Slice v, rust::i32 size, rust::i32 edge_size, rust::i32 *ids, rust::f32 *distances) { + std::vector vquery(v.begin(), v.end()); + auto query = index->allocateObject(vquery); + NGT::SearchContainer sc(*query); + NGT::ObjectDistances ngtresults; + sc.setResults(&ngtresults); + sc.setSize(size); + if (edge_size != INT_MIN) { + sc.setEdgeSize(edge_size); + } + + index->linearSearch(sc); + index->deleteObject(query); + + for (int i = 0; i < size; i++) { + ids[i] = ngtresults[i].id; + distances[i] = ngtresults[i].distance; + } +} + +rust::u32 Index::insert(rust::Slice v) { + return index->insert(std::vector(v.begin(), v.end())); +} + +void Index::create_index(rust::u32 pool_size) { + index->createIndex(pool_size); +} + +void Index::remove(rust::u32 id) { + index->remove(id); +} + +void Index::save(const rust::String& path) { + index->saveIndex(std::string(path)); +} + +rust::Slice Index::get_vector(rust::u32 id) { + NGT::ObjectSpace& os = index->getObjectSpace(); + rust::f32* v = static_cast(os.getObject(id)); + return rust::Slice(v, os.getDimension()); +} + +std::unique_ptr new_property() { + return std::make_unique(); +} + +std::unique_ptr open_index(const rust::String& path, bool read_only) { + return std::make_unique(path, read_only); +} + +std::unique_ptr new_index(const rust::String& path, Property& p) { + return std::make_unique(path, p); +} + +std::unique_ptr new_index_in_memory(Property& p) { + return std::make_unique(p); +} diff --git a/rust/libs/ngt-rs/src/input.h b/rust/libs/ngt-rs/src/input.h new file mode 100644 index 0000000000..6d9ec945f5 --- /dev/null +++ b/rust/libs/ngt-rs/src/input.h @@ -0,0 +1,40 @@ +#pragma once + +#include "NGT/Index.h" +#include +#include "rust/cxx.h" + +enum class DistanceType; +enum class ObjectType; + +class Property { +public: + NGT::Property p; + Property(); + void set_dimension(rust::i32); + rust::i32 get_dimension(); + void set_object_type(const ObjectType); + void set_distance_type(const DistanceType); + void set_edge_size_for_creation(rust::i16); + void set_edge_size_for_search(rust::i16); +}; + +class Index { + NGT::Index* index; +public: + Index(const rust::String&, bool); + Index(const rust::String&, Property&); + Index(Property&); + void search(rust::Slice, rust::i32, rust::f32, rust::f32, rust::i32, rust::i32*, rust::f32*); + void linear_search(rust::Slice, rust::i32, rust::i32, rust::i32*, rust::f32*); + rust::u32 insert(rust::Slice); + void create_index(rust::u32); + void remove(rust::u32); + void save(const rust::String&); + rust::Slice get_vector(rust::u32); +}; + +std::unique_ptr new_property(); +std::unique_ptr open_index(const rust::String&, bool); +std::unique_ptr new_index(const rust::String&, Property&); +std::unique_ptr new_index_in_memory(Property&); diff --git a/rust/libs/ngt-rs/src/lib.rs b/rust/libs/ngt-rs/src/lib.rs new file mode 100644 index 0000000000..654b62efcb --- /dev/null +++ b/rust/libs/ngt-rs/src/lib.rs @@ -0,0 +1,136 @@ +#[cxx::bridge] +pub mod ffi { + #[repr(i32)] + enum ObjectType { + None = 0, + Uint8 = 1, + Float = 2, + Float16 = 3, + } + + #[repr(i32)] + enum DistanceType { + None = -1, + L1 = 0, + L2 = 1, + Hamming = 2, + Angle = 3, + Cosine = 4, + NormalizedAngle = 5, + NormalizedCosine = 6, + Jaccard = 7, + SparseJaccard = 8, + NormalizedL2 = 9, + InnerProduct = 10, + Poincare = 100, + Lorentz = 101, + } + + unsafe extern "C++" { + include!("ngt-rs/src/input.h"); + + type Property; + fn new_property() -> UniquePtr; + fn set_dimension(self: Pin<&mut Property>, dimension: i32); + fn set_object_type(self: Pin<&mut Property>, t: ObjectType); + fn set_distance_type(self: Pin<&mut Property>, t: DistanceType); + + type Index; + fn open_index(path: &String, read_only: bool) -> Result>; + fn new_index(path: &String, p: Pin<&mut Property>) -> Result>; + fn new_index_in_memory(p: Pin<&mut Property>) -> Result>; + unsafe fn search( + self: Pin<&mut Index>, + v: &[f32], + size: i32, + epsilon: f32, + radius: f32, + edge_size: i32, + ids: *mut i32, + distances: *mut f32, + ) -> Result<()>; + unsafe fn linear_search( + self: Pin<&mut Index>, + v: &[f32], + size: i32, + edge_size: i32, + ids: *mut i32, + distances: *mut f32, + ) -> Result<()>; + fn insert(self: Pin<&mut Index>, v: &[f32]) -> Result; + fn create_index(self: Pin<&mut Index>, pool_size: u32) -> Result<()>; + fn remove(self: Pin<&mut Index>, id: u32) -> Result<()>; + fn get_vector(self: Pin<&mut Index>, id: u32) -> Result<&[f32]>; + } +} + +#[cfg(test)] +mod tests { + use std::vec; + + use self::ffi::new_index_in_memory; + use rand::distributions::Standard; + use rand::prelude::*; + + use super::*; + + const DIMENSION: i32 = 128; + const COUNT: u32 = 1000; + const K: usize = 10; + + fn gen_random_vector(dim: i32) -> Vec { + (0..dim) + .map(|_| StdRng::from_entropy().sample(Standard)) + .collect() + } + + #[test] + fn test_ngt() -> Result<()> { + let mut p = ffi::new_property(); + p.pin_mut().set_dimension(DIMENSION); + p.pin_mut().set_distance_type(ffi::DistanceType::L2); + p.pin_mut().set_object_type(ffi::ObjectType::Float); + + let mut index = new_index_in_memory(p.pin_mut())?; + let vectors: Vec> = (0..COUNT).map(|_| gen_random_vector(DIMENSION)).collect(); + for (i, v) in vectors.iter().enumerate() { + let id = index.pin_mut().insert(v.as_slice())?; + assert_eq!(i + 1, id as usize); + } + index.pin_mut().create_index(4)?; + + for i in 0..COUNT { + let mut ids: Vec = vec![-1; K]; + let mut distances: Vec = vec![-1.0; K]; + unsafe { + index.pin_mut().search( + gen_random_vector(DIMENSION).as_slice(), + K as i32, + 0.05, + -1.0, + i32::MIN, + &mut ids[0] as *mut i32, + &mut distances[0] as *mut f32, + )? + }; + for j in 0..K { + assert!( + 1 <= ids[j] && ids[j] <= COUNT as i32, + "invalid id {}", + ids[j] + ); + assert!(distances[j] >= 0.0, "invalid distance {}", distances[j]); + } + } + + for (i, v) in vectors.iter().enumerate() { + let ret = index.pin_mut().get_vector((i + 1) as u32)?; + assert_eq!(v.as_slice(), ret); + } + + for i in 1..COUNT + 1 { + index.pin_mut().remove(i)?; + } + Ok(()) + } +} From 0fe948ba32ded95c492dcb96c16826ac6b86a5c1 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Mon, 15 Apr 2024 11:41:55 +0900 Subject: [PATCH 02/14] update rust dependencies --- rust/Cargo.lock | 396 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 345 insertions(+), 51 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 473b9bb665..e04edfadb7 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -23,9 +23,9 @@ version = "0.1.0" [[package]] name = "anyhow" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" [[package]] name = "async-stream" @@ -51,9 +51,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.79" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", @@ -74,7 +74,7 @@ checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", - "bitflags", + "bitflags 1.3.2", "bytes", "futures-util", "http", @@ -126,6 +126,15 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + [[package]] name = "base64" version = "0.21.7" @@ -138,6 +147,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + [[package]] name = "bytes" version = "1.6.0" @@ -146,9 +161,9 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.90" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" [[package]] name = "cfg-if" @@ -156,11 +171,65 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "cxx" +version = "1.0.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db378d04296a84d8b7d047c36bb3954f0b46529db725d7e62fb02f9ba53ccc" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e5262a7fa3f0bae2a55b767c223ba98032d7c328f5c13fa5cdc980b77fc0658" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8dcadd2e2fb4a501e1d9e93d6e88e6ea494306d8272069c92d5a9edf8855c0" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad08a837629ad949b73d032c637653d069e909cffe4ee7870b02301939ce39cc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" [[package]] name = "equivalent" @@ -168,6 +237,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "fnv" version = "1.0.7" @@ -215,9 +294,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "libc", @@ -351,11 +430,17 @@ dependencies = [ "hashbrown 0.14.3", ] +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "itertools" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -372,6 +457,21 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "link-cplusplus" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +dependencies = [ + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + [[package]] name = "matchit" version = "0.7.3" @@ -384,6 +484,37 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +[[package]] +name = "miette" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" +dependencies = [ + "backtrace", + "backtrace-ext", + "cfg-if", + "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf09caffaac8068c346b6df2a7fc27a177fd20b39421a39ce0a211bde679a6c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "mime" version = "0.3.17" @@ -414,6 +545,17 @@ dependencies = [ name = "ngt" version = "0.1.0" +[[package]] +name = "ngt-rs" +version = "0.1.0" +dependencies = [ + "anyhow", + "cxx", + "cxx-build", + "miette", + "rand", +] + [[package]] name = "object" version = "0.32.2" @@ -429,6 +571,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "owo-colors" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -475,18 +623,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "a56dea16b0a29e94408b9aa5e2940a4eedbd128a1ba20e8f7ae60fd3d465af0e" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" +checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" dependencies = [ "bytes", "prost-derive", @@ -494,9 +642,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" dependencies = [ "anyhow", "itertools", @@ -507,9 +655,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" +checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" dependencies = [ "prost", ] @@ -526,9 +674,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -569,11 +717,30 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustix" +version = "0.38.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" + +[[package]] +name = "scratch" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" [[package]] name = "serde" @@ -604,6 +771,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + [[package]] name = "socket2" version = "0.5.6" @@ -614,11 +787,32 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "supports-color" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9829b314621dfc575df4e409e79f9d6a66a3bd707ab73f23cb4aa3a854ac854f" +dependencies = [ + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c0a1e5168041f5f3ff68ff7d95dcb9c8749df29f6e7e89ada40dd4c9de404ee" + +[[package]] +name = "supports-unicode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" + [[package]] name = "syn" -version = "2.0.58" +version = "2.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" +checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a" dependencies = [ "proc-macro2", "quote", @@ -631,6 +825,56 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "terminal_size" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio" version = "1.37.0" @@ -794,6 +1038,18 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "want" version = "0.3.1" @@ -809,6 +1065,37 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.48.0" @@ -824,7 +1111,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -844,17 +1131,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -865,9 +1153,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -877,9 +1165,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -889,9 +1177,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -901,9 +1195,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -913,9 +1207,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -925,9 +1219,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -937,6 +1231,6 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" From 0b0cb202c9947d1ec517e28db124d1ccda8ae0df Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Mon, 15 Apr 2024 13:28:40 +0900 Subject: [PATCH 03/14] fix --- rust/libs/ngt-rs/src/lib.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rust/libs/ngt-rs/src/lib.rs b/rust/libs/ngt-rs/src/lib.rs index 654b62efcb..7ad13a3085 100644 --- a/rust/libs/ngt-rs/src/lib.rs +++ b/rust/libs/ngt-rs/src/lib.rs @@ -68,7 +68,7 @@ pub mod ffi { mod tests { use std::vec; - use self::ffi::new_index_in_memory; + use anyhow::Result; use rand::distributions::Standard; use rand::prelude::*; @@ -91,7 +91,7 @@ mod tests { p.pin_mut().set_distance_type(ffi::DistanceType::L2); p.pin_mut().set_object_type(ffi::ObjectType::Float); - let mut index = new_index_in_memory(p.pin_mut())?; + let mut index = ffi::new_index_in_memory(p.pin_mut())?; let vectors: Vec> = (0..COUNT).map(|_| gen_random_vector(DIMENSION)).collect(); for (i, v) in vectors.iter().enumerate() { let id = index.pin_mut().insert(v.as_slice())?; @@ -99,7 +99,7 @@ mod tests { } index.pin_mut().create_index(4)?; - for i in 0..COUNT { + for _ in 0..COUNT { let mut ids: Vec = vec![-1; K]; let mut distances: Vec = vec![-1.0; K]; unsafe { @@ -113,13 +113,13 @@ mod tests { &mut distances[0] as *mut f32, )? }; - for j in 0..K { + for i in 0..K { assert!( - 1 <= ids[j] && ids[j] <= COUNT as i32, + 1 <= ids[i] && ids[i] <= COUNT as i32, "invalid id {}", - ids[j] + ids[i] ); - assert!(distances[j] >= 0.0, "invalid distance {}", distances[j]); + assert!(distances[i] >= 0.0, "invalid distance {}", distances[i]); } } From f24c989a07d685bd5b4592b004181868ea7599dc Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Tue, 16 Apr 2024 09:27:09 +0900 Subject: [PATCH 04/14] make format --- rust/libs/ngt-rs/Cargo.toml | 15 +++++++++++++++ rust/libs/ngt-rs/build.rs | 15 +++++++++++++++ rust/libs/ngt-rs/src/input.cpp | 15 +++++++++++++++ rust/libs/ngt-rs/src/input.h | 15 +++++++++++++++ rust/libs/ngt-rs/src/lib.rs | 15 +++++++++++++++ 5 files changed, 75 insertions(+) diff --git a/rust/libs/ngt-rs/Cargo.toml b/rust/libs/ngt-rs/Cargo.toml index 7527ffe330..7caa2b26a0 100644 --- a/rust/libs/ngt-rs/Cargo.toml +++ b/rust/libs/ngt-rs/Cargo.toml @@ -1,3 +1,18 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# [package] name = "ngt-rs" version = "0.1.0" diff --git a/rust/libs/ngt-rs/build.rs b/rust/libs/ngt-rs/build.rs index 0b2349617c..15e18a7a51 100644 --- a/rust/libs/ngt-rs/build.rs +++ b/rust/libs/ngt-rs/build.rs @@ -1,3 +1,18 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// fn main() -> miette::Result<()> { let current_dir = std::env::current_dir().unwrap(); println!("cargo:rustc-link-search=native={}", current_dir.display()); diff --git a/rust/libs/ngt-rs/src/input.cpp b/rust/libs/ngt-rs/src/input.cpp index 2529437d72..7e35fbd91d 100644 --- a/rust/libs/ngt-rs/src/input.cpp +++ b/rust/libs/ngt-rs/src/input.cpp @@ -1,3 +1,18 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include #include "ngt-rs/src/input.h" #include "ngt-rs/src/lib.rs.h" diff --git a/rust/libs/ngt-rs/src/input.h b/rust/libs/ngt-rs/src/input.h index 6d9ec945f5..46c061a65c 100644 --- a/rust/libs/ngt-rs/src/input.h +++ b/rust/libs/ngt-rs/src/input.h @@ -1,3 +1,18 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #pragma once #include "NGT/Index.h" diff --git a/rust/libs/ngt-rs/src/lib.rs b/rust/libs/ngt-rs/src/lib.rs index 7ad13a3085..951faff333 100644 --- a/rust/libs/ngt-rs/src/lib.rs +++ b/rust/libs/ngt-rs/src/lib.rs @@ -1,3 +1,18 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #[cxx::bridge] pub mod ffi { #[repr(i32)] From 2e95de4f2b3333eda5bdd71116fb1dbbbfe8b015 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 13:27:06 +0900 Subject: [PATCH 05/14] add Github Actions workflows --- .github/workflows/dockers-agent-image.yml | 74 ++++++++++++++++++ Makefile | 1 + Makefile.d/build.mk | 6 ++ Makefile.d/docker.mk | 13 ++++ dockers/agent/core/agent/Dockerfile | 93 +++++++++++++++++++++++ dockers/agent/core/agent/README.md | 89 ++++++++++++++++++++++ rust/Cargo.lock | 6 ++ rust/bin/agent/Cargo.toml | 1 + rust/libs/ngt/Cargo.toml | 1 + 9 files changed, 284 insertions(+) create mode 100644 .github/workflows/dockers-agent-image.yml create mode 100644 dockers/agent/core/agent/Dockerfile create mode 100644 dockers/agent/core/agent/README.md diff --git a/.github/workflows/dockers-agent-image.yml b/.github/workflows/dockers-agent-image.yml new file mode 100644 index 0000000000..391372f1ef --- /dev/null +++ b/.github/workflows/dockers-agent-image.yml @@ -0,0 +1,74 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Build docker image: agent-ngt" +on: + push: + branches: + - "main" + - "release/v*.*" + - "!release/v*.*.*" + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + paths: + - ".github/actions/docker-build/action.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-agent-ngt-rust-image.yml" + - "rust/Cargo.lock" + - "rust/Cargo.toml" + - "rust/bin/agent/**" + - "rust/libs/ngt/**" + - "rust/libs/ngt-rs/**" + - "rust/libs/proto/**" + - "dockers/agent/core/ngt-rust/Dockerfile" + - "versions/RUST_VERSION" + - "versions/NGT_VERSION" + pull_request: + paths: + - ".github/actions/docker-build/action.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-agent-image.yml" + - "rust/Cargo.lock" + - "rust/Cargo.toml" + - "rust/bin/agent/**" + - "rust/libs/ngt/**" + - "rust/libs/ngt-rs/**" + - "rust/libs/proto/**" + - "dockers/agent/core/agent/Dockerfile" + - "versions/RUST_VERSION" + - "versions/NGT_VERSION" + pull_request_target: + paths: + - ".github/actions/docker-build/action.yaml" + - ".github/workflows/_docker-image.yaml" + - ".github/workflows/dockers-agent-image.yml" + - "rust/Cargo.lock" + - "rust/Cargo.toml" + - "rust/bin/agent/**" + - "rust/libs/ngt/**" + - "rust/libs/ngt-rs/**" + - "rust/libs/proto/**" + - "dockers/agent/core/agent/Dockerfile" + - "versions/RUST_VERSION" + - "versions/NGT_VERSION" +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent-ngt-rust + secrets: inherit diff --git a/Makefile b/Makefile index 5641df39e0..9251b2572b 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ GHCRORG = ghcr.io/$(ORG)/$(NAME) AGENT_NGT_IMAGE = $(NAME)-agent-ngt AGENT_FAISS_IMAGE = $(NAME)-agent-faiss AGENT_SIDECAR_IMAGE = $(NAME)-agent-sidecar +AGENT_IMAGE = $(NAME)-agent-ngt-rust CI_CONTAINER_IMAGE = $(NAME)-ci-container DEV_CONTAINER_IMAGE = $(NAME)-dev-container DISCOVERER_IMAGE = $(NAME)-discoverer-k8s diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index 28c75214d4..17b0a1c11e 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -95,6 +95,12 @@ cmd/tools/benchmark/operator/operator: $(eval CGO_ENABLED = 0) $(call go-build,tools/benchmark/operator,,-static,,,$@) +rust/target/release/agent: + pushd rust && cargo build -p agent --release && popd + +rust/target/debug/agent: + pushd rust && cargo build -p agent && popd + .PHONY: binary/build/zip ## build all binaries and zip them binary/build/zip: \ diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 0f726333e8..a6c20db689 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -108,6 +108,16 @@ docker/build/agent-sidecar: IMAGE=$(AGENT_SIDECAR_IMAGE) \ docker/build/image +.PHONY: docker/name/agent-ngt-rust +docker/name/agent-ngt-rust: + @echo "$(ORG)/$(AGENT_IMAGE)" + +.PHONY: docker/build/agent-ngt-rust +docker/name/agent-ngt-rust: + @make DOCKERFILE="$(ROOTDIR)/dockers/agent/core/agent/Dockerfile" \ + IMAGE=$(AGENT_IMAGE) \ + docker/build/image + .PHONY: docker/name/discoverer-k8s docker/name/discoverer-k8s: @echo "$(ORG)/$(DISCOVERER_IMAGE)" @@ -286,3 +296,6 @@ docker/build/benchmark-operator: @make DOCKERFILE="$(ROOTDIR)/dockers/tools/benchmark/operator/Dockerfile" \ IMAGE=$(BENCHMARK_OPERATOR_IMAGE) \ docker/build/image + +.PHONY: docker/build/agent-ngt-rust + @make DOCKER \ No newline at end of file diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile new file mode 100644 index 0000000000..d1afbf4b68 --- /dev/null +++ b/dockers/agent/core/agent/Dockerfile @@ -0,0 +1,93 @@ +# syntax = docker/dockerfile:latest +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG DISTROLESS_IMAGE=gcr.io/distroless/cc-debian12 +ARG DISTROLESS_IMAGE_TAG=nonroot +ARG MAINTAINER="vdaas.org vald team " + +FROM --platform=${TARGETPLATFORM} ubuntu:devel AS builder + +ARG TARGETARCH +ARG TARGETOS +ARG RUST_VERSION + +ENV RUST_HOME /usr/local/lib/rust +ENV RUSTUP_HOME ${RUST_HOME}/rustup +ENV CARGO_HOME ${RUST_HOME}/cargo +ENV DEBIAN_FRONTEND noninteractive +ENV INITRD No +ENV LANG en_US.UTF-8 +ENV PATH ${PATH}:${RUSTUP_HOME}/bin:${CARGO_HOME}/bin +ENV ORG vdaas +ENV REPO vald +ENV PKG agent/core/ngt +ENV APP_NAME ngt + +# skipcq: DOK-DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + g++ \ + gcc \ + git \ + liblapack-dev \ + libomp-dev \ + libopenblas-dev \ + unzip \ + upx \ + && ldconfig \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d +COPY Makefile.d . +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +COPY Makefile . +COPY .git . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd +COPY cmd . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions +COPY versions . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/rust +COPY rust . + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +RUN make ngt/install +RUN make rust/install + +RUN make rust/target/release/agent \ + && mv ./rust/target/release/agent "/usr/bin/${APP_NAME}" + +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG} +RUN cp sample.yaml /tmp/config.yaml + +FROM --platform=${TARGETPLATFORM} ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG} +LABEL maintainer="${MAINTAINER}" + +ENV APP_NAME ngt + +COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME} +COPY --from=builder /tmp/config.yaml /etc/server/config.yaml + +USER nonroot:nonroot + +ENTRYPOINT ["/usr/bin/ngt"] diff --git a/dockers/agent/core/agent/README.md b/dockers/agent/core/agent/README.md new file mode 100644 index 0000000000..288646f27e --- /dev/null +++ b/dockers/agent/core/agent/README.md @@ -0,0 +1,89 @@ +# Vald Agent + + + +⚠ This image is under developing and runs incorrectly. + +`vald-agent` is the docker image for the vald-agent component. + +This image is responsible for the following: + +- Store index data along with the user request. + - The store destination is In-Memory, Volume Mounts, Persistent Volume, or External Storage(⚠). +- Search the nearest neighbor vectors of the request vector and return the search result. + +⚠ When you'd like to use the external storage, it requires [vald-agent-sidecar](https://hub.docker.com/r/vdaas/vald-agent-sidecar/tags?page=1&name=latest) on the Kubernetes cluster. + +For more details, please refer to the [component document](https://vald.vdaas.org/docs/overview/component/agent). + +
+ +
+ +[![latest Image](https://img.shields.io/docker/v/vdaas/vald-agent/latest?label=vald-agent)](https://hub.docker.com/r/vdaas/vald-agent/tags?page=1&name=latest) +[![License: Apache 2.0](https://img.shields.io/github/license/vdaas/vald.svg?style=flat-square)](https://opensource.org/licenses/Apache-2.0) +[![latest ver.](https://img.shields.io/github/release/vdaas/vald.svg?style=flat-square)](https://github.com/vdaas/vald/releases/latest) +[![Twitter](https://img.shields.io/badge/twitter-follow-blue?logo=twitter&style=flat-square)](https://twitter.com/vdaas_vald) + +## Requirement + + + +### linux/amd64 + +- CPU instruction: requires `AVX2` or `AVX512` + +### linux/arm64 + +⚠ This image does NOT support running on M1/M2 Mac. + +## Get Started + + + + +You can use `vald-agent` in 3 ways. + +1. One of the components of the Vald cluster + - Refer to [Get Started](https://vald.vdaas.org/docs/tutotial/get-started/). +1. Standalone on the Kubernetes cluster + - Refer to [Vald Agent Standalone on Kubernetes](https://vald.vdaas.org/docs/tutorial/vald-agent-standalone-on-k8s/) +1. Standalone on Docker + - Refer to [Vald Agent Standalone on Docker](https://vald.vdaas.org/docs/tutorial/vald-agent-standalone-on-docker/) + +## Versions + +| tag | linux/amd64 | linux/arm64 | description | +| :------ | :---------: | :---------: | :------------------------------------------------------------------------------------------------------------------------------ | +| latest | ✅ | ✅ | the latest image is the same as the latest version of [vdaas/vald](https://github.com/vdaas/vald) repository version. | +| nightly | ✅ | ✅ | the nightly applies the main branch's source code of the [vdaas/vald](https://github.com/vdaas/vald) repository. | +| vX.Y.Z | ✅ | ✅ | the vX.Y.Z image applies the source code of the [vdaas/vald](https://github.com/vdaas/vald) repository. | +| pr-XXX | ✅ | ❌ | the pr-XXX image applies the source code of the pull request XXX of the [vdaas/vald](https://github.com/vdaas/vald) repository. | + +## Dockerfile + + + +The `Dockerfile` of this image is [here](https://github.com/vdaas/vald/blob/main/dockers/agent/core/agent/Dockerfile). + +## About Vald Project + + + + +The information about the Vald project, please refer to the following: + +- [Official website](https://vald.vdaas.org) +- [GitHub](https://github.com/vdaas/vald) + +## Contacts + +We're love to support you! +Please feel free to contact us anytime with your questions or issue reports. + +- [Official Slack WS](https://join.slack.com/t/vald-community/shared_invite/zt-db2ky9o4-R_9p2sVp8xRwztVa8gfnPA) +- [GitHub Issue](https://github.com/vdaas/vald/issues) + +## License + +This product is under the terms of the Apache License v2.0; refer [LICENSE](https://github.com/vdaas/vald/blob/main/LICENSE) file. diff --git a/rust/Cargo.lock b/rust/Cargo.lock index e04edfadb7..2966c2a092 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -20,6 +20,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "agent" version = "0.1.0" +dependencies = [ + "ngt", +] [[package]] name = "anyhow" @@ -544,6 +547,9 @@ dependencies = [ [[package]] name = "ngt" version = "0.1.0" +dependencies = [ + "ngt-rs", +] [[package]] name = "ngt-rs" diff --git a/rust/bin/agent/Cargo.toml b/rust/bin/agent/Cargo.toml index 26c1fdf548..5fc426d196 100644 --- a/rust/bin/agent/Cargo.toml +++ b/rust/bin/agent/Cargo.toml @@ -21,3 +21,4 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +ngt = { version = "0.1.0", path = "../../libs/ngt" } diff --git a/rust/libs/ngt/Cargo.toml b/rust/libs/ngt/Cargo.toml index cf9c8f82fe..59a5530b8c 100644 --- a/rust/libs/ngt/Cargo.toml +++ b/rust/libs/ngt/Cargo.toml @@ -21,3 +21,4 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +ngt-rs = { version = "0.1.0", path = "../ngt-rs" } From 4ec9b070707c7d39420c5d4ff03c23ad1ce5006b Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 14:45:27 +0900 Subject: [PATCH 06/14] fix --- .github/workflows/dockers-agent-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockers-agent-image.yml b/.github/workflows/dockers-agent-image.yml index 391372f1ef..7e17ad0d81 100644 --- a/.github/workflows/dockers-agent-image.yml +++ b/.github/workflows/dockers-agent-image.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Build docker image: agent-ngt" +name: "Build docker image: agent" on: push: branches: From 523424891d160f824c1ddd38529f9e7ea2b1377d Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 14:46:47 +0900 Subject: [PATCH 07/14] fix --- .github/workflows/dockers-agent-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockers-agent-image.yml b/.github/workflows/dockers-agent-image.yml index 7e17ad0d81..99fb704717 100644 --- a/.github/workflows/dockers-agent-image.yml +++ b/.github/workflows/dockers-agent-image.yml @@ -28,7 +28,7 @@ on: paths: - ".github/actions/docker-build/action.yaml" - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-agent-ngt-rust-image.yml" + - ".github/workflows/dockers-agent-image.yml" - "rust/Cargo.lock" - "rust/Cargo.toml" - "rust/bin/agent/**" From afc1db5313cb7cb7040ec3589a34a2d94e3f35ea Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 14:47:37 +0900 Subject: [PATCH 08/14] fix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9251b2572b..0056296e61 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ GHCRORG = ghcr.io/$(ORG)/$(NAME) AGENT_NGT_IMAGE = $(NAME)-agent-ngt AGENT_FAISS_IMAGE = $(NAME)-agent-faiss AGENT_SIDECAR_IMAGE = $(NAME)-agent-sidecar -AGENT_IMAGE = $(NAME)-agent-ngt-rust +AGENT_IMAGE = $(NAME)-agent CI_CONTAINER_IMAGE = $(NAME)-ci-container DEV_CONTAINER_IMAGE = $(NAME)-dev-container DISCOVERER_IMAGE = $(NAME)-discoverer-k8s From 0b98486d08b12ea4eaf011cd3fa88a19c725ab65 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 14:49:18 +0900 Subject: [PATCH 09/14] fix --- Makefile.d/docker.mk | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index a6c20db689..4911cd61d2 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -108,11 +108,11 @@ docker/build/agent-sidecar: IMAGE=$(AGENT_SIDECAR_IMAGE) \ docker/build/image -.PHONY: docker/name/agent-ngt-rust +.PHONY: docker/name/agent docker/name/agent-ngt-rust: @echo "$(ORG)/$(AGENT_IMAGE)" -.PHONY: docker/build/agent-ngt-rust +.PHONY: docker/build/agent docker/name/agent-ngt-rust: @make DOCKERFILE="$(ROOTDIR)/dockers/agent/core/agent/Dockerfile" \ IMAGE=$(AGENT_IMAGE) \ @@ -296,6 +296,3 @@ docker/build/benchmark-operator: @make DOCKERFILE="$(ROOTDIR)/dockers/tools/benchmark/operator/Dockerfile" \ IMAGE=$(BENCHMARK_OPERATOR_IMAGE) \ docker/build/image - -.PHONY: docker/build/agent-ngt-rust - @make DOCKER \ No newline at end of file From a8061f3e01c3f227ee12b12091c21a7fa5835e35 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 14:50:31 +0900 Subject: [PATCH 10/14] fix --- Makefile.d/docker.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 4911cd61d2..5d161278d0 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -109,11 +109,11 @@ docker/build/agent-sidecar: docker/build/image .PHONY: docker/name/agent -docker/name/agent-ngt-rust: +docker/name/agent: @echo "$(ORG)/$(AGENT_IMAGE)" .PHONY: docker/build/agent -docker/name/agent-ngt-rust: +docker/build/agent: @make DOCKERFILE="$(ROOTDIR)/dockers/agent/core/agent/Dockerfile" \ IMAGE=$(AGENT_IMAGE) \ docker/build/image From 23f0c70febfc2e7bb280ab2b7f8238e7e3354fef Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 14:55:01 +0900 Subject: [PATCH 11/14] fix --- .github/workflows/dockers-agent-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockers-agent-image.yml b/.github/workflows/dockers-agent-image.yml index 99fb704717..b4ae752529 100644 --- a/.github/workflows/dockers-agent-image.yml +++ b/.github/workflows/dockers-agent-image.yml @@ -70,5 +70,5 @@ jobs: build: uses: ./.github/workflows/_docker-image.yaml with: - target: agent-ngt-rust + target: agent secrets: inherit From 9a6ab442a1a1b31d4f495004f7fd3be1b01b872d Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 15:39:13 +0900 Subject: [PATCH 12/14] fix --- dockers/agent/core/agent/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile index d1afbf4b68..c0df58eb31 100644 --- a/dockers/agent/core/agent/Dockerfile +++ b/dockers/agent/core/agent/Dockerfile @@ -57,6 +57,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d COPY Makefile.d . + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . COPY .git . @@ -72,6 +73,7 @@ COPY rust . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} RUN make ngt/install + RUN make rust/install RUN make rust/target/release/agent \ From 138e1c9c1ebd462c89beb549b330c7647b11eda7 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 16:16:57 +0900 Subject: [PATCH 13/14] fix --- dockers/agent/core/agent/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile index c0df58eb31..726571fac6 100644 --- a/dockers/agent/core/agent/Dockerfile +++ b/dockers/agent/core/agent/Dockerfile @@ -62,6 +62,8 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} COPY Makefile . COPY .git . +RUN make RUST_VERSION=${RUST_VERSION} rust/install + WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd COPY cmd . @@ -74,8 +76,6 @@ COPY rust . WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} RUN make ngt/install -RUN make rust/install - RUN make rust/target/release/agent \ && mv ./rust/target/release/agent "/usr/bin/${APP_NAME}" From 7bb5a4bc84cb4f13675466bf78f357877d14ec23 Mon Sep 17 00:00:00 2001 From: Kosuke Morimoto Date: Thu, 18 Apr 2024 16:43:25 +0900 Subject: [PATCH 14/14] fix --- Makefile.d/docker.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 5d161278d0..c28bf82ed8 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -116,6 +116,7 @@ docker/name/agent: docker/build/agent: @make DOCKERFILE="$(ROOTDIR)/dockers/agent/core/agent/Dockerfile" \ IMAGE=$(AGENT_IMAGE) \ + DISTROLESS_IMAGE=gcr.io/distroless/cc-debian12 \ docker/build/image .PHONY: docker/name/discoverer-k8s