From 4b5070bbcb5ed05eebd601e534f289e19173d88f Mon Sep 17 00:00:00 2001 From: atlas Date: Sun, 7 Jun 2020 19:47:31 +0200 Subject: [PATCH] manage models --- service/examples/mpg/CMakeLists.txt | 2 +- service/examples/mpg/main.cpp | 35 +++--- service/include/BasicModelManager.hpp | 146 +++++--------------------- service/include/Loader.hpp | 25 +++-- service/include/LoaderHarness.hpp | 2 +- service/include/MRVLLoader.hpp | 56 ++++++++++ service/include/ModelManager.hpp | 99 ++++++----------- service/include/ONNXLoader.hpp | 57 ++++++++++ service/include/TFLoader.hpp | 10 +- service/src/BasicModelManager.cpp | 104 ++++++++++++++++++ 10 files changed, 324 insertions(+), 212 deletions(-) create mode 100644 service/include/MRVLLoader.hpp create mode 100644 service/include/ONNXLoader.hpp create mode 100644 service/src/BasicModelManager.cpp diff --git a/service/examples/mpg/CMakeLists.txt b/service/examples/mpg/CMakeLists.txt index da6d159..f7abbc1 100644 --- a/service/examples/mpg/CMakeLists.txt +++ b/service/examples/mpg/CMakeLists.txt @@ -6,6 +6,6 @@ set(CMAKE_BUILD_TYPE Debug) set(CMAKE_CXX_STANDARD 17) #include_directories(../../include) -add_executable(mpg main.cpp ../../src/Model.cpp ../../src/Tensor.cpp ../../src/ModelInfos.cpp ../../src/LoaderHarness.cpp ) +add_executable(mpg main.cpp ../../src/Model.cpp ../../src/Tensor.cpp ../../src/ModelInfos.cpp ../../src/LoaderHarness.cpp ../../src/BasicModelManager.cpp ) target_include_directories(mpg PRIVATE ../../include) target_link_libraries (mpg -ltensorflow) \ No newline at end of file diff --git a/service/examples/mpg/main.cpp b/service/examples/mpg/main.cpp index 0842686..1e53a23 100644 --- a/service/examples/mpg/main.cpp +++ b/service/examples/mpg/main.cpp @@ -6,10 +6,12 @@ #include "../../include/Loader.hpp" #include "../../include/LoaderHarness.hpp" -#include "../../include/TFLoader.hpp" + #include "../../include/BasicModelManager.hpp" + + #include #include @@ -45,8 +47,14 @@ int main() { //p_loader = make_unique("../frozen_graph.pb"); - std::array mlist = { ModelId{"mpg",{1,0}}}; + std::array mlist = { ModelId{"mpg",{1,0}},ModelId{"mpg",{1,1}}}; + if( mlist[0] == mlist[1]) + { + cout << "equal" << endl; + } + + //auto p_loader_harness = make_unique(model_id , std::move(p_loader)); //p_loader_harness->Load(); @@ -57,42 +65,45 @@ int main() { //p_loader->Unload(); //cout << "state:" << as_integer(p_loader_harness->state())< handle ; + + std::unique_ptr manager_; BasicModelManager::Create(&manager_); for (auto &el : mlist){ - manager_->LoadModel(el,"../","frozen_graph.pb"); + cout << "load model :" << el.name << ": "<LoadModel(el,"../","frozen_graph.pb", ModelType::TFL); } auto v = manager_->ListAvailableModelIds(); for( auto &elem: v) { - cout << "name : " << elem.name << "version:" << elem.version.major <UnloadModel(mlist[0]); //std::unique_ptr untyped_handle ; - ModelHandle handle ; manager_->GetModelHandle(mlist[0],&handle); - - manager_->UnloadModel(mlist[0]); + Model * md = handle.get(); + //manager_->UnloadModel(mlist[0]); -#if 0 + mInfos.compute_columns(); std::cout << "depth:" < mean {5.477,195.32,104.86,2990.25,15.56,75.89,1.573}; @@ -131,7 +142,7 @@ Origin 314.0 1.573248 0.800988 1.0 1.00 1.0 2.00 input.set_data(norm_data); // Run and show predictions - m.run(input, prediction); + md->run(input, prediction); // Get tensor with predictions auto result = prediction.Tensor::get_data(); @@ -142,7 +153,7 @@ Origin 314.0 1.573248 0.800988 1.0 1.00 1.0 2.00 std::cout << std::endl; -#endif + } diff --git a/service/include/BasicModelManager.hpp b/service/include/BasicModelManager.hpp index 8ceb75e..d46acac 100644 --- a/service/include/BasicModelManager.hpp +++ b/service/include/BasicModelManager.hpp @@ -3,10 +3,25 @@ #include "LoaderHarness.hpp" #include "ModelManager.hpp" +//#define ONNX_MODEL +//#if defined(TENSORFLOW_MODEL) +#include "TFLoader.hpp" +//#elif defined(ONNX_MODEL) +#include "ONNXLoader.hpp" +//#else +#include "MRVLLoader.hpp" +//#endif + namespace ML { +enum class ModelType { + MRVL, + ONNX, + TFL +}; + class SharedPtrHandle final : public UntypedModelHandle { public: ~SharedPtrHandle() override = default; @@ -29,146 +44,41 @@ class BasicModelManager : public ModelManager { using PreHook = std::function; ~BasicModelManager() override { cout << "BasicModelManager destructor "< ListAvailableModelIds() const override; - - void LoadModel(const ModelId& id,std::string path, std::string name); + void LoadModel(const ModelId& id,std::string path, std::string name, ModelType type ); void UnloadModel(const ModelId& id); - static Status Create(std::unique_ptr* manager); + BasicModelManager(){ + handles_map_ = std::make_shared(); + } private : struct Compare { - - bool operator() (const ModelId& l, - const ModelId& r) const{ - return l.name.compare(r.name) && l.version.major == r.version.major && l.version.minor == r.version.minor ; - } + bool operator() (const ModelId& l,const ModelId& r) const + { + return l.name.compare( r.name) <0 || + l.name.compare( r.name) ==0 && l.version.major ,Compare>; + using HandlesMap = std::map , Compare>; using mutex_lock = std::lock_guard; - HandlesMap handles_map_; - + std::shared_ptr handles_map_; + mutable mutex mu; Status GetUntypedModelHandle(const ModelId& request, std::unique_ptr* untyped_handle) override ; std::map>GetAvailableUntypedModelHandles() const override {}; - mutable mutex mu; - - }; -std::vector BasicModelManager::ListAvailableModelIds() const { - std::vector ids; - - HandlesMap handles_map = handles_map_; - for (auto iter = handles_map.begin(); iter != handles_map.end(); iter++) { - // We get the iterator where all the values for a particular key ends. - - if (iter->first.version.major>=1) { - ids.push_back(iter->first); - } - - } - - return ids; -} - - - - -Status BasicModelManager::GetUntypedModelHandle(const ModelId& request,std::unique_ptr* const untyped_handle) { - const auto found_it = handles_map_.find(request); - if (found_it == handles_map_.end()) { - return Status::ERROR; - } - //std::shared_ptr handles_map = handles_map_.get(); - - const LoaderHarness& harness = *found_it->second; - - untyped_handle->reset(new SharedPtrHandle( - harness.id(), std::shared_ptr(harness.loader()))); - - return Status::OK; -} - -/* -std::map>BasicModelManager::GetAvailableUntypedModelHandles() const { - std::map> result; - - //std::shared_ptr handles_map = handles_map_.get(); - - for (const auto& handle : *handles_map_) { - const ModelId& request = handle.first; - if (!request.version) { - continue; - } - const LoaderHarness& harness = *handle.second; - result.emplace(harness.id(), - std::unique_ptr(new SharedPtrHandle( - harness.id(), std::shared_ptr(handles_map_,harness.loader())))); - } - return result; -} - - -*/ - - -void BasicModelManager::LoadModel(const ModelId& id,std::string path, std::string name) { - cout << " load servable " << id.name << " version:" < p_loader; - - p_loader = make_unique(path,name); - - std::shared_ptr p_loader_harness = make_shared(id , std::move(p_loader)); - - Status status = p_loader_harness->Load(); - - if( status == Status::OK){ - handles_map_.emplace(id,p_loader_harness); - cout << "size:" <* manager) { - manager->reset(new BasicModelManager()); - return Status::OK; -} - } \ No newline at end of file diff --git a/service/include/Loader.hpp b/service/include/Loader.hpp index 2a9ffc4..91dd009 100644 --- a/service/include/Loader.hpp +++ b/service/include/Loader.hpp @@ -1,4 +1,11 @@ -#pragma once +/* + * + * + * + * + */ +#ifndef _ML_LOADER_ +#define _ML_LOADER_ #include #include "ModelManager.hpp" @@ -8,12 +15,13 @@ using namespace std; namespace ML { -// Loader abstract interface +// Loader interface to load/unload model +// NVI interface to offer more flexibility to check preconditions/ postconditions class Loader{ public: Loader(){ - cout << "Loader"<< endl; + cout << "Loader ctor"<< endl; } void Load(){ @@ -28,17 +36,10 @@ class Loader{ doEstimate(); } - Status LoadWithMetadata(const ModelId& model){ - doLoadWithMetadata(model); - return Status::OK; - } - virtual ~Loader(){}; - virtual void * servable() = 0; private: - virtual Status doLoadWithMetadata(const ModelId& model) {Load(); return Status::ERROR;}; virtual void doLoad() =0; virtual void doUnload() =0; virtual void doEstimate() {}; @@ -46,4 +47,6 @@ class Loader{ }; -} \ No newline at end of file +} + +#endif //_ML_LOADER_ \ No newline at end of file diff --git a/service/include/LoaderHarness.hpp b/service/include/LoaderHarness.hpp index dee306e..f6b093f 100644 --- a/service/include/LoaderHarness.hpp +++ b/service/include/LoaderHarness.hpp @@ -38,7 +38,7 @@ class LoaderHarness { private: const ModelId _id; - const std::unique_ptr _loader; + const std::shared_ptr _loader; State _state = State::New; mutable std::mutex _mu; diff --git a/service/include/MRVLLoader.hpp b/service/include/MRVLLoader.hpp new file mode 100644 index 0000000..e55aa0c --- /dev/null +++ b/service/include/MRVLLoader.hpp @@ -0,0 +1,56 @@ +#pragma once + + +#include "Model.hpp" +#include "Loader.hpp" + +namespace ML { + +// MRVL Loader implemntation +// Call api and add needed resources : package_id etc +class MRVLLoader : public Loader { + +public: + + MRVLLoader(const std::string & resource_path, const std::string & resource_name) : _resource_path(resource_path), + _resource_name(resource_name) {} + + ~MRVLLoader() + { + cout << "MRVL loader destructor " << endl; + } + + void * servable() + { + return reinterpret_cast (_servable.get()); + } + + private: + + void doLoad() override + { + cout << "doLoad MRVL loader"<< endl; + // send request to load MRVL package, model + } + + void doUnload() override + { + cout << "doUnLoad MRVL loader"<< endl; + //_servable.reset(); + + } + + void doEstimate() override + {} + + //T * _m; + std::string _resource_path; + std::string _resource_name; + + std::unique_ptr _servable; + //ModelHandle _handle; + + +}; + +} \ No newline at end of file diff --git a/service/include/ModelManager.hpp b/service/include/ModelManager.hpp index f0db0fe..6b34068 100644 --- a/service/include/ModelManager.hpp +++ b/service/include/ModelManager.hpp @@ -1,4 +1,13 @@ -#pragma once +/* + * + * + * + * + * + */ + +#ifndef _ML_MODEL_MANAGER_ +#define _ML_MODEL_MANAGER_ #include #include @@ -33,17 +42,20 @@ enum class Version_SELECT FIRST, LAST, }; + struct Version{ size_t major; size_t minor; Version(size_t mj, size_t mn): major(mj),minor(mn){} }; + struct ModelId { std::string name; Version version; }; + class UntypedModelHandle { public: virtual ~UntypedModelHandle() = default; @@ -91,50 +103,27 @@ template class ModelHandle { - template struct ModelContext{ - InputType input_type; - size_t input_size; - OutputType output_type; - size_t output_size; - //void * input_buffer; - InputType * input_buffer; - //void * output_buffer; - OutputType * output_buffer; - -}; - - -template class RefModel : ModelId +template class ModelContext { public: - RefModel (size_t id, ModelContext * p_ctx): id(id), p_Model_Context(p_ctx) {}; + ModelContext (size_t id,uint8_t inf_mask,size_t inp_size, size_t out_size, InputType * in_buf, OutputType * out_buf); private: size_t id; uint8_t inference_mask; - ModelContext * p_Model_Context; -}; - - - - -struct ModelRequest { - - std::string name; - - unsigned version; - - - ModelRequest() = default; - ModelRequest(const std::string& name_in, size_t version_in) - : name(name_in), - version(version_in){} + //InputType input_type; + size_t input_size; + //OutputType output_type; + size_t output_size; + //void * input_buffer; + InputType * input_buffer; + //void * output_buffer; + OutputType * output_buffer; }; - template constexpr bool operator==(const ModelHandle& l, const ModelHandle& r) { @@ -149,42 +138,23 @@ constexpr bool operator!=(const ModelHandle& l, - -inline bool operator==(const ModelRequest& l, - const ModelRequest& r) { - return l.name == r.name && l.version == r.version; -} - - -inline bool operator<(const ModelRequest& l, - const ModelRequest& r) { - return l.name.compare( r.name); -} - - - - - -inline bool operator==(const ModelId& l, - const ModelId& r) { - return l.name.compare( r.name) && l.version.major == r.version.major && l.version.minor == r.version.minor; -} - - -inline bool operator<(const ModelId& l, - const ModelId& r) { - return l.name.compare( r.name); +inline bool operator==(const ModelId& l, const ModelId& r) +{ + return (l.name.compare( r.name)==0) && + l.version.major == r.version.major && + l.version.minor == r.version.minor; } - - - - +/* + * + * + * + * + */ class ModelManager { public: - virtual ~ModelManager() = default; virtual std::vector ListAvailableModelIds() const = 0; @@ -236,3 +206,4 @@ std::map> ModelManager::GetAvailableModelHandles() } +#endif //_ML_MODEL_MANAGER_ \ No newline at end of file diff --git a/service/include/ONNXLoader.hpp b/service/include/ONNXLoader.hpp new file mode 100644 index 0000000..2232083 --- /dev/null +++ b/service/include/ONNXLoader.hpp @@ -0,0 +1,57 @@ +#pragma once + + +#include "Model.hpp" +#include "Loader.hpp" + +namespace ML { + +// Loader implemntation +class ONNXMODEL {}; + +class ONNXLoader : public Loader { + +public: + + ONNXLoader(const std::string & resource_path, const std::string & resource_name) : _resource_path(resource_path), + _resource_name(resource_name) {} + + ~ONNXLoader() { + cout << "basic loader destructor " << endl; + //doUnload(); + } + + void * servable() { return reinterpret_cast (_servable.get());} + + private: + + void doLoad() override + { + //T _m = new (_m) T(_resource_name); + cout << "doLoad ONNX loader"<< endl; + //return _m; + //_servable.reset( new Model(_resource_path+_resource_name)); + } + + void doUnload() override + { + cout << "doUnLoad ONNX loader"<< endl; + _servable.reset(); + + //delete m; + } + + void doEstimate() override + {} + + //T * _m; + std::string _resource_path; + std::string _resource_name; + + std::unique_ptr _servable; + //ModelHandle _handle; + + +}; + +} \ No newline at end of file diff --git a/service/include/TFLoader.hpp b/service/include/TFLoader.hpp index 968fa92..521194b 100644 --- a/service/include/TFLoader.hpp +++ b/service/include/TFLoader.hpp @@ -8,14 +8,14 @@ namespace ML { // Loader implemntation -class BasicLoader : public Loader { +class TFLoader : public Loader { public: - BasicLoader(const std::string & resource_path, const std::string & resource_name) : _resource_path(resource_path), + TFLoader(const std::string & resource_path, const std::string & resource_name) : _resource_path(resource_path), _resource_name(resource_name) {} - ~BasicLoader() { + ~TFLoader() { cout << "basic loader destructor " << endl; //doUnload(); } @@ -47,8 +47,8 @@ class BasicLoader : public Loader { std::string _resource_path; std::string _resource_name; - std::shared_ptr _servable; - ModelHandle _handle; + std::unique_ptr _servable; + //ModelHandle _handle; }; diff --git a/service/src/BasicModelManager.cpp b/service/src/BasicModelManager.cpp new file mode 100644 index 0000000..13e004c --- /dev/null +++ b/service/src/BasicModelManager.cpp @@ -0,0 +1,104 @@ + +#include "BasicModelManager.hpp" + + +using namespace std; + +namespace ML { + +std::vector BasicModelManager::ListAvailableModelIds() const +{ + std::vector ids; + + HandlesMap * handles_map = handles_map_.get(); + for (auto iter = handles_map->begin(); iter != handles_map->end(); iter++) { + if (iter->first.version.major>=1) { + ids.push_back(iter->first); + } + } + + return ids; +} + + +Status BasicModelManager::GetUntypedModelHandle(const ModelId& request,std::unique_ptr* const untyped_handle) +{ + + const auto found_it = handles_map_->find(request); + if (found_it == handles_map_->end()) { + return Status::ERROR; + } + //std::shared_ptr handles_map = handles_map_.get(); + + const LoaderHarness& harness = *found_it->second; + std::shared_ptr handles_map = handles_map_; + + untyped_handle->reset(new SharedPtrHandle( + harness.id(), std::shared_ptr(handles_map,harness.loader()))); + + return Status::OK; +} + + +void BasicModelManager::LoadModel(const ModelId& id,std::string path, std::string name, ModelType type) +{ + cout << " load servable " << id.name << " version:" < p_loader; + + const auto found_it = handles_map_->find(id); + if(found_it == handles_map_->end()) + { + + if (type == ModelType::TFL) + p_loader = make_unique(path,name); + else if (type == ModelType::ONNX) + p_loader = make_unique(path,name); + else + p_loader = make_unique(path,name); + + std::shared_ptr p_loader_harness = make_shared(id , std::move(p_loader)); + + Status status = p_loader_harness->Load(); + + if( status == Status::OK){ + handles_map_->emplace(id,p_loader_harness); + cout << "size:" <size()<find(model); + + mutex_lock l(mu); + if(found_it != handles_map_->end()){ + handles_map_->erase(found_it); + cout << "UnloadModel size:" <size()<* manager){ + manager->reset(new BasicModelManager()); + return Status::OK; +} + +}