Skip to content

Commit

Permalink
manage models
Browse files Browse the repository at this point in the history
  • Loading branch information
@laitassou
laitassou committed Jun 7, 2020
1 parent 8c24e4f commit 4b5070b
Show file tree
Hide file tree
Showing 10 changed files with 324 additions and 212 deletions.
2 changes: 1 addition & 1 deletion service/examples/mpg/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ set(CMAKE_BUILD_TYPE Debug)

set(CMAKE_CXX_STANDARD 17)
#include_directories(../../include)
add_executable(mpg main.cpp ../../src/Model.cpp ../../src/Tensor.cpp ../../src/ModelInfos.cpp ../../src/LoaderHarness.cpp )
add_executable(mpg main.cpp ../../src/Model.cpp ../../src/Tensor.cpp ../../src/ModelInfos.cpp ../../src/LoaderHarness.cpp ../../src/BasicModelManager.cpp )
target_include_directories(mpg PRIVATE ../../include)
target_link_libraries (mpg -ltensorflow)
35 changes: 23 additions & 12 deletions service/examples/mpg/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@
#include "../../include/Loader.hpp"
#include "../../include/LoaderHarness.hpp"

#include "../../include/TFLoader.hpp"

#include "../../include/BasicModelManager.hpp"




#include <algorithm>
#include <iterator>

Expand Down Expand Up @@ -45,8 +47,14 @@ int main() {

//p_loader = make_unique<BasicLoader>("../frozen_graph.pb");

std::array<ModelId,1> mlist = { ModelId{"mpg",{1,0}}};
std::array<const ModelId,2> mlist = { ModelId{"mpg",{1,0}},ModelId{"mpg",{1,1}}};

if( mlist[0] == mlist[1])
{
cout << "equal" << endl;
}


//auto p_loader_harness = make_unique<LoaderHarness>(model_id , std::move(p_loader));

//p_loader_harness->Load();
Expand All @@ -57,42 +65,45 @@ int main() {
//p_loader->Unload();
//cout << "state:" << as_integer(p_loader_harness->state())<<endl;

ModelHandle<Model> handle ;


std::unique_ptr<BasicModelManager> manager_;
BasicModelManager::Create(&manager_);

for (auto &el : mlist){
manager_->LoadModel(el,"../","frozen_graph.pb");
cout << "load model :" << el.name << ": "<<el.version.major << ":" <<el.version.minor<<endl;
manager_->LoadModel(el,"../","frozen_graph.pb", ModelType::TFL);
}

auto v = manager_->ListAvailableModelIds();

for( auto &elem: v)
{
cout << "name : " << elem.name << "version:" << elem.version.major <<endl;
cout << "name : " << elem.name << "version:" << elem.version.major <<"."<< elem.version.minor<<endl;
}

//manager_->UnloadModel(mlist[0]);

//std::unique_ptr<UntypedModelHandle> untyped_handle ;

ModelHandle<int64_t> handle ;
manager_->GetModelHandle(mlist[0],&handle);


manager_->UnloadModel(mlist[0]);
Model * md = handle.get();
//manager_->UnloadModel(mlist[0]);





#if 0


mInfos.compute_columns();

std::cout << "depth:" <<mInfos.get_depth() <<std::endl;
// Create Tensors
Tensor input(m, "x");
Tensor prediction(m, "Identity");
Tensor input(*md, "x");
Tensor prediction(*md, "Identity");


std::array<float,7> mean {5.477,195.32,104.86,2990.25,15.56,75.89,1.573};
Expand Down Expand Up @@ -131,7 +142,7 @@ Origin 314.0 1.573248 0.800988 1.0 1.00 1.0 2.00
input.set_data(norm_data);

// Run and show predictions
m.run(input, prediction);
md->run(input, prediction);

// Get tensor with predictions
auto result = prediction.Tensor::get_data<float>();
Expand All @@ -142,7 +153,7 @@ Origin 314.0 1.573248 0.800988 1.0 1.00 1.0 2.00
std::cout << std::endl;


#endif



}
146 changes: 28 additions & 118 deletions service/include/BasicModelManager.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,25 @@
#include "LoaderHarness.hpp"
#include "ModelManager.hpp"

//#define ONNX_MODEL
//#if defined(TENSORFLOW_MODEL)
#include "TFLoader.hpp"
//#elif defined(ONNX_MODEL)
#include "ONNXLoader.hpp"
//#else
#include "MRVLLoader.hpp"
//#endif

namespace ML {



enum class ModelType {
MRVL,
ONNX,
TFL
};

class SharedPtrHandle final : public UntypedModelHandle {
public:
~SharedPtrHandle() override = default;
Expand All @@ -29,146 +44,41 @@ class BasicModelManager : public ModelManager {
using PreHook = std::function<void(const uint32_t&)>;

~BasicModelManager() override { cout << "BasicModelManager destructor "<<endl;};

std::vector<ModelId> ListAvailableModelIds() const override;


void LoadModel(const ModelId& id,std::string path, std::string name);
void LoadModel(const ModelId& id,std::string path, std::string name, ModelType type );
void UnloadModel(const ModelId& id);


static Status Create(std::unique_ptr<BasicModelManager>* manager);

BasicModelManager(){
handles_map_ = std::make_shared<HandlesMap>();
}

private :

struct Compare
{

bool operator() (const ModelId& l,
const ModelId& r) const{
return l.name.compare(r.name) && l.version.major == r.version.major && l.version.minor == r.version.minor ;
}
bool operator() (const ModelId& l,const ModelId& r) const
{
return l.name.compare( r.name) <0 ||
l.name.compare( r.name) ==0 && l.version.major <r.version.major ||
l.name.compare( r.name) ==0 && l.version.major == r.version.major && l.version.minor < r.version.minor;
}

};

using HandlesMap = std::map<ModelId,std::shared_ptr< LoaderHarness> ,Compare>;
using HandlesMap = std::map<ModelId,std::shared_ptr< LoaderHarness> , Compare>;
using mutex_lock = std::lock_guard<std::mutex>;

HandlesMap handles_map_;

std::shared_ptr<HandlesMap> handles_map_;
mutable mutex mu;

Status GetUntypedModelHandle(const ModelId& request, std::unique_ptr<UntypedModelHandle>* untyped_handle) override ;

std::map<ModelId, std::unique_ptr<UntypedModelHandle>>GetAvailableUntypedModelHandles() const override {};

mutable mutex mu;


};

std::vector<ModelId> BasicModelManager::ListAvailableModelIds() const {
std::vector<ModelId> ids;

HandlesMap handles_map = handles_map_;
for (auto iter = handles_map.begin(); iter != handles_map.end(); iter++) {
// We get the iterator where all the values for a particular key ends.

if (iter->first.version.major>=1) {
ids.push_back(iter->first);
}

}

return ids;
}




Status BasicModelManager::GetUntypedModelHandle(const ModelId& request,std::unique_ptr<UntypedModelHandle>* const untyped_handle) {
const auto found_it = handles_map_.find(request);
if (found_it == handles_map_.end()) {
return Status::ERROR;
}
//std::shared_ptr<const HandlesMap> handles_map = handles_map_.get();

const LoaderHarness& harness = *found_it->second;

untyped_handle->reset(new SharedPtrHandle(
harness.id(), std::shared_ptr<Loader>(harness.loader())));

return Status::OK;
}

/*
std::map<ModelId, std::unique_ptr<UntypedModelHandle>>BasicModelManager::GetAvailableUntypedModelHandles() const {
std::map<ModelId, std::unique_ptr<UntypedModelHandle>> result;
//std::shared_ptr<const HandlesMap> handles_map = handles_map_.get();
for (const auto& handle : *handles_map_) {
const ModelId& request = handle.first;
if (!request.version) {
continue;
}
const LoaderHarness& harness = *handle.second;
result.emplace(harness.id(),
std::unique_ptr<UntypedModelHandle>(new SharedPtrHandle(
harness.id(), std::shared_ptr<Loader>(handles_map_,harness.loader()))));
}
return result;
}
*/


void BasicModelManager::LoadModel(const ModelId& id,std::string path, std::string name) {
cout << " load servable " << id.name << " version:" <<id.version.major<<endl;
mutex_lock l(mu);
std::unique_ptr<Loader> p_loader;

p_loader = make_unique<BasicLoader>(path,name);

std::shared_ptr<LoaderHarness> p_loader_harness = make_shared<LoaderHarness>(id , std::move(p_loader));

Status status = p_loader_harness->Load();

if( status == Status::OK){
handles_map_.emplace(id,p_loader_harness);
cout << "size:" <<handles_map_.size()<<endl;
cout << " emplace model " << id.name << " version:" <<id.version.major<<endl;

}
else
{
cout << "failed to load model :"<< id.name << endl;
}
}


void BasicModelManager::UnloadModel(const ModelId& model) {
//cout << " unload model " << model.name << " version:" <<model.version.major<<"." <<model.version.minor<<endl;
const auto found_it = handles_map_.find(model);

mutex_lock l(mu);
if(found_it != handles_map_.end()){
handles_map_.erase(found_it);
cout << "UnloadModel size:" <<handles_map_.size()<<endl;

}
else
{
cout << "nout found" << endl;
}
}


Status BasicModelManager::Create(std::unique_ptr<BasicModelManager>* manager) {
manager->reset(new BasicModelManager());
return Status::OK;
}


}
25 changes: 14 additions & 11 deletions service/include/Loader.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
#pragma once
/*
*
*
*
*
*/
#ifndef _ML_LOADER_
#define _ML_LOADER_

#include <iostream>
#include "ModelManager.hpp"
Expand All @@ -8,12 +15,13 @@ using namespace std;
namespace ML {


// Loader abstract interface
// Loader interface to load/unload model
// NVI interface to offer more flexibility to check preconditions/ postconditions
class Loader{
public:

Loader(){
cout << "Loader"<< endl;
cout << "Loader ctor"<< endl;
}

void Load(){
Expand All @@ -28,22 +36,17 @@ class Loader{
doEstimate();
}

Status LoadWithMetadata(const ModelId& model){
doLoadWithMetadata(model);
return Status::OK;
}

virtual ~Loader(){};

virtual void * servable() = 0;

private:
virtual Status doLoadWithMetadata(const ModelId& model) {Load(); return Status::ERROR;};
virtual void doLoad() =0;
virtual void doUnload() =0;
virtual void doEstimate() {};


};

}
}

#endif //_ML_LOADER_
2 changes: 1 addition & 1 deletion service/include/LoaderHarness.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class LoaderHarness {
private:

const ModelId _id;
const std::unique_ptr<Loader> _loader;
const std::shared_ptr<Loader> _loader;
State _state = State::New;
mutable std::mutex _mu;

Expand Down
Loading

0 comments on commit 4b5070b

Please sign in to comment.