Skip to content

Commit

Permalink
Merge pull request bsc-dd#461 from bsc-dd/storagenumpy-with-raw-point…
Browse files Browse the repository at this point in the history 
…er-good

StorageNumpy with raw pointers
  • Loading branch information
@yolandab
yolandab authored Sep 13, 2023
2 parents bcd6763 + 105851a commit 73b2372
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 14 deletions.
4 changes: 4 additions & 0 deletions examples/c++/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ OBJS= \
apitest_iterator \
simple_numpy \
simple_numpy_rw \
simple_byte_numpy_rw \
simple_dict_rw \
nested_rw \
test_shared_ptr \
Expand All @@ -30,6 +31,9 @@ simple_numpy: simple_numpy.cpp
simple_numpy_rw: simple_numpy_rw.cpp
g++ -o $@ $^ ${CPPFLAGS}

simple_byte_numpy_rw: simple_byte_numpy_rw.cpp
g++ -o $@ $^ ${CPPFLAGS}

simple_dict_rw: simple_dict_rw.cpp
g++ -o $@ $^ ${CPPFLAGS}

Expand Down
88 changes: 88 additions & 0 deletions examples/c++/simple_byte_numpy_rw.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#include <iostream>

#include <StorageNumpy.h>

#define COLS 3
#define ROWS 4


char * generateNumpyContent(const std::vector<uint32_t> &metas) {

char *numpy=(char*)malloc(sizeof(char)*metas[0]*metas[1]);
char *tmp = numpy;
char num = 'a';
for (int i=0; i<metas[0]; i++) {
*tmp = (num++ % ('z'+1));
std::cout<< "++ "<<i<< " ==> " << (*tmp) << std::endl;
tmp+=1;
}
return (char*) numpy;
}

bool equalsNumpy(const StorageNumpy& s, char *data, std::vector<uint32_t> metas) {
// Check metas
if (s.metas.size() != metas.size()) {
std::cout<< "++ dimensions differs! "<< s.metas.size()<< " != " << metas.size() << std::endl;
return false;
}
for (int i=0; i<metas.size(); i++) {
if (s.metas[i] != metas[i]) {
std::cout<< "++ dimension "<<i<<" differs! "<< s.metas[i]<< " != " << metas[i] << std::endl;
return false;
}
}
// Check Content
char* p = (char*)s.data;
char* q = data;
for (int i=0; i<metas[0]; i++) {
if (*p != *q) {
return false;
}
p++;
q++;
}
return true;
}


void test_retrieve_simple(const char *name) {
std::vector<uint32_t> metadata = {12};
char *data = generateNumpyContent(metadata);
StorageNumpy sn(data,metadata,'i'); // 'b' is for BOOLEAN not bytes

sn.make_persistent(name);

sn.sync();

StorageNumpy sn2;
sn2.getByAlias(name);

if (!equalsNumpy(sn2, (char*)data, metadata)) {
std::cout << "FAILED Retrieved Numpy ["<< name<< "] contains unexpected content (differnt from stored). " <<std::endl;
} else {
std::cout << "SUCCESS Retrieved Numpy ["<< name<< "] " <<std::endl;
}
}


void test_retrieve_from_python(const char *name) {
StorageNumpy sn;
sn.getByAlias(name);
char* p = (char*)sn.data;


for (int i=0; i<sn.metas[0]; i++) {
std::cout << "Retrieved " << i << " " <<*p<<std::endl;
p++;
}
}

int main() {
std::cout<< "+ STARTING C++ APP"<<std::endl;

std::cout << "Starting test 1 " <<std::endl;
test_retrieve_simple("mynumpybytetoread");
//test_retrieve_from_python("python"); // asumes there is a previously stored sn named python. To test that we can read somethin stored from python

std::cout << "End tests " <<std::endl;
}
50 changes: 36 additions & 14 deletions hecuba_core/src/api/StorageNumpy.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class StorageNumpy:virtual public IStorage {
void *data = nullptr; /* Pointer to memory containing the object. */
std::vector<uint32_t> metas;
ArrayMetadata numpy_metas; /* Pointer to memory containing the metadata. READ ONLY. DO NOT FREE. This object does NOT own the memory! */
void initObjSpec() {
void initObjSpec(char dtype) {
std::vector<std::pair<std::string, std::string>> pkeystypes_numpy = {
{"storage_id", "uuid"},
{"cluster_id", "int"}
Expand All @@ -51,26 +51,30 @@ class StorageNumpy:virtual public IStorage {
, std::string("")
);
setObjSpec(snSpec);
numpy_metas.typekind = dtype;
initializeClassName("StorageNumpy");
}

StorageNumpy() {
HecubaExtrae_event(HECUBAEV, HECUBA_SN|HECUBA_INSTANTIATION);
initObjSpec();
initObjSpec('f'); // Float by default
HecubaExtrae_event(HECUBAEV, HECUBA_END);
}

StorageNumpy(void *datasrc, const std::vector<uint32_t> &metas) {
/* Note: 'dtype' uses the array-interface api which is slightly different from Python
* array-interface:https://numpy.org/doc/1.21/reference/arrays.interface.html#arrays-interface
* python: https://numpy.org/doc/stable/reference/arrays.dtypes.html */
StorageNumpy(void *datasrc, const std::vector<uint32_t> &metas,char dtype='f') { // TODO: change 'dtype' type to string or better add another parameter with the size
HecubaExtrae_event(HECUBAEV, HECUBA_SN|HECUBA_INSTANTIATION);
setNumpy(datasrc, metas);
initObjSpec();
setNumpy(datasrc, metas, dtype);
initObjSpec(dtype);
HecubaExtrae_event(HECUBAEV, HECUBA_END);
}

void setNumpy(void *datasrc, const std::vector<uint32_t>&metas) {
void setNumpy(void *datasrc, const std::vector<uint32_t>&metas, char dtype) {
// Transform user metas to ArrayMetadata
this->metas = metas; // make a copy of user 'metas'
uint32_t numpy_size = extractNumpyMetaData(metas, this->numpy_metas);
uint32_t numpy_size = extractNumpyMetaData(metas, dtype, this->numpy_metas );

// Make a copy of 'datasrc'
if (this->data != nullptr)
Expand All @@ -85,21 +89,22 @@ class StorageNumpy:virtual public IStorage {
//JJ StorageNumpy(src.data, src.metas);
// Transform user metas to ArrayMetadata
this->metas = src.metas; // make a copy of user 'metas'
uint32_t numpy_size = extractNumpyMetaData(src.metas, this->numpy_metas);
uint32_t numpy_size = extractNumpyMetaData(src.metas, src.numpy_metas.typekind, this->numpy_metas);

// Make a copy of 'datasrc'
this->data = malloc(numpy_size);
memcpy(this->data, src.data, numpy_size);

initObjSpec();
initObjSpec(src.numpy_metas.typekind);
HecubaExtrae_event(HECUBAEV, HECUBA_END);
}

// StorageNumpy sn; sn = misn;
StorageNumpy &operator = (const StorageNumpy & w) {
HecubaExtrae_event(HECUBAEV, HECUBA_SN|HECUBA_ASSIGNMENT);
this->metas = w.metas;
uint32_t numpy_size = extractNumpyMetaData(metas, this->numpy_metas);
this->numpy_metas=w.numpy_metas;
uint32_t numpy_size = extractNumpyMetaData(metas, numpy_metas.typekind, this->numpy_metas);
this->data = malloc(numpy_size);
memcpy(this->data, w.data, numpy_size);
HecubaExtrae_event(HECUBAEV, HECUBA_END);
Expand Down Expand Up @@ -215,10 +220,28 @@ class StorageNumpy:virtual public IStorage {

std::shared_ptr<ArrayDataStore> arrayStore = nullptr; /* Cache of written/read elements */

uint32_t extractNumpyMetaData(const std::vector<uint32_t> &raw_numpy_meta, ArrayMetadata &arr_metas) {
uint32_t getDtypeSize(char dtype) const {
switch(dtype) {
case 'f': //NPY_FLOAT
return sizeof(double);
case 'i': //NPY_BYTE
case 'u': //NPY_UBYTE
case 'b': //NPY_BOOL
return sizeof(char);
default: {
std::string msg ("StorageNumpy::getDtypeSize: Unsupported type [");
msg += dtype;
msg += "] ";
throw ModuleException(msg);
}
}
}

uint32_t extractNumpyMetaData(const std::vector<uint32_t> &raw_numpy_meta, char dtype, ArrayMetadata &arr_metas) {
std::vector <uint32_t> dims;
std::vector <uint32_t> strides;

arr_metas.elem_size = getDtypeSize(dtype); // TODO: This should be a parameter!
// decode void *metadatas
uint32_t acum=1;
uint32_t numpy_size=0;
Expand All @@ -229,16 +252,15 @@ class StorageNumpy:virtual public IStorage {
numpy_size = acum;
for (uint32_t i=0; i < raw_numpy_meta.size(); i++) {
acum/=raw_numpy_meta[i];
strides.push_back(acum * sizeof(double));
strides.push_back(acum * arr_metas.elem_size);
}
uint32_t flags=NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_WRITEABLE | NPY_ARRAY_ALIGNED;

arr_metas.dims = dims;
arr_metas.strides = strides;
arr_metas.elem_size = sizeof(double);
arr_metas.flags = flags;
arr_metas.partition_type = ZORDER_ALGORITHM;
arr_metas.typekind = 'f';
arr_metas.typekind = dtype;
arr_metas.byteorder = '=';
return numpy_size*arr_metas.elem_size;
}
Expand Down

0 comments on commit 73b2372

Please sign in to comment.