Skip to content

Commit

Permalink
upgrade json library
Browse files Browse the repository at this point in the history 
Signed-off-by: Dusan Malusev <dusan@dusanmalusev.dev>
  • Loading branch information
@CodeLieutenant
CodeLieutenant committed Jul 6, 2023
1 parent 42560b2 commit 595ce4a
Show file tree
Hide file tree
Showing 5 changed files with 24,712 additions and 17,291 deletions.
24 changes: 13 additions & 11 deletions .vscode/settings.json
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
{
"files.associations": {
"*.mc": "cpp",
"*.cu": "cpp",
"*.m": "mathematica",
"*.mt": "mathematica",
"*.hpp": "cpp",
"*.cuh": "cuda",
"__locale": "cpp",
"ios": "cpp",
"memory": "cpp",
"utility": "cpp"
"*main.yml": "ansible",
"*.mc": "cpp",
"*.cu": "cpp",
"*.m": "mathematica",
"*.mt": "mathematica",
"*.hpp": "cpp",
"*.cuh": "cuda",
"__locale": "cpp",
"ios": "cpp",
"memory": "cpp",
"utility": "cpp",
"iomanip": "cpp"
}
}
}
189 changes: 102 additions & 87 deletions cbits.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,119 +3,134 @@
#include <memory>
#include <streambuf>

#include "cbits.h"
#include "fasttext.h"
#include "real.h"

#include "args.cc"
#include "dictionary.cc"
#include "fasttext.cc"
#include "matrix.cc"
#include "model.cc"
#include "productquantizer.cc"
#include "quantmatrix.cc"
#include "utils.cc"
#include "vector.cc"
#include <json.hpp>

#include <fasttext.h>
#include <real.h>

#include <args.cc>
#include <autotune.cc>
#include <dictionary.cc>
#include <fasttext.cc>
#include <matrix.cc>
#include <meter.cc>
#include <model.cc>
#include <productquantizer.cc>
#include <quantmatrix.cc>
#include <utils.cc>
#include <vector.cc>

#include "json.hpp"
#include "cbits.h"

using json = nlohmann::json;

struct membuf : std::streambuf {
membuf(char *begin, char *end) { this->setg(begin, begin, end); }
struct membuf : std::streambuf
{
membuf(char *begin, char *end)
{
this->setg(begin, begin, end);
}
};

template <class Dest, class Source> inline Dest bit_cast(Source const &source) {
static_assert(sizeof(Dest) == sizeof(Source),
"size of destination and source objects must be equal");
static_assert(std::is_trivially_copyable<Dest>::value,
"destination type must be trivially copyable.");
static_assert(std::is_trivially_copyable<Source>::value,
"source type must be trivially copyable");

Dest dest;
std::memcpy(&dest, &source, sizeof(dest));
return dest;
}
template <class Dest, class Source> inline Dest bit_cast(Source const &source)
{
static_assert(sizeof(Dest) == sizeof(Source), "size of destination and source objects must be equal");
static_assert(std::is_trivially_copyable<Dest>::value, "destination type must be trivially copyable.");
static_assert(std::is_trivially_copyable<Source>::value, "source type must be trivially copyable");

FastTextHandle NewHandle(const char *path) {
auto model = new fasttext::FastText();
model->loadModel(std::string(path));
return bit_cast<FastTextHandle>(model);
Dest dest;
std::memcpy(&dest, &source, sizeof(dest));
return dest;
}

void DeleteHandle(FastTextHandle handle) {
auto model = bit_cast<fasttext::FastText *>(handle);
if (model != nullptr) {
delete model;
}
FastTextHandle NewHandle(const char *path)
{
auto model = new fasttext::FastText();
model->loadModel(std::string(path));
return bit_cast<FastTextHandle>(model);
}

char *Predict(FastTextHandle handle, char *query) {
auto model = bit_cast<fasttext::FastText *>(handle);

membuf sbuf(query, query + strlen(query));
std::istream in(&sbuf);

std::vector<std::pair<fasttext::real, std::string>> predictions;
model->predictLine(in, predictions, 1, 0.0f);

size_t ii = 0;
auto res = json::array();
for (const auto it : predictions) {
float p = std::exp(it.first);
res.push_back({
{"index", ii++},
{"probability", p},
{"label", it.second},
});
}
void DeleteHandle(FastTextHandle handle)
{
auto model = bit_cast<fasttext::FastText *>(handle);
if (model != nullptr)
{
delete model;
}
}

return strdup(res.dump().c_str());
char *Predict(FastTextHandle handle, char *query)
{
auto model = bit_cast<fasttext::FastText *>(handle);

membuf sbuf(query, query + strlen(query));
std::istream in(&sbuf);

std::vector<std::pair<fasttext::real, std::string>> predictions;
model->predictLine(in, predictions, 1, 0.0f);

size_t ii = 0;
auto res = json::array();
for (const auto it : predictions)
{
float p = std::exp(it.first);
res.push_back({
{"index", ii++},
{"probability", p},
{"label", it.second},
});
}

return strdup(res.dump().c_str());
}

char *Analogy(FastTextHandle handle, char *query) {
auto model = bit_cast<fasttext::FastText *>(handle);
char *Analogy(FastTextHandle handle, char *query)
{
return "";

// auto model = bit_cast<fasttext::FastText *>(handle);

model->getAnalogies(1, query, 10);
// model->getAnalogies(1, query, 10);

size_t ii = 0;
auto res = json::array();
// size_t ii = 0;
// auto res = json::array();

return strdup(res.dump().c_str());
// return strdup(res.dump().c_str());
}

char *Wordvec(FastTextHandle handle, char *query) {
auto model = bit_cast<fasttext::FastText *>(handle);
char *Wordvec(FastTextHandle handle, char *query)
{
auto model = bit_cast<fasttext::FastText *>(handle);

fasttext::Vector vec(model->getDimension());
// fasttext::Matrix wordVectors(model->dict_->nwords(), model->getDimension());
// model->precomputeWordVectors(wordVectors);
model->getWordVector(vec, query);
fasttext::Vector vec(model->getDimension());
// fasttext::Matrix wordVectors(model->dict_->nwords(), model->getDimension());
// model->precomputeWordVectors(wordVectors);
model->getWordVector(vec, query);

auto res = json::array();
for (int i = 0; i < vec.size(); i++) {
res.push_back(vec[i]);
}
auto res = json::array();
for (int i = 0; i < vec.size(); i++)
{
res.push_back(vec[i]);
}

return strdup(res.dump().c_str());
return strdup(res.dump().c_str());
}

char *Sentencevec(FastTextHandle handle, char *query) {
auto model = bit_cast<fasttext::FastText *>(handle);
char *Sentencevec(FastTextHandle handle, char *query)
{
auto model = bit_cast<fasttext::FastText *>(handle);

membuf sbuf(query, query + strlen(query));
std::istream in(&sbuf);
membuf sbuf(query, query + strlen(query));
std::istream in(&sbuf);

fasttext::Vector vec(model->getDimension());
// fasttext::Matrix wordVectors(model->dict_->nwords(), model->getDimension());
// model->precomputeWordVectors(wordVectors);
model->getSentenceVector(in, vec);
fasttext::Vector vec(model->getDimension());
model->getSentenceVector(in, vec);

auto res = json::array();
for (int i = 0; i < vec.size(); i++) {
res.push_back(vec[i]);
}
auto res = json::array();
for (int i = 0; i < vec.size(); i++)
{
res.push_back(vec[i]);
}

return strdup(res.dump().c_str());
return strdup(res.dump().c_str());
}
4 changes: 1 addition & 3 deletions fasttext.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package fasttext

// #cgo CXXFLAGS: -I${SRCDIR}/fastText/src -I${SRCDIR} -std=c++20 -O3 -fPIC
// #cgo CXXFLAGS: -I${SRCDIR}/fastText/src -I${SRCDIR} -I${SRCDIR}/include -std=c++17 -O3 -fPIC
// #cgo LDFLAGS: -lstdc++
// #include <stdio.h>
// #include <stdlib.h>
Expand All @@ -16,7 +16,6 @@ import (
// around the C fasttext handle
type Model struct {
handle C.FastTextHandle
path string
}

// Opens a model from a path and returns a model
Expand All @@ -30,7 +29,6 @@ func Open(path string) *Model {
defer C.free(unsafe.Pointer(cpath))

return &Model{
path: path,
handle: C.NewHandle(cpath),
}
}
Expand Down
Loading

0 comments on commit 595ce4a

Please sign in to comment.