3.0

.circleci/config.yml

@@ -1,61 +1,19 @@
-version: 2
+version: 2.1
+
+orbs:
+  python: circleci/python@0.2.1
+
 jobs:
-  test_api_and_main_and_upload:
-    docker:
-      - image: circleci/python
+  build-and-test:
+    executor: python/default
     steps:
       - checkout
-      - run: 
-          name: init .pypirc
-          command: |
-            echo -e "[pypi]" >> ~/.pypirc
-      - run:
-          name: install requirements
-          command: |
-            sudo apt-get install -y libblas3 liblapack3
-            sudo apt-get install -y liblapack-dev libblas-dev
-            cd /home/circleci/project/
-            pip3 install --user -r requirements.txt
-      - run:
-          name: test main
-          command: |
-            cd /home/circleci/project/
-            python3 tests/main_tests.py 
-      - run:
-          name: test api
-          command: |
-            cd /home/circleci/project/
-            python3 tests/api_tests.py
-      - run:
-          name: create packages
-          command: |
-            python3 setup.py sdist
-            python3 setup.py bdist_wheel
-      - run:
-          name: upload to pypi
-          command: |
-            if [[ "$PYPI_USERNAME" == "" ]]; then
-              echo "Skip upload"
-              exit 0
-            fi
-            python3 -m pip install --user jq
-            if [[ "$CIRCLE_BRANCH" == "master" ]]; then
-              PYPI="pypi.org"
-            else
-              PYPI="test.pypi.org"
-            fi
-            LATEST_VERSION="$(curl -s https://$PYPI/pypi/nlpcube/json | jq -r '.info.version')"
-            THIS_VERSION=`python3 <<< "import pkg_resources;print(pkg_resources.require('nlpcube')[0].version)"`
-            if [[ $THIS_VERSION != $LATEST_VERSION ]]; then
-              echo "\n\nthis: $THIS_VERSION - latest: $LATEST_VERSION => releasing to $PYPI\n\n"
-              python3 -m pip install --user --upgrade twine
-              python3 -m twine upload --repository-url https://$PYPI/legacy/ dist/* -u $PYPI_USERNAME -p $PYPI_PASSWORD || echo "Package already exists"
-            else
-              echo "this: $THIS_VERSION = latest: $LATEST_VERSION => skip release"
-            fi
+      - python/load-cache
+      - python/install-deps
+      - python/save-cache
+      - run: echo "done"
 
 workflows:
-  version: 2
-  test_api_and_main_and_upload:
+  main:
     jobs:
-      - test_api_and_main_and_upload
+      - build-and-test

.gitignore

@@ -1,3 +1,15 @@
+.DS_Store
+Languasito/data/
+*.txt
+lightning_logs
+*.gz
+*.encodings
+*.npy
+data/*
+nlp-cube-models/*
+corpus/
+models/
+scripts/packer
 *.pyc
 build/
 dist/
@@ -11,12 +23,14 @@ cube/venv/*
 .idea/*
 venv/*
 cube/*.py
+*.json
 
-models/
+scratch/
 tests/scratch/*
 scripts/*.json
 scripts/*.conllu
 scripts/*.md
+scripts/wikiextractor.py
 
 # Jupyter notebooks
 notebooks/.ipynb_checkpoints/*

Languasito/languasito/api.py

@@ -0,0 +1,63 @@
+import sys
+import torch
+from typing import *
+
+sys.path.append('')
+
+from languasito.model import Languasito
+from languasito.utils import LanguasitoCollate
+from languasito.utils import Encodings
+
+
+class LanguasitoAPI:
+
+    def __init__(self, languasito: Languasito, encodings: Encodings):
+        self._languasito = languasito
+        self._languasito.eval()
+        self._encodings = encodings
+        self._collate = LanguasitoCollate(encodings, live=True)
+        self._device = 'cpu'
+
+    def to(self, device: str):
+        self._languasito.to(device)
+        self._device = device
+
+    def __call__(self, batch):
+        with torch.no_grad():
+            x = self._collate.collate_fn(batch)
+            for key in x:
+                if isinstance(x[key], torch.Tensor):
+                    x[key] = x[key].to(self._device)
+            rez = self._languasito(x)
+        emb = []
+        pred_emb = rez['emb'].detach().cpu().numpy()
+        for ii in range(len(batch)):
+            c_emb = []
+            for jj in range(len(batch[ii])):
+                c_emb.append(pred_emb[ii, jj])
+            emb.append(c_emb)
+        return emb
+
+    @staticmethod
+    def load(model_name: str):
+        from pathlib import Path
+        home = str(Path.home())
+        filename = '{0}/.languasito/{1}'.format(home, model_name)
+        import os
+        if os.path.exists(filename + '.encodings'):
+            return LanguasitoAPI.load_local(filename)
+        else:
+            print("UserWarning: Model not found and automatic downloading is not yet supported")
+            return None
+
+    @staticmethod
+    def load_local(model_name: str):
+        enc = Encodings()
+        enc.load('{0}.encodings'.format(model_name))
+        model = Languasito(enc)
+        tmp = torch.load('{0}.best'.format(model_name), map_location='cpu')
+        # model.load(tmp['state_dict'])
+        model.load_state_dict(tmp['state_dict'])
+        model.eval()
+        api = LanguasitoAPI(model, enc)
+        return api