Dnnc with no finetuning

deeppavlov/configs/classifiers/few_shot_roberta.json

@@ -0,0 +1,56 @@
+{
+  "chainer": {
+    "in": ["texts", "dataset"],
+    "in_y": ["y_true"],
+    "pipe": [
+      {
+        "class_name": "dnnc_pair_generator",
+        "in": ["texts", "dataset"],
+        "out": ["x", "x_support", "x_populated", "y_support"],
+        "bidirectional": true
+      },
+      {
+        "class_name": "torch_transformers_preprocessor",
+        "in": ["x_populated", "x_support"],
+        "out": ["bert_features"],
+        "vocab_file": "{BASE_MODEL}",
+        "do_lower_case": true,
+        "max_seq_length": 128
+      },
+      {
+        "class_name": "torch_transformers_classifier",
+        "main": true,
+        "in": ["bert_features"],
+        "out": ["simmilarity_scores"],
+        "n_classes": 2,
+        "return_probas": true,
+        "pretrained_bert": "{BASE_MODEL}",
+        "save_path": "{MODEL_PATH}/model",
+        "load_path": "{MODEL_PATH}/model",
+        "is_binary": "{BINARY_CLASSIFICATION}"
+      },
+      {
+        "class_name": "dnnc_proba2labels",
+        "is_binary": "{BINARY_CLASSIFICATION}",
+        "in": ["simmilarity_scores", "x", "x_populated", "x_support", "y_support"],
+        "out": ["y_pred"],
+        "confidence_threshold": 0.0
+      }
+    ],
+    "out": ["y_pred"]
+  },
+  "metadata": {
+    "variables": {
+      "ROOT_PATH": "~/.deeppavlov",
+      "MODEL_PATH": "{ROOT_PATH}/models/fewshot/roberta_nli_mrpc_1_10",
+      "BINARY_CLASSIFICATION": true,
+      "BASE_MODEL": "roberta-base"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/v1/classifiers/fewshot/roberta_nli_mrpc_1_10.tar.gz",
+        "subdir": "{MODEL_PATH}"
+      }
+    ]
+  }
+}

deeppavlov/core/common/registry.json

@@ -16,6 +16,8 @@
   "dirty_comments_preprocessor": "deeppavlov.models.preprocessors.dirty_comments_preprocessor:DirtyCommentsPreprocessor",
   "docred_reader": "deeppavlov.dataset_readers.docred_reader:DocREDDatasetReader",
   "document_chunker": "deeppavlov.models.preprocessors.odqa_preprocessors:DocumentChunker",
+  "dnnc_pair_generator": "deeppavlov.models.preprocessors.dnnc_preprocessor:PairGenerator",
+  "dnnc_proba2labels": "deeppavlov.models.classifiers.dnnc_proba2labels:Proba2Labels",
   "entity_detection_parser": "deeppavlov.models.entity_extraction.entity_detection_parser:EntityDetectionParser",
   "entity_linker": "deeppavlov.models.entity_extraction.entity_linking:EntityLinker",
   "entity_type_split": "deeppavlov.models.entity_extraction.entity_detection_parser:entity_type_split",

deeppavlov/models/classifiers/dnnc_proba2labels.py

@@ -0,0 +1,90 @@
+# Copyright 2017 Neural Networks and Deep Learning lab, MIPT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from logging import getLogger
+from typing import List
+
+import numpy as np
+
+from deeppavlov.core.common.registry import register
+from deeppavlov.core.models.component import Component
+
+log = getLogger(__name__)
+
+
+@register('dnnc_proba2labels')
+class Proba2Labels(Component):
+    """
+    Converts pairwise simmilarity scores into class label
+
+    Args:
+        confidence_threshold: used to determine whether example belongs to one 
+                              of the classes in 'y_support' or not
+        pooling: strategy for averaging similarity scores for each label
+        is_binary: determines whether the similarity is a number or a probability vector
+    """
+
+    def __init__(self,
+                 confidence_threshold: float = 0.0,
+                 pooling: str = 'max',
+                 is_binary: bool = True,
+                 **kwargs) -> None:
+
+        self.confidence_threshold = confidence_threshold
+        self.pooling = pooling
+        self.is_binary = is_binary
+
+    def __call__(self,
+                 simmilarity_scores: List[float],
+                 x: List[str],
+                 x_populated: List[str],
+                 x_support: List[str],
+                 y_support: List[str]
+                ) -> List[str]:
+
+        y_pred = []
+
+        simmilarity_scores = np.array(simmilarity_scores)
+        x_populated = np.array(x_populated)
+        x_support = np.array(x_support)
+        y_support = np.array(y_support)
+        unique_labels = np.unique(y_support)
+
+        # Transform probits vector into a simmilarity score
+        if not self.is_binary:
+            simmilarity_scores = simmilarity_scores[:, 1]
+
+        for example in x:
+            example_mask = np.where(np.logical_xor(x_populated == example, x_support == example))
+            example_simmilarity_scores = simmilarity_scores[example_mask]
+            example_y_support = y_support[example_mask]
+
+            probability_by_label = []
+            for label in unique_labels:
+                label_mask = np.where(example_y_support == label)
+                label_simmilarity_scores = example_simmilarity_scores[label_mask]
+                if self.pooling == 'avg':
+                    label_probability = np.mean(label_simmilarity_scores)
+                elif self.pooling == 'max':
+                    label_probability = np.max(label_simmilarity_scores)
+                probability_by_label.append(label_probability)
+
+            probability_by_label = np.array(probability_by_label)
+            max_probability = max(probability_by_label)
+            max_probability_label = unique_labels[np.argmax(probability_by_label)]
+            prediction = "oos" if max_probability < self.confidence_threshold else max_probability_label
+
+            y_pred.append(prediction)
+
+        return y_pred

deeppavlov/models/preprocessors/dnnc_preprocessor.py

@@ -0,0 +1,55 @@
+# Copyright 2017 Neural Networks and Deep Learning lab, MIPT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from logging import getLogger
+from typing import List, Tuple
+
+import numpy as np
+
+from deeppavlov.core.common.registry import register
+from deeppavlov.core.models.component import Component
+
+log = getLogger(__name__)
+
+
+@register('dnnc_pair_generator')
+class PairGenerator(Component):
+    """
+    Generates all possible ordered pairs from 'texts_batch' and 'support_dataset'
+
+    Args:
+        bidirectional: adds pairs in reverse order
+    """
+
+    def __init__(self, bidirectional: bool = False, **kwargs) -> None:
+        self.bidirectional = bidirectional
+
+    def __call__(self,
+                 texts: List[str],
+                 dataset: List[List[str]],
+                ) -> Tuple[List[str], List[str], List[str], List[str]]:
+        hypotesis_batch = []
+        premise_batch = []
+        hypotesis_labels_batch = []
+        for [premise, [hypotesis, hypotesis_labels]] in zip(texts * len(dataset),
+                                                            np.repeat(dataset, len(texts), axis=0)):
+            premise_batch.append(premise)
+            hypotesis_batch.append(hypotesis)
+            hypotesis_labels_batch.append(hypotesis_labels)
+
+            if self.bidirectional:
+                premise_batch.append(hypotesis)
+                hypotesis_batch.append(premise)
+                hypotesis_labels_batch.append(hypotesis_labels)
+        return texts, hypotesis_batch, premise_batch, hypotesis_labels_batch