Refactor Coval Scoring code

licenses/3rd_party_licenses.txt

@@ -127,3 +127,36 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+
+coval
+-----
+
+* Files: scorer.py
+
+The implementations of ClusterEvaluator, lea, get_cluster_info, and
+get_markable_assignments are adapted from coval, which is distributed
+under the following license:
+
+The MIT License (MIT)
+
+Copyright 2018 Nafise Sadat Moosavi (ns.moosavi at gmail dot com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

spacy/ml/models/coref.py

@@ -53,10 +53,13 @@ def convert_coref_clusterer_inputs(model: Model, X: List[Floats2d], is_train: bo
     X = X[0]
     word_features = xp2torch(X, requires_grad=is_train)
 
-    def backprop(args: ArgsKwargs) -> List[Floats2d]:
+    # TODO fix or remove type annotations
+    def backprop(args: ArgsKwargs): #-> List[Floats2d]:
         # convert to xp and wrap in list
         gradients = torch2xp(args.args[0])
-        assert isinstance(gradients, Floats2d)
+        #TODO why did this change? This was fine before merging master.
+        # It's still 2d but this assert fails.
+        #assert isinstance(gradients, Floats2d)
         return [gradients]
 
     return ArgsKwargs(args=(word_features,), kwargs={}), backprop

spacy/ml/models/coref_util.py

@@ -32,23 +32,6 @@ def get_sentence_ids(doc):
     return out
 
 
-def doc2clusters(doc: Doc, prefix=DEFAULT_CLUSTER_PREFIX) -> MentionClusters:
-    """Given a doc, give the mention clusters.
-
-    This is useful for scoring.
-    """
-    out = []
-    for name, val in doc.spans.items():
-        if not name.startswith(prefix):
-            continue
-
-        cluster = []
-        for mention in val:
-            cluster.append((mention.start, mention.end))
-        out.append(cluster)
-    return out
-
-
 # from model.py, refactored to be non-member
 def get_predicted_antecedents(xp, antecedent_idx, antecedent_scores):
     """Get the ID of the antecedent for each span. -1 if no antecedent."""

spacy/ml/models/span_predictor.py

@@ -48,23 +48,24 @@ def build_span_predictor(
 
 
 def convert_span_predictor_inputs(
-    model: Model, X: Tuple[Ints1d, Tuple[Floats2d, Ints1d]], is_train: bool
+    model: Model, X: Tuple[List[Floats2d], Tuple[List[Ints1d], List[Ints1d]]], is_train: bool
 ):
     tok2vec, (sent_ids, head_ids) = X
     # Normally we should use the input is_train, but for these two it's not relevant
-
-    def backprop(args: ArgsKwargs) -> List[Floats2d]:
+    # TODO fix the type here, or remove it
+    def backprop(args: ArgsKwargs): #-> Tuple[List[Floats2d], None]:
         gradients = torch2xp(args.args[1])
+        # The sent_ids and head_ids are None because no gradients
         return [[gradients], None]
 
     word_features = xp2torch(tok2vec[0], requires_grad=is_train)
-    sent_ids = xp2torch(sent_ids[0], requires_grad=False)
+    sent_ids_tensor = xp2torch(sent_ids[0], requires_grad=False)
     if not head_ids[0].size:
-        head_ids = torch.empty(size=(0,))
+        head_ids_tensor = torch.empty(size=(0,))
     else:
-        head_ids = xp2torch(head_ids[0], requires_grad=False)
+        head_ids_tensor = xp2torch(head_ids[0], requires_grad=False)
 
-    argskwargs = ArgsKwargs(args=(sent_ids, word_features, head_ids), kwargs={})
+    argskwargs = ArgsKwargs(args=(sent_ids_tensor, word_features, head_ids_tensor), kwargs={})
     return argskwargs, backprop
 
 

spacy/pipeline/coref.py

@@ -13,6 +13,7 @@
 from ..errors import Errors
 from ..tokens import Doc
 from ..vocab import Vocab
+from ..util import registry
 
 from ..ml.models.coref_util import (
     create_gold_scores,
@@ -21,10 +22,9 @@
     get_clusters_from_doc,
     get_predicted_clusters,
     DEFAULT_CLUSTER_PREFIX,
-    doc2clusters,
 )
 
-from ..coref_scorer import Evaluator, get_cluster_info, lea
+from ..scorer import Scorer
 
 
 default_config = """
@@ -56,7 +56,14 @@
 """
 DEFAULT_COREF_MODEL = Config().from_str(default_config)["model"]
 
-DEFAULT_CLUSTERS_PREFIX = "coref_clusters"
+
+def coref_scorer(examples: Iterable[Example], **kwargs) -> Dict[str, Any]:
+    return Scorer.score_coref_clusters(examples, **kwargs)
+
+
+@registry.scorers("spacy.coref_scorer.v1")
+def make_coref_scorer():
+    return coref_scorer
 
 
 @Language.factory(
@@ -66,19 +73,21 @@
     default_config={
         "model": DEFAULT_COREF_MODEL,
         "span_cluster_prefix": DEFAULT_CLUSTER_PREFIX,
+        "scorer": {"@scorers": "spacy.coref_scorer.v1"},
     },
     default_score_weights={"coref_f": 1.0, "coref_p": None, "coref_r": None},
 )
 def make_coref(
     nlp: Language,
     name: str,
     model,
-    span_cluster_prefix: str = "coref",
+    scorer: Optional[Callable],
+    span_cluster_prefix: str,
 ) -> "CoreferenceResolver":
     """Create a CoreferenceResolver component."""
 
     return CoreferenceResolver(
-        nlp.vocab, model, name, span_cluster_prefix=span_cluster_prefix
+        nlp.vocab, model, name, span_cluster_prefix=span_cluster_prefix, scorer=scorer
     )
 
 
@@ -95,7 +104,8 @@ def __init__(
         name: str = "coref",
         *,
         span_mentions: str = "coref_mentions",
-        span_cluster_prefix: str,
+        span_cluster_prefix: str = DEFAULT_CLUSTER_PREFIX,
+        scorer: Optional[Callable] = coref_scorer,
     ) -> None:
         """Initialize a coreference resolution component.
 
@@ -117,7 +127,8 @@ def __init__(
         self.span_cluster_prefix = span_cluster_prefix
         self._rehearsal_model = None
 
-        self.cfg: Dict[str, Any] = {}
+        self.cfg: Dict[str, Any] = {"span_cluster_prefix": span_cluster_prefix}
+        self.scorer = scorer
 
     def predict(self, docs: Iterable[Doc]) -> List[MentionClusters]:
         """Apply the pipeline's model to a batch of docs, without modifying them.
@@ -275,7 +286,6 @@ def get_loss(
             log_marg = ops.softmax(score_matrix + ops.xp.log(top_gscores), axis=1)
         log_norm = ops.softmax(score_matrix, axis=1)
         grad = log_norm - log_marg
-        # gradients.append((grad, cidx))
         loss = float((grad**2).sum())
 
         return loss, grad
@@ -305,26 +315,3 @@ def initialize(
 
         assert len(X) > 0, Errors.E923.format(name=self.name)
         self.model.initialize(X=X, Y=Y)
-
-    def score(self, examples, **kwargs):
-        """Score a batch of examples using LEA.
-        For details on how LEA works and why to use it see the paper:
-        Which Coreference Evaluation Metric Do You Trust? A Proposal for a Link-based Entity Aware Metric
-        Moosavi and Strube, 2016
-        https://api.semanticscholar.org/CorpusID:17606580
-        """
-
-        evaluator = Evaluator(lea)
-
-        for ex in examples:
-            p_clusters = doc2clusters(ex.predicted, self.span_cluster_prefix)
-            g_clusters = doc2clusters(ex.reference, self.span_cluster_prefix)
-            cluster_info = get_cluster_info(p_clusters, g_clusters)
-            evaluator.update(cluster_info)
-
-        score = {
-            "coref_f": evaluator.get_f1(),
-            "coref_p": evaluator.get_precision(),
-            "coref_r": evaluator.get_recall(),
-        }
-        return score