Krippendorff's Alpha

data/annotation_comparison_detailed.csv

@@ -1,19 +1,19 @@
 Comparison,Technique,Krippendorff Alpha,Label Agreement
+human_vs_annotations,Appeal to Urgency,0.3467566915842778,0.9639175257731959
+human_vs_annotations,Bandwagon Effect,0.41891891891891886,0.9484536082474226
+human_vs_annotations,Appeal to Emotion,-0.010443864229765065,0.9742268041237113
+human_vs_annotations,Deception,-0.023809523809523947,0.9484536082474226
+human_vs_annotations,Lying,-0.03200000000000003,0.9329896907216495
+human_vs_annotations,Appeal to Rules,-0.0078125,0.979381443298969
 human_vs_annotations,Distraction,0.21756329113924044,0.7628865979381443
+human_vs_annotations,Confirmation Bias Exploitation,0.2744656917885264,0.9742268041237113
+human_vs_annotations,Projection,0.20901462994836484,0.8041237113402062
+human_vs_annotations,Appeal to Credibility,0.4218226778630674,0.8505154639175257
 human_vs_annotations,Appeal to Logic,0.48053691275167787,0.8144329896907216
+human_vs_annotations,Minimization,0.0,0.9948453608247423
+human_vs_annotations,Gaslighting,-0.054495912806539426,0.8917525773195877
 human_vs_annotations,Withholding Information,0.0,0.9948453608247423
-human_vs_annotations,Confirmation Bias Exploitation,0.2744656917885264,0.9742268041237113
-human_vs_annotations,Feigning Ignorance,0.0,0.9948453608247423
-human_vs_annotations,Strategic Voting Suggestion,0.7336911643270025,0.9484536082474226
-human_vs_annotations,Appeal to Rules,-0.0078125,0.979381443298969
-human_vs_annotations,Appeal to Emotion,-0.010443864229765065,0.9742268041237113
-human_vs_annotations,Appeal to Urgency,0.3467566915842778,0.9639175257731959
 human_vs_annotations,Shifting the Burden of Proof,0.43311190597853866,0.7783505154639175
-human_vs_annotations,Lying,-0.03200000000000003,0.9329896907216495
-human_vs_annotations,Bandwagon Effect,0.41891891891891886,0.9484536082474226
-human_vs_annotations,Gaslighting,-0.054495912806539426,0.8917525773195877
-human_vs_annotations,Projection,0.20901462994836484,0.8041237113402062
+human_vs_annotations,Feigning Ignorance,0.0,0.9948453608247423
 human_vs_annotations,Vagueness,-0.01308900523560208,0.9690721649484536
-human_vs_annotations,Minimization,0.0,0.9948453608247423
-human_vs_annotations,Appeal to Credibility,0.4218226778630674,0.8505154639175257
-human_vs_annotations,Deception,-0.023809523809523947,0.9484536082474226
+human_vs_annotations,Strategic Voting Suggestion,0.7336911643270025,0.9484536082474226

src/among_them/analysis/compare_annotations_krippendorff.py

@@ -55,28 +55,26 @@ def calculate_label_agreement(annotations1: Dict[str, Set[str]], annotations2: D
 def calculate_krippendorff_alpha(annotations1: Dict[str, Set[str]], annotations2: Dict[str, Set[str]], 
                               common_texts: Set[str]) -> float:
     """
-    Calculate Krippendorff's alpha for multi-label annotations.
+    Calculate Krippendorff's alpha for multi-label annotations using matrix reshaping.
     Each text-label pair is treated as a separate coding decision.
     """
     # Get all unique annotation labels
-    all_annotations = get_all_unique_annotations(annotations1, annotations2)
+    all_annotations = list(get_all_unique_annotations(annotations1, annotations2))
+    common_texts = list(common_texts)
 
-    # Create reliability data matrix
-    # Each row represents one unit (text-annotation pair)
-    # Each column represents one coder
-    reliability_data = []
+    # Create a 3D matrix: (texts x labels x annotators)
+    num_texts = len(common_texts)
+    num_labels = len(all_annotations)
+    annotations_matrix = np.zeros((num_texts, num_labels, 2), dtype=int)
 
-    for text in common_texts:
-        for annotation in all_annotations:
-            # For each text-annotation pair, create a row with two coders' decisions
-            reliability_data.append([
-                1 if annotation in annotations1[text] else 0,
-                1 if annotation in annotations2[text] else 0
-            ])
+    # Fill the matrix
+    for i, text in enumerate(common_texts):
+        for j, label in enumerate(all_annotations):
+            annotations_matrix[i, j, 0] = 1 if label in annotations1[text] else 0
+            annotations_matrix[i, j, 1] = 1 if label in annotations2[text] else 0
 
-    # Convert to numpy array and transpose to get format expected by krippendorff.alpha
-    reliability_data = np.array(reliability_data).T
-    # print(reliability_data)
+    # Reshape to (labels x texts x annotators) format expected by krippendorff.alpha
+    reliability_data = annotations_matrix.transpose(1, 0, 2).reshape(num_labels * num_texts, 2).T
 
     try:
         # Calculate Krippendorff's alpha with nominal metric (since we have binary data)