From ff3151c31c44bf72e2339671b68d573b03d76984 Mon Sep 17 00:00:00 2001 From: Luncenok Date: Sat, 14 Dec 2024 01:26:05 +0100 Subject: [PATCH] Krippendorff's Alpha --- data/annotation_comparison_detailed.csv | 26 ++++++++-------- .../compare_annotations_krippendorff.py | 30 +++++++++---------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/data/annotation_comparison_detailed.csv b/data/annotation_comparison_detailed.csv index cba8993..a430092 100644 --- a/data/annotation_comparison_detailed.csv +++ b/data/annotation_comparison_detailed.csv @@ -1,19 +1,19 @@ Comparison,Technique,Krippendorff Alpha,Label Agreement +human_vs_annotations,Appeal to Urgency,0.3467566915842778,0.9639175257731959 +human_vs_annotations,Bandwagon Effect,0.41891891891891886,0.9484536082474226 +human_vs_annotations,Appeal to Emotion,-0.010443864229765065,0.9742268041237113 +human_vs_annotations,Deception,-0.023809523809523947,0.9484536082474226 +human_vs_annotations,Lying,-0.03200000000000003,0.9329896907216495 +human_vs_annotations,Appeal to Rules,-0.0078125,0.979381443298969 human_vs_annotations,Distraction,0.21756329113924044,0.7628865979381443 +human_vs_annotations,Confirmation Bias Exploitation,0.2744656917885264,0.9742268041237113 +human_vs_annotations,Projection,0.20901462994836484,0.8041237113402062 +human_vs_annotations,Appeal to Credibility,0.4218226778630674,0.8505154639175257 human_vs_annotations,Appeal to Logic,0.48053691275167787,0.8144329896907216 +human_vs_annotations,Minimization,0.0,0.9948453608247423 +human_vs_annotations,Gaslighting,-0.054495912806539426,0.8917525773195877 human_vs_annotations,Withholding Information,0.0,0.9948453608247423 -human_vs_annotations,Confirmation Bias Exploitation,0.2744656917885264,0.9742268041237113 -human_vs_annotations,Feigning Ignorance,0.0,0.9948453608247423 -human_vs_annotations,Strategic Voting Suggestion,0.7336911643270025,0.9484536082474226 -human_vs_annotations,Appeal to Rules,-0.0078125,0.979381443298969 -human_vs_annotations,Appeal to Emotion,-0.010443864229765065,0.9742268041237113 -human_vs_annotations,Appeal to Urgency,0.3467566915842778,0.9639175257731959 human_vs_annotations,Shifting the Burden of Proof,0.43311190597853866,0.7783505154639175 -human_vs_annotations,Lying,-0.03200000000000003,0.9329896907216495 -human_vs_annotations,Bandwagon Effect,0.41891891891891886,0.9484536082474226 -human_vs_annotations,Gaslighting,-0.054495912806539426,0.8917525773195877 -human_vs_annotations,Projection,0.20901462994836484,0.8041237113402062 +human_vs_annotations,Feigning Ignorance,0.0,0.9948453608247423 human_vs_annotations,Vagueness,-0.01308900523560208,0.9690721649484536 -human_vs_annotations,Minimization,0.0,0.9948453608247423 -human_vs_annotations,Appeal to Credibility,0.4218226778630674,0.8505154639175257 -human_vs_annotations,Deception,-0.023809523809523947,0.9484536082474226 +human_vs_annotations,Strategic Voting Suggestion,0.7336911643270025,0.9484536082474226 diff --git a/src/among_them/analysis/compare_annotations_krippendorff.py b/src/among_them/analysis/compare_annotations_krippendorff.py index 65a708b..a1eca68 100644 --- a/src/among_them/analysis/compare_annotations_krippendorff.py +++ b/src/among_them/analysis/compare_annotations_krippendorff.py @@ -55,28 +55,26 @@ def calculate_label_agreement(annotations1: Dict[str, Set[str]], annotations2: D def calculate_krippendorff_alpha(annotations1: Dict[str, Set[str]], annotations2: Dict[str, Set[str]], common_texts: Set[str]) -> float: """ - Calculate Krippendorff's alpha for multi-label annotations. + Calculate Krippendorff's alpha for multi-label annotations using matrix reshaping. Each text-label pair is treated as a separate coding decision. """ # Get all unique annotation labels - all_annotations = get_all_unique_annotations(annotations1, annotations2) + all_annotations = list(get_all_unique_annotations(annotations1, annotations2)) + common_texts = list(common_texts) - # Create reliability data matrix - # Each row represents one unit (text-annotation pair) - # Each column represents one coder - reliability_data = [] + # Create a 3D matrix: (texts x labels x annotators) + num_texts = len(common_texts) + num_labels = len(all_annotations) + annotations_matrix = np.zeros((num_texts, num_labels, 2), dtype=int) - for text in common_texts: - for annotation in all_annotations: - # For each text-annotation pair, create a row with two coders' decisions - reliability_data.append([ - 1 if annotation in annotations1[text] else 0, - 1 if annotation in annotations2[text] else 0 - ]) + # Fill the matrix + for i, text in enumerate(common_texts): + for j, label in enumerate(all_annotations): + annotations_matrix[i, j, 0] = 1 if label in annotations1[text] else 0 + annotations_matrix[i, j, 1] = 1 if label in annotations2[text] else 0 - # Convert to numpy array and transpose to get format expected by krippendorff.alpha - reliability_data = np.array(reliability_data).T - # print(reliability_data) + # Reshape to (labels x texts x annotators) format expected by krippendorff.alpha + reliability_data = annotations_matrix.transpose(1, 0, 2).reshape(num_labels * num_texts, 2).T try: # Calculate Krippendorff's alpha with nominal metric (since we have binary data)