Skip to content

Commit

Permalink
Closes #25 | Create dataset loader for Typhoon Yolanda Tweets (#56)
Browse files Browse the repository at this point in the history
* Typhoon Yolanda Tweets dataloader

* Create __init__.py

* Update seacrowd/sea_datasets/typhoon_yolanda_tweets/typhoon_yolanda_tweets.py

Co-authored-by: James Jaya <2089265+jamesjaya@users.noreply.github.com>

* Update typhoon_yolanda_tweets.py

Updated according to comments.
Please tell me if there are something else that I miss.

* Update typhoon_yolanda_tweets.py

removed "TODO" and extra newlines

---------

Co-authored-by: James Jaya <2089265+jamesjaya@users.noreply.github.com>
  • Loading branch information
IvanHalimP and jamesjaya authored Nov 23, 2023
1 parent 3c325c8 commit 4922146
Show file tree
Hide file tree
Showing 2 changed files with 134 additions and 0 deletions.
1 change: 1 addition & 0 deletions seacrowd/sea_datasets/typhoon_yolanda_tweets/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@

133 changes: 133 additions & 0 deletions seacrowd/sea_datasets/typhoon_yolanda_tweets/typhoon_yolanda_tweets.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
import os
from pathlib import Path
from typing import Dict, List, Tuple

import datasets
import pandas as pd

from seacrowd.utils import schemas
from seacrowd.utils.configs import SEACrowdConfig
from seacrowd.utils.constants import Licenses, Tasks

_CITATION = """\
@misc{imperial2019sentiment,
title={Sentiment Analysis of Typhoon Related Tweets using Standard and Bidirectional Recurrent Neural Networks},
author={Joseph Marvin Imperial and Jeyrome Orosco and Shiela Mae Mazo and Lany Maceda},
year={2019},
eprint={1908.01765},
archivePrefix={arXiv},
primaryClass={cs.NE}
}
"""

_DATASETNAME = "typhoon_yolanda_tweets"

_DESCRIPTION = """\
The dataset contains annotated typhoon and disaster-related tweets in Filipino collected before, during,
and after one month of Typhoon Yolanda in 2013. The dataset has been annotated by an expert into three
sentiment categories: positive, negative, and neutral.
"""

_HOMEPAGE = "https://github.com/imperialite/Philippine-Languages-Online-Corpora/tree/master/Tweets/Annotated%20Yolanda"

_LICENSE = Licenses.CC_BY_4_0.value

_ROOT_URL = "https://raw.githubusercontent.com/imperialite/Philippine-Languages-Online-Corpora/master/Tweets/Annotated%20Yolanda/"
_URLS = {"train": {-1: _ROOT_URL + "train/-1.txt", 0: _ROOT_URL + "train/0.txt", 1: _ROOT_URL + "train/1.txt"}, "test": {-1: _ROOT_URL + "test/-1.txt", 0: _ROOT_URL + "test/0.txt", 1: _ROOT_URL + "test/1.txt"}}

_SUPPORTED_TASKS = [Tasks.SENTIMENT_ANALYSIS]

_SOURCE_VERSION = "1.0.0"

_SEACROWD_VERSION = "1.0.0"

class TyphoonYolandaTweets(datasets.GeneratorBasedBuilder):
"""
The dataset contains annotated typhoon and disaster-related tweets in Filipino collected before, during, and
after one month of Typhoon Yolanda in 2013. The dataset has been annotated by an expert into three sentiment
categories: positive, negative, and neutral.
"""

SOURCE_VERSION = datasets.Version(_SOURCE_VERSION)
SEACROWD_VERSION = datasets.Version(_SEACROWD_VERSION)

BUILDER_CONFIGS = [
SEACrowdConfig(
name="typhoon_yolanda_tweets_source",
version=SOURCE_VERSION,
description="Typhoon Yolanda Tweets source schema",
schema="source",
subset_id="typhoon_yolanda_tweets",
),
SEACrowdConfig(
name="typhoon_yolanda_tweets_seacrowd_text",
version=SEACROWD_VERSION,
description="Typhoon Yolanda Tweets SEACrowd schema",
schema="seacrowd_text",
subset_id="typhoon_yolanda_tweets",
),
]

DEFAULT_CONFIG_NAME = "typhoon_yolanda_tweets_source"

def _info(self) -> datasets.DatasetInfo:
if self.config.schema == "source":
features = datasets.Features(
{
"id": datasets.Value("string"),
"text": datasets.Value("string"),
"label": datasets.Value("string"),
}
)
elif self.config.schema == "seacrowd_text":
features = schemas.text_features(["-1", "0", "1"])

return datasets.DatasetInfo(
description=_DESCRIPTION,
features=features,
homepage=_HOMEPAGE,
license=_LICENSE,
citation=_CITATION,
)

def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datasets.SplitGenerator]:
emos = [-1, 0, 1]
if self.config.name == "typhoon_yolanda_tweets_source" or self.config.name == "typhoon_yolanda_tweets_seacrowd_text":
train_path = dl_manager.download_and_extract({emo: _URLS["train"][emo] for emo in emos})

test_path = dl_manager.download_and_extract({emo: _URLS["test"][emo] for emo in emos})

return [
datasets.SplitGenerator(
name=datasets.Split.TRAIN,
gen_kwargs={
"filepath": train_path,
"split": "train",
},
),
datasets.SplitGenerator(
name=datasets.Split.TEST,
gen_kwargs={
"filepath": test_path,
"split": "test",
},
),
]

def _generate_examples(self, filepath: Path, split: str) -> Tuple[int, Dict]:
if self.config.schema != "source" and self.config.schema != "seacrowd_text":
raise ValueError(f"Invalid config: {self.config.name}")

df = pd.DataFrame(columns=["text", "label"])

if self.config.name == "typhoon_yolanda_tweets_source" or self.config.name == "typhoon_yolanda_tweets_seacrowd_text":
for emo, file in filepath.items():
with open(file) as f:
t = f.readlines()
l = [str(emo)]*(len(t))
tmp_df = pd.DataFrame.from_dict({"text": t, "label": l})
df = pd.concat([df, tmp_df], ignore_index=True)

for row in df.itertuples():
ex = {"id": str(row.Index), "text": row.text, "label": row.label}
yield row.Index, ex

0 comments on commit 4922146

Please sign in to comment.