generated from 10xac/Twitter-Data-Analysis-Template
-
Notifications
You must be signed in to change notification settings - Fork 0
/
clean_tweets_dataframe.py
74 lines (53 loc) · 2.33 KB
/
clean_tweets_dataframe.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
import pandas as pd
class Clean_Tweets:
"""
The PEP8 Standard AMAZING!!!
"""
def __init__(self, df:pd.DataFrame):
self.df = df
print('Automation in Action...!!!')
def drop_unwanted_column(self, df:pd.DataFrame)->pd.DataFrame:
"""
Remove rows that has column names.
This error originated from the data collection stage.
"""
unwanted_rows = df[df['retweet_count'] == 'retweet_count'].index
self.df = df.drop(unwanted_rows, inplace=True)
self.df = df[df['polarity'] != 'polarity']
# self.drop_duplicate(self.df)
return self.df
def drop_duplicate(self, df:pd.DataFrame)->pd.DataFrame:
"""
drop duplicate rows
"""
self.df = self.df.drop_duplicates().drop_duplicates(subset='original_text') # self.df.drop_duplicates(subset='original_text')
return df
def convert_to_datetime(self, df:pd.DataFrame)->pd.DataFrame:
"""
convert column to datetime
"""
self.df['created_at'] = pd.to_datetime(df['created_at'], errors='coerce')
self.df = self.df[self.df['created_at'] >= '2020-12-31' ]
return self.df
def convert_to_numbers(self, df: pd.DataFrame) -> pd.DataFrame:
"""Convert columns like polarity, subjectivity, favorite_count, retweet_count to numbers."""
self.df['polarity'] = pd.to_numeric(df['polarity'], errors='coerce')
self.df['retweet_count'] = pd.to_numeric(df['retweet_count'], errors='coerce')
self.df['favourite_count'] = pd.to_numeric(df['favourite_count'], errors='coerce')
return self.df
def remove_non_english_tweets(self, df:pd.DataFrame)->pd.DataFrame:
"""
remove non english tweets from lang
"""
self.df = self.df.query("lang == 'en' ")
return self.df
if __name__ == "__main__":
tweets_json = pd.read_csv("./processed_tweet_data.csv")
cleaner = Clean_Tweets(tweets_json)
df = cleaner.drop_unwanted_column(cleaner.df)
df = cleaner.drop_duplicate(df)
df = cleaner.convert_to_numbers(df)
df = cleaner.convert_to_datetime(df)
df = cleaner.remove_non_english_tweets(df)
df.to_csv('cleaned_tweet_data.csv', index=False)
df.to_json('data/cleaned_tweet_data.json')