-
Notifications
You must be signed in to change notification settings - Fork 0
/
pre-processing of tweets.R
97 lines (67 loc) · 2.21 KB
/
pre-processing of tweets.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
library(caret)
library(magrittr)
library(dplyr)
library(lattice)
library(mice)
library(caTools)
library(aod)
library(ISLR2)
library(mgcv)
library(MASS)
library(leaps)
library(xgboost)
library(rpart)
library(ggplot2)
library(tidyr)
library(skimr)
library(ROCR)
library(rpart.plot)
library(Metrics)
library(ipred)
library(ranger)
library(gbm)
library(e1071)
library(stringr)
library(readr)
library(forcats)
library(mgcv)
library(tm)
library(SnowballC)
file1 = read.csv('tweets_stocks.csv')
#create corpus vector
corpus = Corpus(VectorSource(file1$body))
#lowercase
corpus = tm_map(corpus,FUN = content_transformer(tolower))
#remove urls
corpus = tm_map(corpus,
FUN = content_transformer(FUN = function(x)gsub(pattern = 'http[[:alnum:][:punct:]]*',
replacement = ' ',x = x)))
#remove special chars and punctuation
corpus = tm_map(corpus,FUN = removePunctuation)
#remove whitespace and stem document
corpus = tm_map(corpus,FUN = stripWhitespace)
corpus = tm_map(corpus,FUN = stemDocument)
corpus$content
# code to convert corpus to dataframe and extract it
df1 = data.frame(text = sapply(corpus, as.character), stringsAsFactors = FALSE)
write.csv(df1 , 'corpusFinal.csv')
### This part is not related to the preprocessing directly and is more related to tokenisation and creating dictionaries
## however still mentioning the rough code here on what more we may do with data before the training process.
#create document matrix
dtm = DocumentTermMatrix(corpus)
dtm
#remove sparse terms
xdtm = removeSparseTerms(dtm,sparse = 0.95)
xdtm
#create dictionary from original data
dict = findFreqTerms(DocumentTermMatrix(Corpus(VectorSource(file1))),
lowfreq = 0)
dict_corpus = Corpus(VectorSource(dict))
#unstem the words back to their original form
xdtm = as.data.frame(as.matrix(xdtm))
colnames(xdtm) = stemCompletion(x = colnames(xdtm),
dictionary = dict_corpus,
type='prevalent')
colnames(xdtm) = make.names(colnames(xdtm))
#view the terms
sort(colSums(xdtm),decreasing = T)