forked from alexgaarciia/ExperimentsMachineLearning
-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
213 lines (163 loc) · 8.8 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
# Import necessary libraries/modules
import sys
import pandas as pd
import matplotlib.pyplot as plt
from PIL import Image
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split
from functions import base_dt, top_dt, base_mlp, top_mlp, compute_metrics, print_info, print_info2, evaluate_models
# Define class needed to store information from the output
class FileOutput:
def __init__(self, file_name):
self.file = open(file_name, 'w')
self.stdout = sys.stdout
sys.stdout = self
self.suppress_output = False # New flag to control output
def write(self, text):
if not self.suppress_output:
self.file.write(text)
self.stdout.write(text)
def close(self):
sys.stdout = self.stdout
self.file.close()
# Load the Penguin and Abalone dataset
penguin_data = pd.read_csv('./penguins.csv')
abalone_data = pd.read_csv('./abalone.csv')
########################################################################################################################
# EXERCISE 1
########################################################################################################################
# 1A) METHOD 1: Convert 'island' and 'sex' features into 1-hot vectors (dummy-coded data)
# Drop first category to avoid the 'dummy variable trap'
encoder = OneHotEncoder(sparse_output=False, drop='first')
# The "fit_transform" method fits the encoder to the specified columns and transforms the data simultaneously
encoded_island_sex = encoder.fit_transform(penguin_data[['island', 'sex']])
# The resulting sparse matrix is transformed to a DataFrame
encoded_feature_names = encoder.get_feature_names_out(['island', 'sex'])
encoded_df = pd.DataFrame(encoded_island_sex, columns=encoded_feature_names)
# Add the one-hot transformed columns to the original data and drop the original 'sex' and 'penguin' columns
penguin_data_encoded_1hot = pd.concat([penguin_data, encoded_df], axis=1)
penguin_data_encoded_1hot.drop(['island', 'sex'], axis=1, inplace=True)
# 1A) METHOD 2: Convert 'island' and 'sex' features into categories manually
# These dictionaries serve as lookup tables where the keys represent the original ones but formatted
island_mapping = {'Biscoe': 0, 'Dream': 1, 'Torgersen': 2}
sex_mapping = {'FEMALE': 0, 'MALE': 1}
# Now, penguin_data contains 'island' and 'sex' features in numerical format based on manual categorization.
penguin_data['island'] = penguin_data['island'].map(island_mapping)
penguin_data['sex'] = penguin_data['sex'].map(sex_mapping)
# 1B) Abalone dataset can be as it is, as it only contains float types and one variable ('rings')
# whose type is an integer
abalone_data_types = abalone_data.dtypes
print(abalone_data_types)
########################################################################################################################
# EXERCISE 2
########################################################################################################################
# PENGUIN DATASET:
# Assuming the target in 'penguin_data' is 'species':
class_counts = penguin_data['species'].value_counts(normalize=True) * 100
# Plot the percentage of the instances in each output class
plt.figure(figsize=(8, 6))
class_counts.plot(kind='bar', color='skyblue')
plt.xlabel('Species')
plt.ylabel('Percentage of Instances')
plt.title('Percentage of Instances in Each Species (Penguin Dataset)')
plt.xticks(rotation=45, ha='right') # Rotate x-axis labels for better visibility
plt.tight_layout()
# Save the plot as a PNG file
plt.savefig('penguin-classes.png')
# Convert the PNG file to GIF using Pillow
img = Image.open('penguin-classes.png')
img.save('penguin-classes.gif', format='GIF')
# Show the graph
plt.show()
# ABALONE DATASET:
# Assuming the target in 'abalone_data' is 'Type'
class_counts = abalone_data['Type'].value_counts(normalize=True) * 100
# Plot the percentage of the instances in each output class
plt.figure(figsize=(8, 6))
class_counts.plot(kind='bar', color='lavender')
plt.xlabel('Type')
plt.ylabel('Percentage of Instances')
plt.title('Percentage of Instances in Each Type (Abalone Dataset)')
plt.xticks(rotation=45, ha='right')
plt.tight_layout()
# Save the plot as a PNG file
plt.savefig('abalone-classes.png')
# Convert the PNG file to GIF using Pillow
img = Image.open('abalone-classes.png')
img.save('abalone-classes.gif', format='GIF')
# Show the graph
plt.show()
"""
- Which metric is more appropriate to use to evaluate the performance?
We can see in the plots that the penguin dataset is more imbalanced
than the abalone one, so we will take this into consideration to measure the
performance.
We will also consider that this is a classification task.
The metrics are the following:
1. ACCURACY: Accuracy is the most straightforward metric,
but it can be misleading, it usually works well for balanced datasets (it would work for the 'abalone' dataset).
It calculates the ratio of correctly predicted instances to the total instances.
It's suitable for balanced datasets where classes have similar proportions.
- Formula: No. of correct predictions / Total no. of observations
- The next evaluation metrics work better if one class is more
important than others (the case in 'penguins' dataset)
1. PRECISION: Precision focuses on the ratio of correctly predicted positive observations to the total predicted
positive observations. It is suitable when the cost of false positives is high.
- Formula: TP/(TP + FP)
2. RECALL (Sensitivity or True Positive Rate): Recall calculates the ratio of correctly
predicted positive observations to all the actual positives. It is suitable when the cost
of false negatives is high.
- Formula: TP/(TP + FN)
3. ROC-AUC (Receiver Operating Characteristic - Area Under Curve): ROC-AUC measures the area under the ROC curve,
which represents the true positive rate against the false positive rate. It is suitable for binary classification
tasks, especially when dealing with imbalanced datasets.
4. F1-Score: F1-score is the harmonic mean of precision and recall. It is a balance between precision and recall
and is suitable when there is an uneven class distribution.
- Formula: F1 − Score = 2 × (Precision+Recall)/(Precision×Recall)
"""
########################################################################################################################
# EXERCISE 3
########################################################################################################################
# PENGUIN DATASET:
# Split the dataset into features (X) and target variable (y)
X_penguin = penguin_data[['island', 'culmen_length_mm', 'culmen_depth_mm', 'flipper_length_mm', 'body_mass_g', 'sex']]
y_penguin = penguin_data['species']
X_train_penguin, X_test_penguin, y_train_penguin, y_test_penguin = train_test_split(X_penguin, y_penguin)
# ABALONE DATASET:
X_abalone = abalone_data[['LongestShell', 'Diameter', 'Height', 'WholeWeight', 'ShuckedWeight', 'VisceraWeight', 'ShellWeight', 'Rings']]
y_abalone = abalone_data['Type']
X_train_abalone, X_test_abalone, y_train_abalone, y_test_abalone = train_test_split(X_abalone, y_abalone)
########################################################################################################################
# EXERCISE 4, 5
########################################################################################################################
# Run all the classifiers for the penguin dataset:
penguin_file = FileOutput('penguin-performance.txt')
evaluate_models(X_penguin, X_train_penguin, X_test_penguin, y_train_penguin, y_test_penguin, 1)
penguin_file.close()
# Run all the classifiers for the penguin dataset:
abalone_file = FileOutput('abalone-performance.txt')
evaluate_models(X_abalone, X_train_abalone, X_test_abalone, y_train_abalone, y_test_abalone, 1)
abalone_file.close()
########################################################################################################################
# EXERCISE 6
########################################################################################################################
# Run the model for the penguin dataset:
penguin_file_ex6 = FileOutput('penguin-performance-5times.txt')
# Obtain the metrics for a specific number of iterations:
metrics = evaluate_models(X_penguin, X_train_penguin, X_test_penguin, y_train_penguin, y_test_penguin, 5)
# Store information:
for model in metrics:
metrics_model = metrics[model]
print_info2(model, metrics_model)
# Close the file:
penguin_file_ex6.close()
# Run the model for the abalone dataset:
abalone_file_ex6 = FileOutput('abalone-performance-5times.txt')
# Obtain the metrics for a specific number of iterations:
metrics = evaluate_models(X_abalone, X_train_abalone, X_test_abalone, y_train_abalone, y_test_abalone, 5)
# Store information:
for model in metrics:
metrics_model = metrics[model]
print_info2(model, metrics_model)
# Close the file:
abalone_file_ex6.close()