-
Notifications
You must be signed in to change notification settings - Fork 21
/
insurance_compare.py
105 lines (87 loc) · 2.92 KB
/
insurance_compare.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
import pandas as pd
import numpy as np
import scipy
from scipy.stats import ks_2samp
from statsmodels.distributions.empirical_distribution import ECDF
dataset = 'insurance_compare.csv'
url = "https://raw.githubusercontent.com/VincentGranville/Main/main/" + dataset ## insurance_compare.csv"
df = pd.read_csv(url)
# df = pd.read_csv(dataset)
if dataset == 'insurance_compare.csv':
df = df.drop('region', axis=1)
df = df.dropna(axis='columns')
print(df.head())
data_real = df.loc[df['Data'] == 'Real']
data_real = data_real.drop('Data', axis=1)
data_real = data_real.to_numpy()
print(data_real)
r_corr = np.corrcoef(data_real.T) # need to transpose the data to make sense
print(r_corr)
ltests = df.Data.unique().tolist()
popped_item = ltests.pop(0) # remove real data from the tests
print(ltests)
for test in ltests:
data_test = df.loc[df['Data'] == test]
data_test = data_test.drop('Data', axis=1)
data_test = data_test.to_numpy()
t_corr = np.corrcoef(data_test.T)
delta = np.abs(t_corr - r_corr)
dim = delta.shape[0] # number of features
ks = np.zeros(dim)
out_of_range = 0
for idx in range(dim):
dr = data_real[:,idx]
dt = data_test[:,idx]
stats = ks_2samp(dr, dt)
ks[idx] = stats.statistic
if np.min(dt) < np.min(dr) or np.max(dt) > np.max(dr):
out_of_range = 1
str = "%20s %14s %8.6f %8.6f %8.6f %8.6f %1d" % (dataset, test, np.mean(delta),
np.max(delta), np.mean(ks), np.max(ks), out_of_range)
print(str)
def vg_scatter(df, test, counter):
# customized plots, insurance data
# one of 6 plots, subplot position based on counter
data_plot = df.loc[df['Data'] == test]
x = data_plot[['age']].to_numpy()
y = data_plot[['charges']].to_numpy()
plt.subplot(2, 3, counter)
plt.scatter(x, y, s = 0.1, c ="blue")
plt.xlabel(test, fontsize = 7)
plt.xticks([])
plt.yticks([])
plt.ylim(0,70000)
plt.xlim(18,64)
return()
def vg_histo(df, test, counter):
# customized plots, insurance data
# one of 6 plots, subplot position based on counter
data_plot = df.loc[df['Data'] == test]
y = data_plot[['charges']].to_numpy()
plt.subplot(2, 3, counter)
binBoundaries = np.linspace(0, 70000, 30)
plt.hist(y, bins=binBoundaries, color='white', align='mid',edgecolor='red',
linewidth = 0.3)
plt.xlabel(test, fontsize = 7)
plt.xticks([])
plt.yticks([])
plt.xlim(0,70000)
plt.ylim(0, 250)
return()
import matplotlib.pyplot as plt
import matplotlib as mpl
mpl.rcParams['axes.linewidth'] = 0.3
vg_scatter(df, 'Real', 1)
vg_scatter(df, 'YData1', 2)
vg_scatter(df, 'Gretel', 3)
vg_scatter(df, 'Mostly.ai', 4)
vg_scatter(df, 'Synthesize.io', 5)
vg_scatter(df, 'SDV', 6)
plt.show()
vg_histo(df, 'Real', 1)
vg_histo(df, 'YData1', 2)
vg_histo(df, 'Gretel', 3)
vg_histo(df, 'Mostly.ai', 4)
vg_histo(df, 'Synthesize.io', 5)
vg_histo(df, 'SDV', 6)
plt.show()