-
Notifications
You must be signed in to change notification settings - Fork 0
/
dataProcessor.py
65 lines (52 loc) · 2.11 KB
/
dataProcessor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
from DataInterface import extractor, convertDatatoPd, sectionAtor, cleaner, DataInterface, basicExtractor
import pandas as pd
"""
class InadquateData(error):
pass
"""
def compa_adjust(newScan, masterIndex, masterData):
"""Compares the newly taken scan to master data and make adjustments to make the comparision right
Args:
data (list[pd.DataFrame]): Oncoming scan
returns: index object of the comon indexes
"""
relevant_master_data=masterData[masterIndex]
#get index comon to both dataframes
master_index=relevant_master_data.index
newScan_index=newScan.index
comonIndexes=master_index.intersection(newScan_index)
return comonIndexes
def comparator(newScanData,comonIndexes, masterIndex,masterD):
"""_summary_: takes in newly taken scan and compares it to adequate element in master data
Args:
data (list[pd.DataFrame]): newly taken scan
Raises:
InadquateData: flag if there is a mismatch in the size of the data
Returns:
_type_: _description_
"""
masterData=masterD[masterIndex]
matching_masterData=masterData.loc[comonIndexes]
matching_newScanData=newScanData.loc[comonIndexes]
differences:list[float]=[]
#we're now comparing a dataframe to another dataframe
if len(matching_masterData)==len(matching_newScanData):
percentageDiff=abs(((matching_masterData['Dist']-matching_newScanData['Dist'])/matching_masterData['Dist'])*100)
else:
#raise InadquateData("The data is not adequate")
print("The data is not adequate")
return percentageDiff
def thresholder(data):
change:bool=False
overall_difference=sum(data)/len(data)
#print(overall_difference)
if overall_difference>=10:
change=True
return change
def mainComparator():
masterData=DataInterface()
incommingData=cleaner(convertDatatoPd(basicExtractor("data/all_close_run2_good.txt")))
common_indexes:pd.Index=compa_adjust(incommingData,6,masterData)
comparing=comparator(incommingData,common_indexes,6,masterData)
isDifferent=thresholder(comparing)
return(isDifferent)