Skip to content

Commit

Permalink
Fix processing mavlog data since move to retention policies, #573
Browse files Browse the repository at this point in the history
  • Loading branch information
@fnoop
fnoop committed Aug 22, 2017
1 parent 071654e commit f28bd48
Show file tree
Hide file tree
Showing 7 changed files with 530 additions and 527 deletions.
32 changes: 17 additions & 15 deletions manifests/maverick-modules/maverick_analysis/files/maverick-mavlogd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@ class ProcessLog(ProcessEvent):

def process_IN_CLOSE_WRITE(self, event):
logger.debug("Received IN_CLOSE_WRITE event")
self.process(event.pathname)
if "LASTLOG.TXT" not in event.pathname:
self.process(event.pathname)
def process_IN_MOVED_TO(self, event):
logger.debug("Received IN_MOVED_TO event")
self.process(event.pathname)
if "LASTLOG.TXT" not in event.pathname:
self.process(event.pathname)

def process(self, filename):
logger.info("Processing data from "+filename)
Expand Down Expand Up @@ -148,17 +150,17 @@ class ProcessLog(ProcessEvent):
ekf3 = True if ekf_data['ekf3_imu1'] or ekf_data['ekf3_imu2'] else False
ekf2ekf3 = True if ekf_data['ekf2_imu1'] and ekf_data['ekf3_imu1'] else False
if not id_exists:
cursor.execute('INSERT INTO logfiles (filename,start,finish,source,ekf2,ekf3,ekf2ekf3) VALUES (?,?,?,?,?,?,?)', (filename,start,finish,dirfrags[-1].title(),ekf2,ekf3,ekf2ekf3))
cursor.execute('INSERT INTO logfiles (filename,start,finish,source,ekf2,ekf3,ekf2ekf3) VALUES (?,?,?,?,?,?,?)', (filename,start,finish,dirfrags[-2].title(),ekf2,ekf3,ekf2ekf3))
else:
cursor.execute('UPDATE logfiles SET start=?,finish=?,source=?,ekf2=?,ekf3=?,ekf2ekf3=? WHERE id=?', (start,finish,dirfrags[-1].title(),ekf2,ekf3,ekf2ekf3,id_exists))
cursor.execute('UPDATE logfiles SET start=?,finish=?,source=?,ekf2=?,ekf3=?,ekf2ekf3=? WHERE id=?', (start,finish,dirfrags[-2].title(),ekf2,ekf3,ekf2ekf3,id_exists))
conn.commit()
conn.close()

def archive_file(self, filename):
file = os.path.basename(filename)
dirs = os.path.split(filename)
dirfrags = dirs[0].split(os.path.sep)
destfile = os.path.sep.join(["/srv/maverick/data/mavlink/archive",dirfrags[-1],dirs[1]])
destfile = os.path.sep.join(["/srv/maverick/data/mavlink/archive",dirfrags[-2],dirs[1]])
shutil.move(filename, destfile)
logger.info("Archiving file from " +filename+ " to " +destfile)

Expand All @@ -175,20 +177,20 @@ class ProcessLog(ProcessEvent):
todt=str(int(float(row[2]))*1000)
link="<tr><td>"+row[3]+"</td><td>"+row[0]+"</td><td>"+datetime.datetime.fromtimestamp(float(row[1])).strftime('%Y-%m-%d %H:%M:%S')+"</td><td>"+datetime.datetime.fromtimestamp(float(row[2])).strftime('%Y-%m-%d %H:%M:%S')+"</td>"
link += "<td>"
link += "<a href='/dashboard/db/flight-data-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(117, 117, 117); border-color: rgb(155, 155, 155); color: #fff;\">Flight Data</a>"
link += "<a href='dashboard/db/flight-data-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(117, 117, 117); border-color: rgb(155, 155, 155); color: #fff;\">Flight Data</a>"
link += "&nbsp;"
link += "<a href='/dashboard/db/mavexplorer-mavgraphs?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' background-color: rgb(31, 120, 193); border-color: rgb(69, 158, 231); color: #fff;\">Mavgraphs</a>"
link += "<a href='dashboard/db/mavexplorer-mavgraphs?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' background-color: rgb(31, 120, 193); border-color: rgb(69, 158, 231); color: #fff;\">Mavgraphs</a>"
link += "</td>"
if row[4] == 1:
link += "<td><a href='/dashboard/db/flight-ekf2-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(88, 68, 119); border-color: rgb(126, 106, 157); color: #fff;\">EKF2</a></td>"
link += "<td><a href='dashboard/db/flight-ekf2-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(88, 68, 119); border-color: rgb(126, 106, 157); color: #fff;\">EKF2</a></td>"
else:
link += "<td></td>"
if row[5] == 1:
link += "<td><a href='/dashboard/db/flight-ekf3-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(98, 158, 81); border-color: rgb(136, 196, 119); color: #fff;\">EKF3</a></td>"
link += "<td><a href='dashboard/db/flight-ekf3-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(98, 158, 81); border-color: rgb(136, 196, 119); color: #fff;\">EKF3</a></td>"
else:
link += "<td></td>"
if row[6] == 1:
link += "<td><a href='/dashboard/db/flight-ekf2-ekf3-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"color: #fff\">EKF2-EKF3</a></td>"
link += "<td><a href='dashboard/db/flight-ekf2-ekf3-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"color: #fff\">EKF2-EKF3</a></td>"
else:
link += "<td></td>"
link += "</tr>"
Expand Down Expand Up @@ -232,13 +234,13 @@ class ProcessLog(ProcessEvent):
todt=str(int(error['time']) + 30000)
link = "<tr><td>"+error['text']+"</td><td>"+datetime.datetime.fromtimestamp(int(error['time'])/1000).strftime('%Y-%m-%d %H:%M:%S')+"</td>"
link += "<td>"
link += "<a href='/dashboard/db/flight-data-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(117, 117, 117); border-color: rgb(155, 155, 155); color: #fff;\">Flight Data</a>"
link += "<a href='dashboard/db/flight-data-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(117, 117, 117); border-color: rgb(155, 155, 155); color: #fff;\">Flight Data</a>"
link += "&nbsp;"
link += "<a href='/dashboard/db/mavexplorer-mavgraphs?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' background-color: rgb(31, 120, 193); border-color: rgb(69, 158, 231); color: #fff;\">Mavgraphs</a>"
link += "<a href='dashboard/db/mavexplorer-mavgraphs?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' background-color: rgb(31, 120, 193); border-color: rgb(69, 158, 231); color: #fff;\">Mavgraphs</a>"
link += "</td>"
link += "<td><a href='/dashboard/db/flight-ekf2-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(88, 68, 119); border-color: rgb(126, 106, 157); color: #fff;\">EKF2</a></td>"
link += "<td><a href='/dashboard/db/flight-ekf3-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(98, 158, 81); border-color: rgb(136, 196, 119); color: #fff;\">EKF3</a></td>"
link += "<td><a href='/dashboard/db/flight-ekf2-ekf3-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"color: #fff\">EKF2-EKF3</a></td>"
link += "<td><a href='dashboard/db/flight-ekf2-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(88, 68, 119); border-color: rgb(126, 106, 157); color: #fff;\">EKF2</a></td>"
link += "<td><a href='dashboard/db/flight-ekf3-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"background-color: rgb(98, 158, 81); border-color: rgb(136, 196, 119); color: #fff;\">EKF3</a></td>"
link += "<td><a href='dashboard/db/flight-ekf2-ekf3-analysis?orgId=10&from="+fromdt+"&to="+todt+"' class='label label-tag' style=\"color: #fff\">EKF2-EKF3</a></td>"
link += "</tr>"
error_entries.append(link)
# If dates are in old format, skip entry
Expand Down
1 change: 1 addition & 0 deletions manifests/maverick-modules/maverick_analysis/manifests/mavlogd.pp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
group => "mav",
#replace => false,
source => "puppet:///modules/maverick_analysis/maverick-mavlogd.conf",
notify => Service_wrapper["maverick-mavlogd"],
} ->
file { "/etc/systemd/system/maverick-mavlogd.service":
ensure => present,
Expand Down
Loading

0 comments on commit f28bd48

Please sign in to comment.