From 7e2c663f2b854f51877202184ebaf092d3972dff Mon Sep 17 00:00:00 2001 From: Graham Gilbert Date: Thu, 21 Jun 2018 15:58:25 -0700 Subject: [PATCH] Revert "Feature rework install log processing (#240)" This reverts commit 77a2e36f0c635c323cb767f2dd109d9152b8341b. --- api/fixtures/machine_fixtures.json | 10 +- api/fixtures/search_fixtures.json | 2 +- api/v1/serializers.py | 2 +- api/v2/serializers.py | 2 +- api/v2/tests/test_machines.py | 22 +- api/v2/views.py | 8 +- catalog/views.py | 10 +- inventory/fixtures/machine_fixture.json | 10 +- inventory/views.py | 5 +- profiles/views.py | 5 +- sal/plugin.py | 2 +- search/forms.py | 4 +- .../management/commands/search_maintenance.py | 4 +- search/views.py | 12 +- server/admin.py | 5 +- .../0069_remove_machine_install_log.py | 19 - .../0070_remove_machine_install_log_hash.py | 19 - ...1_remove_updatehistory_pending_recorded.py | 19 - server/migrations/0072_auto_20180430_0920.py | 25 - .../0073_remove_machine_report_format.py | 19 - server/models.py | 121 ++- server/non_ui_views.py | 897 ++++++++++++------ server/plugins/status/status.py | 2 +- server/templates/plugins/pendingupdates.html | 33 +- server/templates/server/bu_dashboard.html | 36 + server/templates/server/machine_detail.html | 2 + server/text_utils.py | 40 +- server/urls.py | 4 - server/utils.py | 12 +- server/views.py | 18 +- 30 files changed, 891 insertions(+), 478 deletions(-) delete mode 100644 server/migrations/0069_remove_machine_install_log.py delete mode 100644 server/migrations/0070_remove_machine_install_log_hash.py delete mode 100644 server/migrations/0071_remove_updatehistory_pending_recorded.py delete mode 100644 server/migrations/0072_auto_20180430_0920.py delete mode 100644 server/migrations/0073_remove_machine_report_format.py diff --git a/api/fixtures/machine_fixtures.json b/api/fixtures/machine_fixtures.json index 47599299..1044f4bc 100644 --- a/api/fixtures/machine_fixtures.json +++ b/api/fixtures/machine_fixtures.json @@ -23,13 +23,16 @@ "last_checkin": "2016-10-06T19:29:38.376Z", "first_checkin": "2016-10-06T15:51:21.145Z", "report": "", + "report_format": "base64bz2", "errors": 4, "warnings": 7, - "activity": false, + "activity": "", "puppet_version": "4.7.0", "sal_version": "1.0.6", "last_puppet_run": "2016-10-06T19:06:21Z", "puppet_errors": 0, + "install_log_hash": "ed168dbcf43e8c89ec3bc1a5708a7210cfeeedf0fc4e8cbbda2e0b50af782e72", + "install_log": "", "deployed": true, "broken_client": false } @@ -58,13 +61,16 @@ "last_checkin": "2017-05-16T19:04:51.336Z", "first_checkin": "2017-05-16T19:04:51.728Z", "report": "", + "report_format": "base64bz2", "errors": 0, "warnings": 3, - "activity": false, + "activity": "", "puppet_version": "4.10.1", "sal_version": "2.0.3", "last_puppet_run": "2017-05-16T18:54:56Z", "puppet_errors": 0, + "install_log_hash": null, + "install_log": null, "deployed": true, "broken_client": false } diff --git a/api/fixtures/search_fixtures.json b/api/fixtures/search_fixtures.json index f14f46aa..f60442ec 100644 --- a/api/fixtures/search_fixtures.json +++ b/api/fixtures/search_fixtures.json @@ -56,7 +56,7 @@ "fields": { "search_group": 3, "search_models": "Machine", - "search_field": "serial", + "search_field": "hostname", "and_or": "AND", "operator": "=", "search_term": "C0DEADBEEF", diff --git a/api/v1/serializers.py b/api/v1/serializers.py index 3daa9d59..0fc5641d 100644 --- a/api/v1/serializers.py +++ b/api/v1/serializers.py @@ -97,4 +97,4 @@ class MachineSerializer(serializers.ModelSerializer): class Meta: model = Machine - exclude = ('report', ) + exclude = ('report', 'install_log', 'install_log_hash') diff --git a/api/v2/serializers.py b/api/v2/serializers.py index 102f9204..e3e587e9 100644 --- a/api/v2/serializers.py +++ b/api/v2/serializers.py @@ -106,7 +106,7 @@ class MachineSerializer(QueryFieldsMixin, serializers.ModelSerializer): 'machine_model_friendly', 'memory', 'memory_kb', 'warnings', 'first_checkin', 'last_checkin', 'hd_total', 'os_family', 'deployed', 'operating_system', 'machine_group', 'sal_version', 'manifest', - 'hd_percent', 'cpu_type', 'broken_client', 'activity') + 'hd_percent', 'cpu_type', 'broken_client', 'report_format') class Meta: model = Machine diff --git a/api/v2/tests/test_machines.py b/api/v2/tests/test_machines.py index c98907fe..f8765b5c 100644 --- a/api/v2/tests/test_machines.py +++ b/api/v2/tests/test_machines.py @@ -9,13 +9,15 @@ ALL_MACHINE_COLUMNS = { - 'console_user', 'munki_version', 'hd_space', 'machine_model', 'cpu_speed', 'serial', 'id', - 'last_puppet_run', 'errors', 'puppet_version', 'hostname', 'puppet_errors', - 'machine_model_friendly', 'memory', 'memory_kb', 'warnings', 'first_checkin', 'last_checkin', - 'broken_client', 'hd_total', 'os_family', 'report', 'deployed', 'operating_system', - 'machine_group', 'sal_version', 'manifest', 'hd_percent', 'cpu_type', - 'activity'} -REMOVED_MACHINE_COLUMNS = {'report'} + 'console_user', 'munki_version', 'hd_space', 'machine_model', 'cpu_speed', + 'serial', 'id', 'last_puppet_run', 'errors', 'puppet_version', 'hostname', + 'puppet_errors', 'machine_model_friendly', 'memory', 'memory_kb', + 'warnings', 'install_log', 'first_checkin', 'last_checkin', + 'broken_client', 'hd_total', 'os_family', 'report', 'deployed', + 'operating_system', 'report_format', 'machine_group', 'sal_version', + 'manifest', 'hd_percent', 'cpu_type', 'activity', 'install_log_hash'} +REMOVED_MACHINE_COLUMNS = { + 'activity', 'report', 'install_log', 'install_log_hash'} class MachinesTest(SalAPITestCase): @@ -91,13 +93,15 @@ def test_detail_include_fields(self): """Test the field inclusion/exclusion params for detail.""" response = self.authed_get( 'machine-detail', args=('C0DEADBEEF',), - params={'fields!': 'hostname'}) + params={'fields': 'activity', 'fields!': 'hostname'}) + self.assertIn('activity', response.data) self.assertNotIn('hostname', response.data) def test_list_include_fields(self): """Test the field inclusion/exclusion params for list.""" response = self.authed_get( 'machine-list', - params={'fields!': 'hostname'}) + params={'fields': 'activity', 'fields!': 'hostname'}) record = response.data['results'][0] + self.assertIn('activity', record) self.assertNotIn('hostname', record) diff --git a/api/v2/views.py b/api/v2/views.py index fa2d8e35..1037863a 100644 --- a/api/v2/views.py +++ b/api/v2/views.py @@ -82,10 +82,11 @@ class MachineViewSet(QueryFieldsMixin, viewsets.ModelViewSet): - Include Example: `/api/machines/?fields=console_user,hostname` - Exclude Example: `/api/machines/?fields!=report` - The abbreviated form excludes the `report` field. + The abbreviated form excludes the `report`, `install_log`, + `install_log_hash`, and `activity` fields. You may also use the `search` querystring to perform text searches - across the `console_user`, `cpu_speed`, `cpu_type`, + across the `activity`, `console_user`, `cpu_speed`, `cpu_type`, `hostname`, `machine_model`, `machine_model_friendly`, `manifest`, and `memory` fields. @@ -103,7 +104,8 @@ class MachineViewSet(QueryFieldsMixin, viewsets.ModelViewSet): - Include Example: `/api/machines/C0DEADBEEF/?fields=console_user,hostname` - Exclude Example: `/api/machines/C0DEADBEEF/?fields!=report` - The abbreviated form excludes the `report` field. + The abbreviated form excludes the `activity`, `report`, `install_log`, and + `install_log_hash` fields. """ queryset = Machine.objects.all() serializer_class = MachineSerializer diff --git a/catalog/views.py b/catalog/views.py index 33a6dc22..6de79546 100644 --- a/catalog/views.py +++ b/catalog/views.py @@ -13,7 +13,6 @@ from django.template import Context, RequestContext, Template from django.template.context_processors import csrf from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import require_POST from models import * from sal.decorators import * @@ -23,9 +22,11 @@ @csrf_exempt -@require_POST @key_auth_required def submit_catalog(request): + if request.method != 'POST': + raise Http404 + submission = request.POST key = submission.get('key') name = submission.get('name') @@ -57,9 +58,12 @@ def submit_catalog(request): @csrf_exempt -@require_POST @key_auth_required def catalog_hash(request): + if request.method != 'POST': + print 'method not post' + raise Http404 + output = [] submission = request.POST key = submission.get('key') diff --git a/inventory/fixtures/machine_fixture.json b/inventory/fixtures/machine_fixture.json index 47599299..1044f4bc 100644 --- a/inventory/fixtures/machine_fixture.json +++ b/inventory/fixtures/machine_fixture.json @@ -23,13 +23,16 @@ "last_checkin": "2016-10-06T19:29:38.376Z", "first_checkin": "2016-10-06T15:51:21.145Z", "report": "", + "report_format": "base64bz2", "errors": 4, "warnings": 7, - "activity": false, + "activity": "", "puppet_version": "4.7.0", "sal_version": "1.0.6", "last_puppet_run": "2016-10-06T19:06:21Z", "puppet_errors": 0, + "install_log_hash": "ed168dbcf43e8c89ec3bc1a5708a7210cfeeedf0fc4e8cbbda2e0b50af782e72", + "install_log": "", "deployed": true, "broken_client": false } @@ -58,13 +61,16 @@ "last_checkin": "2017-05-16T19:04:51.336Z", "first_checkin": "2017-05-16T19:04:51.728Z", "report": "", + "report_format": "base64bz2", "errors": 0, "warnings": 3, - "activity": false, + "activity": "", "puppet_version": "4.10.1", "sal_version": "2.0.3", "last_puppet_run": "2017-05-16T18:54:56Z", "puppet_errors": 0, + "install_log_hash": null, + "install_log": null, "deployed": true, "broken_client": false } diff --git a/inventory/views.py b/inventory/views.py index a66a84f9..03cc307a 100644 --- a/inventory/views.py +++ b/inventory/views.py @@ -18,7 +18,6 @@ from django.template import RequestContext from django.utils import timezone from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import require_POST from django.views.generic import DetailView, View from datatableview import Datatable @@ -525,9 +524,11 @@ def get_machine_entry(self, item, queryset): @csrf_exempt -@require_POST @key_auth_required def inventory_submit(request): + if request.method != 'POST': + return HttpResponseNotFound('No POST data sent') + # list of bundleids to ignore bundleid_ignorelist = [ 'com.apple.print.PrinterProxy' diff --git a/profiles/views.py b/profiles/views.py index be39e236..4e177410 100644 --- a/profiles/views.py +++ b/profiles/views.py @@ -12,7 +12,6 @@ from django.shortcuts import get_object_or_404, render_to_response from django.utils import dateparse from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import require_POST # Local @@ -25,9 +24,11 @@ # Create your views here. @csrf_exempt -@require_POST @key_auth_required def submit_profiles(request): + if request.method != 'POST': + return HttpResponseNotFound('No POST data sent') + submission = request.POST serial = submission.get('serial').upper() machine = None diff --git a/sal/plugin.py b/sal/plugin.py index 7a8c2c62..eca02592 100644 --- a/sal/plugin.py +++ b/sal/plugin.py @@ -31,11 +31,11 @@ from yapsy.IPlugin import IPlugin import yapsy.PluginManager -from django.conf import settings from django.http import Http404 from django.shortcuts import get_object_or_404 from django.template import loader +from sal import settings from sal.decorators import handle_access, is_global_admin from server.models import Machine, Plugin, MachineDetailPlugin, Report from server.text_utils import class_to_title diff --git a/search/forms.py b/search/forms.py index 255eb94e..ecb14cb2 100644 --- a/search/forms.py +++ b/search/forms.py @@ -23,7 +23,9 @@ class SearchRowForm(forms.ModelForm): 'activity', 'errors', 'warnings', - 'puppet_errors' + 'install_log', + 'puppet_errors', + 'install_log_hash' ] search_fields = [] for f in Machine._meta.fields: diff --git a/search/management/commands/search_maintenance.py b/search/management/commands/search_maintenance.py index 60255a23..995a35a1 100755 --- a/search/management/commands/search_maintenance.py +++ b/search/management/commands/search_maintenance.py @@ -38,7 +38,9 @@ def handle(self, *args, **options): 'activity', 'errors', 'warnings', - 'puppet_errors' + 'install_log', + 'puppet_errors', + 'install_log_hash' ] inventory_fields = [ diff --git a/search/views.py b/search/views.py index 6048d61f..48930b90 100644 --- a/search/views.py +++ b/search/views.py @@ -57,8 +57,11 @@ def quick_search(machines, query_string): 'activity', 'errors', 'warnings', + 'install_log', 'puppet_errors', + 'install_log_hash', 'deployed', + 'report_format', 'broken_client', 'hd_percent', 'memory', @@ -579,6 +582,9 @@ def get_csv_row(machine, facter_headers, condition_headers, plugin_script_header skip_fields = [ 'id', 'report', + 'activity', + 'install_log', + 'install_log_hash', 'machine_group' ] for name, value in machine.get_fields(): @@ -603,7 +609,8 @@ def stream_csv(header_row, machines, facter_headers, condition_headers, plugin_s @login_required def export_csv(request, search_id): - machines = Machine.objects.all().defer('report') + machines = Machine.objects.all().defer('report', 'activity', 'install_log', + 'install_log_hash') machines = search_machines(search_id, machines, full=True) @@ -617,6 +624,9 @@ def export_csv(request, search_id): skip_fields = [ 'id', 'report', + 'activity', + 'install_log', + 'install_log_hash' ] for field in fields: if not field.is_relation and field.name not in skip_fields: diff --git a/server/admin.py b/server/admin.py index c8ab8631..185d38b3 100755 --- a/server/admin.py +++ b/server/admin.py @@ -135,9 +135,10 @@ class MachineAdmin(admin.ModelAdmin): ('last_checkin', 'first_checkin'), ('puppet_version', 'last_puppet_run', 'puppet_errors'), ('sal_version', 'deployed', 'broken_client'), - 'report' + 'report', 'report_format', 'install_log_hash', 'install_log' ) - readonly_fields = (business_unit, 'first_checkin', 'last_checkin', 'last_puppet_run') + readonly_fields = (business_unit, 'first_checkin', 'last_checkin', 'last_puppet_run', + 'report_format') search_fields = ('hostname', 'console_user') diff --git a/server/migrations/0069_remove_machine_install_log.py b/server/migrations/0069_remove_machine_install_log.py deleted file mode 100644 index 9e760845..00000000 --- a/server/migrations/0069_remove_machine_install_log.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2018-04-20 17:39 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('server', '0068_auto_20180313_1440'), - ] - - operations = [ - migrations.RemoveField( - model_name='machine', - name='install_log', - ), - ] diff --git a/server/migrations/0070_remove_machine_install_log_hash.py b/server/migrations/0070_remove_machine_install_log_hash.py deleted file mode 100644 index ec199a09..00000000 --- a/server/migrations/0070_remove_machine_install_log_hash.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2018-04-25 18:29 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('server', '0069_remove_machine_install_log'), - ] - - operations = [ - migrations.RemoveField( - model_name='machine', - name='install_log_hash', - ), - ] diff --git a/server/migrations/0071_remove_updatehistory_pending_recorded.py b/server/migrations/0071_remove_updatehistory_pending_recorded.py deleted file mode 100644 index efc94e58..00000000 --- a/server/migrations/0071_remove_updatehistory_pending_recorded.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2018-04-25 18:36 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('server', '0070_remove_machine_install_log_hash'), - ] - - operations = [ - migrations.RemoveField( - model_name='updatehistory', - name='pending_recorded', - ), - ] diff --git a/server/migrations/0072_auto_20180430_0920.py b/server/migrations/0072_auto_20180430_0920.py deleted file mode 100644 index 52bdee4d..00000000 --- a/server/migrations/0072_auto_20180430_0920.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2018-04-30 13:20 -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('server', '0071_remove_updatehistory_pending_recorded'), - ] - - operations = [ - migrations.AlterField( - model_name='machine', - name='activity', - field=models.BooleanField(default=False), - ), - migrations.AlterField( - model_name='machine', - name='report_format', - field=models.CharField(choices=[(b'base64', b'base64'), (b'base64bz2', b'base64bz2'), (b'bz2', b'bz2')], default=b'base64bz2', editable=False, max_length=256), - ), - ] diff --git a/server/migrations/0073_remove_machine_report_format.py b/server/migrations/0073_remove_machine_report_format.py deleted file mode 100644 index d2c25056..00000000 --- a/server/migrations/0073_remove_machine_report_format.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11 on 2018-04-30 14:28 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('server', '0072_auto_20180430_0920'), - ] - - operations = [ - migrations.RemoveField( - model_name='machine', - name='report_format', - ), - ] diff --git a/server/models.py b/server/models.py index 97326a10..0179a4c7 100644 --- a/server/models.py +++ b/server/models.py @@ -1,7 +1,6 @@ import base64 import bz2 import plistlib -import pytz import random import string from datetime import datetime @@ -25,7 +24,6 @@ REPORT_CHOICES = ( ('base64', 'base64'), ('base64bz2', 'base64bz2'), - ('bz2', 'bz2'), ) @@ -137,13 +135,17 @@ class Machine(models.Model): last_checkin = models.DateTimeField(db_index=True, blank=True, null=True) first_checkin = models.DateTimeField(db_index=True, blank=True, null=True, auto_now_add=True) report = models.TextField(editable=True, null=True) + report_format = models.CharField( + max_length=256, choices=REPORT_CHOICES, default="base64bz2", editable=False) errors = models.IntegerField(default=0) warnings = models.IntegerField(default=0) - activity = models.BooleanField(editable=True, default=False) + activity = models.TextField(editable=False, null=True, blank=True) puppet_version = models.CharField(db_index=True, null=True, blank=True, max_length=256) sal_version = models.CharField(db_index=True, null=True, blank=True, max_length=255) last_puppet_run = models.DateTimeField(db_index=True, blank=True, null=True) puppet_errors = models.IntegerField(db_index=True, default=0) + install_log_hash = models.CharField(max_length=200, blank=True, null=True) + install_log = models.TextField(null=True, blank=True) deployed = models.BooleanField(default=True) broken_client = models.BooleanField(default=False) @@ -153,8 +155,104 @@ class Machine(models.Model): def get_fields(self): return [(field.name, field.value_to_string(self)) for field in Machine._meta.fields] + def encode(self, plist): + string = plistlib.writePlistToString(plist) + bz2data = bz2.compress(string) + b64data = base64.b64encode(bz2data) + return b64data + + def decode(self, data): + # this has some sucky workarounds for odd handling + # of UTF-8 data in sqlite3 + try: + plist = plistlib.readPlistFromString(data) + return plist + except Exception: + try: + plist = plistlib.readPlistFromString(data.encode('UTF-8')) + return plist + except Exception: + try: + if self.report_format == 'base64bz2': + return self.b64bz_decode(data) + elif self.report_format == 'base64': + return self.b64_decode(data) + + except Exception: + return dict() + + def b64bz_decode(self, data): + try: + bz2data = base64.b64decode(data) + string = bz2.decompress(bz2data) + plist = plistlib.readPlistFromString(string) + return plist + except Exception: + return {} + + def b64_decode(self, data): + try: + string = base64.b64decode(data) + plist = plistlib.readPlistFromString(string) + return plist + except Exception: + return {} + def get_report(self): - return plistlib.readPlistFromString(self.report) + return self.decode(self.report) + + def get_activity(self): + return self.decode(self.activity) + + def update_report(self, encoded_report, report_format='base64bz2'): + # Save report. + try: + if report_format == 'base64bz2': + plist = self.b64bz_decode(encoded_report) + elif report_format == 'base64': + plist = self.b64_decode(encoded_report) + self.report = plistlib.writePlistToString(plist) + self.report_format = report_format + except Exception: + plist = None + self.report = '' + + if plist is None: + self.activity = None + self.errors = 0 + self.warnings = 0 + self.console_user = "" + return + + # Check activity. + activity = dict() + for section in ("ItemsToInstall", + "InstallResults", + "ItemsToRemove", + "RemovalResults", + "AppleUpdates"): + if (section in plist) and len(plist[section]): + activity[section] = plist[section] + if activity: + self.activity = plistlib.writePlistToString(activity) + else: + self.activity = None + + # Check errors and warnings. + if "Errors" in plist: + self.errors = len(plist["Errors"]) + else: + self.errors = 0 + + if "Warnings" in plist: + self.warnings = len(plist["Warnings"]) + else: + self.warnings = 0 + + # Check console user. + self.console_user = "unknown" + if "ConsoleUser" in plist: + self.console_user = unicode(plist["ConsoleUser"]) def __unicode__(self): if self.hostname: @@ -169,6 +267,13 @@ def display_name(cls): class Meta: ordering = ['hostname'] + def save(self, *args, **kwargs): + self.serial = self.serial.replace('/', '') + self.serial = self.serial.replace('+', '') + if not self.hostname: + self.hostname = self.serial + super(Machine, self).save() + GROUP_NAMES = { 'all': None, @@ -187,6 +292,7 @@ class UpdateHistory(models.Model): update_type = models.CharField(max_length=254, choices=UPDATE_TYPE, verbose_name="Update Type") name = models.CharField(max_length=255, db_index=True) version = models.CharField(max_length=254, db_index=True) + pending_recorded = models.BooleanField(default=False) def __unicode__(self): return u"%s: %s %s" % (self.machine, self.name, self.version) @@ -299,10 +405,7 @@ def save(self): self.pluginscript_data_string = "" try: - date_data = parse(self.pluginscript_data) - if not date_data.tzinfo: - date_data = date_data.replace(tzinfo=pytz.UTC) - self.pluginscript_data_date = date_data + self.pluginscript_data_date = parse(self.pluginscript_data) except Exception: # Try converting it to an int if we're here try: @@ -310,7 +413,7 @@ def save(self): try: self.pluginscript_data_date = datetime.fromtimestamp( - int(self.pluginscript_data), tz=pytz.UTC) + int(self.pluginscript_data)) except Exception: self.pluginscript_data_date = None else: diff --git a/server/non_ui_views.py b/server/non_ui_views.py index d8a34d32..76c935b9 100644 --- a/server/non_ui_views.py +++ b/server/non_ui_views.py @@ -1,9 +1,7 @@ import hashlib import itertools import json -import plistlib import re -from collections import defaultdict from datetime import datetime, timedelta import dateutil.parser @@ -14,12 +12,11 @@ from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.urlresolvers import reverse -from django.db.models import Q, Count +from django.db.models import Q from django.http import (HttpResponse, HttpResponseNotFound, JsonResponse, StreamingHttpResponse) from django.http.response import Http404 from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import require_POST from inventory.models import Inventory from sal.decorators import key_auth_required @@ -39,23 +36,9 @@ # The database probably isn't going to change while this is loaded. IS_POSTGRES = utils.is_postgres() -IGNORED_CSV_FIELDS = ('id', 'machine_group', 'report', 'os_family') -HISTORICAL_FACTS = utils.get_django_setting('HISTORICAL_FACTS', []) -IGNORE_PREFIXES = utils.get_django_setting('IGNORE_FACTS', []) -MACHINE_KEYS = { - 'machine_model': {'old': 'MachineModel', 'new': 'machine_model'}, - 'cpu_type': {'old': 'CPUType', 'new': 'cpu_type'}, - 'cpu_speed': {'old': 'CurrentProcessorSpeed', 'new': 'current_processor_speed'}, - 'memory': {'old': 'PhysicalMemory', 'new': 'physical_memory'}} -UPDATE_META = { - 'AppleUpdates': {'update_type': 'apple'}, - 'ManagedInstalls': {'update_type': 'third_party'}, - 'InstallResults': {'status': 'install'}, - 'RemovalResults': {'status': 'removal'}} -MEMORY_EXPONENTS = {'KB': 0, 'MB': 1, 'GB': 2, 'TB': 3} -# Build a translation table for serial numbers, to remove garbage -# VMware puts in. -SERIAL_TRANSLATE = {ord(c): None for c in '+/'} + +IGNORED_CSV_FIELDS = ('id', 'machine_group', 'report', 'activity', 'os_family', 'install_log', + 'install_log_hash') @login_required @@ -259,8 +242,6 @@ def preflight_v2(request): enabled_detail_plugins = MachineDetailPlugin.objects.all() for enabled_plugin in itertools.chain(enabled_reports, enabled_plugins, enabled_detail_plugins): plugin = manager.get_plugin_by_name(enabled_plugin.name) - if not plugin: - continue if os_family is None or os_family in plugin.get_supported_os_families(): scripts = utils.get_plugin_scripts(plugin, hash_only=True) if scripts: @@ -283,16 +264,32 @@ def preflight_v2_get_script(request, plugin_name, script_name): @csrf_exempt -@require_POST @key_auth_required def checkin(request): + if request.method != 'POST': + print 'not post data' + return HttpResponseNotFound('No POST data sent') + data = request.POST + key = data.get('key') + uuid = data.get('uuid') + serial = data.get('serial') + serial = serial.upper() + broken_client = data.get('broken_client', False) # Take out some of the weird junk VMware puts in. Keep an eye out in case # Apple actually uses these: - serial = data.get('serial', '').upper().translate(SERIAL_TRANSLATE) + serial = serial.replace('/', '') + serial = serial.replace('+', '') + # Are we using Sal for some sort of inventory (like, I don't know, Puppet?) - if utils.get_django_setting('ADD_NEW_MACHINES', True): + try: + add_new_machines = settings.ADD_NEW_MACHINES + except Exception: + add_new_machines = True + + if add_new_machines: + # look for serial number - if it doesn't exist, create one if serial: try: machine = Machine.objects.get(serial=serial) @@ -301,100 +298,134 @@ def checkin(request): else: machine = get_object_or_404(Machine, serial=serial) - machine_group_key = data.get('key') - if machine_group_key in (None, 'None'): - machine_group_key = utils.get_django_setting('DEFAULT_MACHINE_GROUP_KEY') - machine.machine_group = get_object_or_404(MachineGroup, key=machine_group_key) + try: + deployed_on_checkin = settings.DEPLOYED_ON_CHECKIN + except Exception: + deployed_on_checkin = True - machine.last_checkin = django.utils.timezone.now() - machine.hostname = data.get('name', '') - machine.sal_version = data.get('sal_version') - machine.console_user = data.get('username') if data.get('username') != '_mbsetupuser' else None + if key is None or key == 'None': + try: + key = settings.DEFAULT_MACHINE_GROUP_KEY + except Exception: + pass - if utils.get_django_setting('DEPLOYED_ON_CHECKIN', True): - machine.deployed = True + machine_group = get_object_or_404(MachineGroup, key=key) + machine.machine_group = machine_group - if bool(data.get('broken_client', False)): + machine.last_checkin = django.utils.timezone.now() + + if bool(broken_client): machine.broken_client = True machine.save() - return HttpResponse("Broken Client report submmitted for %s" % data.get('serial')) - - report = None - # Find the report in the submitted data. It could be encoded - # and/or compressed with base64 and bz2. - for key in ('bz2report', 'base64report', 'base64bz2report'): - if key in data: - encoded_report = data[key] - report = text_utils.decode_to_string(encoded_report, compression=key) - break - - machine.report = report - - if not report: - machine.activity = False - machine.errors = machine.warnings = 0 - return - - report_data = plistlib.readPlistFromString(report) + return HttpResponse("Broken Client report submmitted for %s" + % data.get('serial')) + else: + machine.broken_client = False - machine.activity = any(report_data.get(s) for s in UPDATE_META.keys()) + historical_days = utils.get_setting('historical_retention') - # Check errors and warnings. - machine.errors = len(report_data.get("Errors", [])) - machine.warnings = len(report_data.get("Warnings", [])) + machine.hostname = data.get('name', '') - machine.puppet_version = report_data.get('Puppet_Version') - machine.manifest = report_data.get('ManifestName') - machine.munki_version = report_data.get('ManagedInstallVersion') + if 'username' in data: + if data.get('username') != '_mbsetupuser': + machine.console_user = data.get('username') + + if 'base64bz2report' in data: + machine.update_report(data.get('base64bz2report')) + + if 'base64report' in data: + machine.update_report(data.get('base64report'), 'base64') + + if 'sal_version' in data: + machine.sal_version = data.get('sal_version') + + # extract machine data from the report + report_data = machine.get_report() + if 'Puppet_Version' in report_data: + machine.puppet_version = report_data['Puppet_Version'] + if 'ManifestName' in report_data: + manifest = report_data['ManifestName'] + machine.manifest = manifest + if 'MachineInfo' in report_data: + machine.operating_system = report_data['MachineInfo'].get( + 'os_vers', 'UNKNOWN') + # some machines are reporting 10.9, some 10.9.0 - make them the same + if len(machine.operating_system) <= 4: + machine.operating_system = machine.operating_system + '.0' - puppet = report_data.get('Puppet', {}) - if 'time' in puppet: - last_run_epoch = float(puppet['time']['last_run']) - machine.last_puppet_run = datetime.fromtimestamp(last_run_epoch, tz=pytz.UTC) - if 'events' in puppet: - machine.puppet_errors = puppet['events']['failure'] + # if gosal is the sender look for OSVers key + if 'OSVers' in report_data['MachineInfo']: + machine.operating_system = report_data['MachineInfo'].get( + 'OSVers') - # Handle gosal submissions slightly differently from others. - machine.os_family = ( - report_data['OSFamily'] if 'OSFamily' in report_data else report_data.get('os_family')) + machine.hd_space = report_data.get('AvailableDiskSpace') or 0 + machine.hd_total = int(data.get('disk_size')) or 0 - machine_info = report_data.get('MachineInfo', {}) - if 'os_vers' in machine_info: - machine.operating_system = machine_info['os_vers'] - # macOS major OS updates don't have a minor version, so add one. - if len(machine.operating_system) <= 4: - machine.operating_system = machine.operating_system + '.0' - else: - # Handle gosal and missing os_vers cases. - machine.operating_system = machine_info.get('OSVers') - - # TODO: These should be a number type. - # TODO: Cleanup all of the casting to str if we make a number. - machine.hd_space = report_data.get('AvailableDiskSpace', '0') - machine.hd_total = data.get('disk_size', '0') - space = float(machine.hd_space) - total = float(machine.hd_total) if machine.hd_total == 0: - machine.hd_percent = '0' + machine.hd_percent = 0 else: - machine.hd_percent = str(int((total - space) / total * 100)) - - # Get macOS System Profiler hardware info. - # Older versions use `HardwareInfo` key, so start there. - hwinfo = machine_info.get('HardwareInfo', {}) - if not hwinfo: - for profile in machine_info.get('SystemProfile', []): + machine.hd_percent = int( + round( + ((float( + machine.hd_total) - + float( + machine.hd_space)) / + float( + machine.hd_total)) * + 100)) + machine.munki_version = report_data.get('ManagedInstallVersion') or 0 + hwinfo = {} + # macOS System Profiler + if 'SystemProfile' in report_data.get('MachineInfo', []): + for profile in report_data['MachineInfo']['SystemProfile']: if profile['_dataType'] == 'SPHardwareDataType': hwinfo = profile._items[0] break + if 'HardwareInfo' in report_data.get('MachineInfo', []): + hwinfo = report_data['MachineInfo']['HardwareInfo'] + if 'Puppet' in report_data: + puppet = report_data.get('Puppet') + if 'time' in puppet: + machine.last_puppet_run = datetime.fromtimestamp(float(puppet['time']['last_run'])) + if 'events' in puppet: + machine.puppet_errors = puppet['events']['failure'] if hwinfo: - key_style = 'old' if 'MachineModel' in hwinfo else 'new' - machine.machine_model = hwinfo.get(MACHINE_KEYS['machine_model'][key_style]) - machine.cpu_type = hwinfo.get(MACHINE_KEYS['cpu_type'][key_style]) - machine.cpu_speed = hwinfo.get(MACHINE_KEYS['cpu_speed'][key_style]) - machine.memory = hwinfo.get(MACHINE_KEYS['memory'][key_style]) - machine.memory_kb = int(machine.memory[:-3]) ** MEMORY_EXPONENTS[machine.memory[-2:]] + # setup vars for hash keys we might get sent + if 'MachineModel' in hwinfo: + var_machine_model = 'MachineModel' + var_cpu_type = 'CPUType' + var_cpu_speed = 'CurrentProcessorSpeed' + var_memory = 'PhysicalMemory' + else: + var_machine_model = 'machine_model' + var_cpu_type = 'cpu_type' + var_cpu_speed = 'current_processor_speed' + var_memory = 'physical_memory' + + machine.machine_model = hwinfo.get(var_machine_model) + machine.cpu_type = hwinfo.get(var_cpu_type) + machine.cpu_speed = hwinfo.get(var_cpu_speed) + machine.memory = hwinfo.get(var_memory) + + if hwinfo.get(var_memory)[-2:] == 'KB': + machine.memory_kb = int(hwinfo.get(var_memory)[:-3]) + if hwinfo.get(var_memory)[-2:] == 'MB': + memory_mb = float(hwinfo.get(var_memory)[:-3]) + machine.memory_kb = int(memory_mb * 1024) + if hwinfo.get(var_memory)[-2:] == 'GB': + memory_gb = float(hwinfo.get(var_memory)[:-3]) + machine.memory_kb = int(memory_gb * 1024 * 1024) + if hwinfo.get(var_memory)[-2:] == 'TB': + memory_tb = float(hwinfo.get(var_memory)[:-3]) + machine.memory_kb = int(memory_tb * 1024 * 1024 * 1024) + + if 'os_family' in report_data: + machine.os_family = report_data['os_family'] + + # support golang strict structure + if 'OSFamily' in report_data: + machine.os_family = report_data['OSFamily'] if not machine.machine_model_friendly: try: @@ -402,20 +433,326 @@ def checkin(request): except Exception: machine.machine_model_friendly = machine.machine_model + if deployed_on_checkin is True: + machine.deployed = True + machine.save() - historical_days = utils.get_setting('historical_retention') + # If Plugin_Results are in the report, handle them + try: + datelimit = django.utils.timezone.now() - timedelta(days=historical_days) + PluginScriptSubmission.objects.filter(recorded__lt=datelimit).delete() + except Exception: + pass + + if 'Plugin_Results' in report_data: + utils.process_plugin_script(report_data.get('Plugin_Results'), machine) + + # Remove existing PendingUpdates for the machine + machine.pending_updates.all().delete() now = django.utils.timezone.now() - datelimit = now - timedelta(days=historical_days) + if 'ItemsToInstall' in report_data: + pending_update_to_save = [] + update_history_item_to_save = [] + for update in report_data.get('ItemsToInstall'): + display_name = update.get('display_name', update['name']) + update_name = update.get('name') + version = str(update['version_to_install']) + if version: + pending_update = PendingUpdate( + machine=machine, + display_name=display_name, + update_version=version, + update=update_name) + if IS_POSTGRES: + pending_update_to_save.append(pending_update) + else: + pending_update.save() + # Let's handle some of those lovely pending installs into the UpdateHistory Model + try: + update_history = UpdateHistory.objects.get( + name=update_name, + version=version, + machine=machine, + update_type='third_party' + ) + except UpdateHistory.DoesNotExist: + update_history = UpdateHistory( + name=update_name, + version=version, + machine=machine, + update_type='third_party') + update_history.save() + + if not update_history.pending_recorded: + update_history_item = UpdateHistoryItem( + update_history=update_history, status='pending', + recorded=now, uuid=uuid) + + update_history.pending_recorded = True + update_history.save() + + if IS_POSTGRES: + update_history_item_to_save.append(update_history_item) + else: + update_history_item.save() - # Process plugin scripts. - # Clear out too-old plugin script submissions first. - PluginScriptSubmission.objects.filter(recorded__lt=datelimit).delete() - utils.process_plugin_script(report_data.get('Plugin_Results', []), machine) + if IS_POSTGRES: + PendingUpdate.objects.bulk_create(pending_update_to_save) + UpdateHistoryItem.objects.bulk_create(update_history_item_to_save) + + machine.installed_updates.all().delete() + + if 'ManagedInstalls' in report_data: + # Due to a quirk in how Munki 3 processes updates with dependencies, + # it's possible to have multiple entries in the ManagedInstalls list + # that share an update_name and installed_version. This causes an + # IntegrityError in Django since (machine_id, update, update_version) + # must be unique.Until/(unless!) this is addressed in Munki, we need to + # be careful to not add multiple items with the same name and version. + # We'll store each (update_name, version) combo as we see them. + seen_names_and_versions = [] + installed_updates_to_save = [] + for update in report_data.get('ManagedInstalls'): + display_name = update.get('display_name', update['name']) + update_name = update.get('name') + version = str(update.get('installed_version', 'UNKNOWN')) + installed = update.get('installed') + if (update_name, version) not in seen_names_and_versions: + seen_names_and_versions.append((update_name, version)) + if (version != 'UNKNOWN' and version is not None and + len(version) != 0): + installed_update = InstalledUpdate( + machine=machine, display_name=display_name, + update_version=version, update=update_name, + installed=installed) + if IS_POSTGRES: + installed_updates_to_save.append(installed_update) + else: + installed_update.save() + if IS_POSTGRES: + InstalledUpdate.objects.bulk_create(installed_updates_to_save) + + # Remove existing PendingAppleUpdates for the machine + machine.pending_apple_updates.all().delete() + if 'AppleUpdates' in report_data: + for update in report_data.get('AppleUpdates'): + display_name = update.get('display_name', update['name']) + update_name = update.get('name') + version = str(update['version_to_install']) + try: + pending_update = PendingAppleUpdate.objects.get( + machine=machine, + display_name=display_name, + update_version=version, + update=update_name + ) + except PendingAppleUpdate.DoesNotExist: + pending_update = PendingAppleUpdate( + machine=machine, + display_name=display_name, + update_version=version, + update=update_name) + pending_update.save() + # Let's handle some of those lovely pending installs into the UpdateHistory Model + try: + update_history = UpdateHistory.objects.get( + name=update_name, version=version, machine=machine, update_type='apple') + except UpdateHistory.DoesNotExist: + update_history = UpdateHistory( + name=update_name, version=version, machine=machine, update_type='apple') + update_history.save() + + if not update_history.pending_recorded: + update_history_item = UpdateHistoryItem( + update_history=update_history, status='pending', recorded=now, uuid=uuid) + update_history_item.save() + update_history.pending_recorded = True + update_history.save() - process_managed_items(machine, report_data, data.get('uuid'), now, datelimit) - process_facts(machine, report_data, datelimit) - process_conditions(machine, report_data) + # if Facter data is submitted, we need to first remove any existing facts for this machine + if IS_POSTGRES: + # If we are using postgres, we can just dump them all and do a bulk create + if 'Facter' in report_data: + facts = machine.facts.all() + if facts.exists(): + facts._raw_delete(facts.db) + try: + datelimit = django.utils.timezone.now() - timedelta(days=historical_days) + hist_to_delete = HistoricalFact.objects.filter(fact_recorded__lt=datelimit) + if hist_to_delete.exists(): + hist_to_delete._raw_delete(hist_to_delete.db) + except Exception: + pass + try: + historical_facts = settings.HISTORICAL_FACTS + except Exception: + historical_facts = [] + pass + + facts_to_be_created = [] + historical_facts_to_be_created = [] + for fact_name, fact_data in report_data['Facter'].iteritems(): + skip = False + if hasattr(settings, 'IGNORE_FACTS'): + for prefix in settings.IGNORE_FACTS: + if fact_name.startswith(prefix): + skip = True + if skip: + continue + facts_to_be_created.append( + Fact( + machine=machine, + fact_data=fact_data, + fact_name=fact_name + ) + ) + if fact_name in historical_facts: + historical_facts_to_be_created.append( + HistoricalFact( + machine=machine, + fact_data=fact_data, + fact_name=fact_name + ) + ) + Fact.objects.bulk_create(facts_to_be_created) + if len(historical_facts_to_be_created) != 0: + HistoricalFact.objects.bulk_create(historical_facts_to_be_created) + + else: + if 'Facter' in report_data: + facts = machine.facts.all() + for fact in facts: + skip = False + if hasattr(settings, 'IGNORE_FACTS'): + for prefix in settings.IGNORE_FACTS: + + if fact.fact_name.startswith(prefix): + skip = True + fact.delete() + break + if not skip: + continue + found = False + for fact_name, fact_data in report_data['Facter'].iteritems(): + + if fact.fact_name == fact_name: + found = True + break + if not found: + fact.delete() + + # Delete old historical facts + + try: + datelimit = django.utils.timezone.now() - timedelta(days=historical_days) + HistoricalFact.objects.filter(fact_recorded__lt=datelimit).delete() + except Exception: + pass + try: + historical_facts = settings.HISTORICAL_FACTS + except Exception: + historical_facts = [] + pass + # now we need to loop over the submitted facts and save them + facts = machine.facts.all() + for fact_name, fact_data in report_data['Facter'].iteritems(): + if machine.os_family == 'Windows': + # We had a little trouble parsing out facts on Windows, clean up here + if fact_name.startswith('value=>'): + fact_name = fact_name.replace('value=>', '', 1) + + # does fact exist already? + found = False + skip = False + if hasattr(settings, 'IGNORE_FACTS'): + for prefix in settings.IGNORE_FACTS: + + if fact_name.startswith(prefix): + skip = True + break + if skip: + continue + for fact in facts: + if fact_name == fact.fact_name: + # it exists, make sure it's got the right info + found = True + if fact_data == fact.fact_data: + # it's right, break + break + else: + fact.fact_data = fact_data + fact.save() + break + if not found: + + fact = Fact(machine=machine, fact_data=fact_data, fact_name=fact_name) + fact.save() + + if fact_name in historical_facts: + fact = HistoricalFact(machine=machine, fact_name=fact_name, + fact_data=fact_data, fact_recorded=datetime.now()) + fact.save() + + if IS_POSTGRES: + if 'Conditions' in report_data: + conditions_to_delete = machine.conditions.all() + if conditions_to_delete.exists(): + conditions_to_delete._raw_delete(conditions_to_delete.db) + conditions_to_be_created = [] + for condition_name, condition_data in report_data['Conditions'].iteritems(): + # Skip the conditions that come from facter + if 'Facter' in report_data and condition_name.startswith('facter_'): + continue + + condition_data = text_utils.stringify(condition_data) + conditions_to_be_created.append( + Condition( + machine=machine, + condition_name=condition_name, + condition_data=text_utils.safe_unicode(condition_data) + ) + ) + + Condition.objects.bulk_create(conditions_to_be_created) + else: + if 'Conditions' in report_data: + conditions = machine.conditions.all() + for condition in conditions: + found = False + for condition_name, condition_data in report_data['Conditions'].iteritems(): + if condition.condition_name == condition_name: + found = True + break + if found is False: + condition.delete() + + conditions = machine.conditions.all() + for condition_name, condition_data in report_data['Conditions'].iteritems(): + # Skip the conditions that come from facter + if 'Facter' in report_data and condition_name.startswith('facter_'): + continue + + # if it's a list (more than one result), + # we're going to conacetnate it into one comma separated string. + condition_data = text_utils.stringify(condition_data) + + found = False + for condition in conditions: + if condition_name == condition.condition_name: + # it exists, make sure it's got the right info + found = True + if condition_data == condition.condition_data: + # it's right, break + break + else: + condition.condition_data = condition_data + condition.save() + break + if found is False: + condition = Condition(machine=machine, condition_name=condition_name, + condition_data=text_utils.safe_unicode(condition_data)) + condition.save() utils.run_plugin_processing(machine, report_data) @@ -423,174 +760,192 @@ def checkin(request): # If setting is None, it hasn't been configured yet; assume True utils.send_report() - return HttpResponse("Sal report submmitted for %s" % data.get('name')) - - -def process_managed_items(machine, report_data, uuid, now, datelimit): - """Process Munki updates and removals.""" - items_to_create = defaultdict(list) - - # Delete all of these every run, as its faster than comparing - # between the client/server and removing the difference. - for related in ('pending_updates', 'pending_apple_updates', 'installed_updates'): - to_delete = getattr(machine, related).all() - if to_delete.exists(): - to_delete._raw_delete(to_delete.db) - - # Process ManagedInstalls for pending and already installed - # updates, AppleUpdates for pending, and - # [Install|Removal]Results for history items. - managed_item_histories = set() - for report_key, args in UPDATE_META.items(): - seen_updates = set() - for item in report_data.get(report_key, []): - kwargs = {'update': item['name'], 'machine': machine} - kwargs['display_name'] = item.get('display_name', item['name']) - installed = item.get('installed') - if installed is not None: - kwargs['installed'] = installed - version_key = 'installed_version' if installed else 'version_to_install' - kwargs['update_version'] = item.get(version_key, '') - - update_type = args.get('update_type') - if not update_type: - update_type = 'apple' if item['applesus'] else 'third_party' - - if installed is not None: - model = InstalledUpdate - elif update_type == 'apple': - model = PendingAppleUpdate - else: - model = PendingUpdate - - install_time = dateutil.parser.parse(item['time']) if 'time' in item else now - - status = args.get('status') - if status and item['status'] != 0: - status = 'error' - elif not status: - status = 'install' if installed else 'pending' - - # Due to a quirk in how Munki 3 processes updates with - # dependencies, it's possible to have multiple entries in the - # ManagedInstalls list that share an update_name and - # installed_version. This causes an IntegrityError in Django - # since (machine_id, update, update_version) must be - # unique.Until/(unless!) this is addressed in Munki, we need - # to be careful to not add multiple items with the same name - # and version. We'll store each (update_name, version) combo - # as we see them. - # TODO: Process on the client side to avoid this. - item_key = (kwargs['update'], kwargs['update_version']) - if item_key not in seen_updates: - items_to_create[model].append(model(**kwargs)) - seen_updates.add(item_key) - - update_history, _ = UpdateHistory.objects.get_or_create( - name=kwargs['update'], version=kwargs['update_version'], machine=machine, - update_type=update_type) - managed_item_histories.add(update_history.pk) - # Only create a history item if there are none or - # if the last one is not the same status. - items_set = update_history.updatehistoryitem_set - if not items_set.exists() or items_set.last().status != status: - UpdateHistoryItem.objects.create( - update_history=update_history, status=status, recorded=install_time, uuid=uuid) - - for model, updates_to_save in items_to_create.items(): - if IS_POSTGRES: - model.objects.bulk_create(updates_to_save) - else: - for item in updates_to_save: - item.save() - - # Clean up UpdateHistory and items which are over our retention - # limit and are no longer managed, or which have no history items. - histories_to_delete = UpdateHistory.objects.exclude(pk__in=managed_item_histories) - for history in histories_to_delete: - try: - latest = history.updatehistoryitem_set.latest('recorded').recorded - except UpdateHistoryItem.DoesNotExist: - history.delete() - continue - - if latest < datelimit: - history.delete() - - -def process_facts(machine, report_data, datelimit): - # TODO: May need to come through and do get_or_create on machine, name, updating data, and - # deleting now missing facts and conditions for non-postgres. - # if Facter data is submitted, we need to first remove any existing facts for this machine - facts = machine.facts.all() - if facts.exists(): - facts._raw_delete(facts.db) - hist_to_delete = HistoricalFact.objects.filter(fact_recorded__lt=datelimit) - if hist_to_delete.exists(): - hist_to_delete._raw_delete(hist_to_delete.db) - - facts_to_be_created = [] - historical_facts_to_be_created = [] - for fact_name, fact_data in report_data.get('Facter', {}).items(): - if any(fact_name.startswith(p) for p in IGNORE_PREFIXES): - continue - - facts_to_be_created.append( - Fact(machine=machine, fact_data=fact_data, fact_name=fact_name)) - - if fact_name in HISTORICAL_FACTS: - historical_facts_to_be_created.append( - HistoricalFact(machine=machine, fact_data=fact_data, fact_name=fact_name)) - - if facts_to_be_created: - if IS_POSTGRES: - Fact.objects.bulk_create(facts_to_be_created) - else: - for fact in facts_to_be_created: - fact.save() - if historical_facts_to_be_created: - if IS_POSTGRES: - HistoricalFact.objects.bulk_create(historical_facts_to_be_created) - else: - for fact in historical_facts_to_be_created: - fact.save() - - -def process_conditions(machine, report_data): - conditions_to_delete = machine.conditions.all() - if conditions_to_delete.exists(): - conditions_to_delete._raw_delete(conditions_to_delete.db) - conditions_to_be_created = [] - for condition_name, condition_data in report_data.get('Conditions', {}).items(): - # Skip the conditions that come from facter - if 'Facter' in report_data and condition_name.startswith('facter_'): - continue - - condition_data = text_utils.safe_unicode(text_utils.stringify(condition_data)) - condition = Condition( - machine=machine, condition_name=condition_name, condition_data=condition_data) - conditions_to_be_created.append(condition) - - if conditions_to_be_created: - if IS_POSTGRES: - Condition.objects.bulk_create(conditions_to_be_created) - else: - for condition in conditions_to_be_created: - condition.save() + return HttpResponse("Sal report submmitted for %s" + % data.get('name')) -# TODO: Remove after sal-scripts have reasonably been updated to not hit -# this endpoint. @csrf_exempt @key_auth_required def install_log_hash(request, serial): - sha256hash = hashlib.sha256('Update sal-scripts!').hexdigest() + sha256hash = '' + machine = None + if serial: + try: + machine = Machine.objects.get(serial=serial) + sha256hash = machine.install_log_hash + except (Machine.DoesNotExist, Inventory.DoesNotExist): + pass + else: + return HttpResponse("MACHINE NOT FOUND") return HttpResponse(sha256hash) -# TODO: Remove after sal-scripts have reasonably been updated to not hit -# this endpoint. +def process_update_item(name, version, update_type, action, recorded, machine, uuid, extra=None): + # Get a parent update history item, or create one + try: + update_history = UpdateHistory.objects.get(name=name, + version=version, + update_type=update_type, + machine=machine) + except UpdateHistory.DoesNotExist: + update_history = UpdateHistory(name=name, + version=version, + update_type=update_type, + machine=machine) + update_history.save() + + # Now make sure it's not already in there + try: + update_history_item = UpdateHistoryItem.objects.get( + recorded=recorded, + status=action, + update_history=update_history + ) + except UpdateHistoryItem.DoesNotExist: + # Make one if it doesn't exist + update_history_item = UpdateHistoryItem( + recorded=recorded, + status=action, + update_history=update_history) + update_history_item.save() + if extra: + update_history_item.extra = extra + update_history_item.save() + + if action == 'install' or action == 'removal': + # Make sure there has't been a pending in the same sal run + # Remove them if there are + remove_items = UpdateHistoryItem.objects.filter( + uuid=uuid, + status='pending', + update_history=update_history + ) + remove_items.delete() + update_history.pending_recorded = False + update_history.save() + + @csrf_exempt @key_auth_required def install_log_submit(request): - return HttpResponse("Update sal-scripts!") + if request.method != 'POST': + return HttpResponseNotFound('No POST data sent') + + submission = request.POST + serial = submission.get('serial') + key = submission.get('key') + uuid = submission.get('run_uuid') + machine = None + if serial: + try: + machine = Machine.objects.get(serial=serial) + except Machine.DoesNotExist: + return HttpResponseNotFound('Machine not found') + + # Check the key + machine_group = get_object_or_404(MachineGroup, key=key) + if machine_group.id != machine.machine_group.id: + return HttpResponseNotFound('No machine group found') + + compressed_log = submission.get('base64bz2installlog') + if compressed_log: + compressed_log = compressed_log.replace(" ", "+") + log_str = text_utils.decode_to_string(compressed_log) + machine.install_log = log_str + machine.save() + + for line in log_str.splitlines(): + # Third party install successes first + m = re.search('(.+) Install of (.+): (.+)$', line) + if m: + try: + if m.group(3) == 'SUCCESSFUL': + the_date = dateutil.parser.parse(m.group(1)) + (name, version) = m.group(2).rsplit('-', 1) + process_update_item(name, version, 'third_party', 'install', the_date, + machine, uuid) + # We've processed this line, move on + continue + + except IndexError: + pass + # Third party install failures + m = re.search('(.+) Install of (.+): FAILED (.+)$', line) + if m: + try: + the_date = dateutil.parser.parse(m.group(1)) + (name, version) = m.group(2).rsplit('-', 1) + extra = m.group(3) + process_update_item(name, version, 'third_party', 'error', the_date, + machine, uuid, extra) + # We've processed this line, move on + continue + + except IndexError: + pass + + # Third party removals + m = re.search('(.+) Removal of (.+): (.+)$', line) + if m: + try: + if m.group(3) == 'SUCCESSFUL': + the_date = dateutil.parser.parse(m.group(1)) + # (name, version) = m.group(2).rsplit('-',1) + name = m.group(2) + version = '' + process_update_item(name, version, 'third_party', 'removal', the_date, + machine, uuid) + # We've processed this line, move on + continue + + except IndexError: + pass + # Third party removal failures + m = re.search('(.+) Removal of (.+): FAILED (.+)$', line) + if m: + try: + the_date = dateutil.parser.parse(m.group(1)) + (name, version) = m.group(2).rsplit('-', 1) + extra = m.group(3) + process_update_item(name, version, 'third_party', 'error', the_date, + machine, uuid, extra) + # We've processed this line, move on + continue + + except IndexError: + pass + + # Apple update install successes + m = re.search('(.+) Apple Software Update install of (.+): (.+)$', line) + if m: + try: + if m.group(3) == 'FAILED': + the_date = dateutil.parser.parse(m.group(1)) + (name, version) = m.group(2).rsplit('-', 1) + process_update_item(name, version, 'apple', 'install', the_date, + machine, uuid) + # We've processed this line, move on + continue + + except IndexError: + pass + + # Apple install failures + m = re.search('(.+) Apple Software Update install of (.+): FAILED (.+)$', line) + if m: + try: + the_date = dateutil.parser.parse(m.group(1)) + (name, version) = m.group(2).rsplit('-', 1) + extra = m.group(3) + process_update_item(name, version, 'apple', 'error', the_date, + machine, uuid, extra) + # We've processed this line, move on + continue + + except IndexError: + pass + machine.install_log_hash = \ + hashlib.sha256(log_str).hexdigest() + machine.install_log = log_str + machine.save() + return HttpResponse("Install Log processed for %s" % serial) diff --git a/server/plugins/status/status.py b/server/plugins/status/status.py index b5984ee4..5221c4c1 100644 --- a/server/plugins/status/status.py +++ b/server/plugins/status/status.py @@ -21,7 +21,7 @@ STATUSES['broken_clients'] = ('Machines with broken Python', Q(broken_client=True)) STATUSES['errors'] = ('Machines with MSU errors', Q(errors__gt=0)) STATUSES['warnings'] = ('Machines with MSU warnings', Q(warnings__gt=0)) -STATUSES['activity'] = ('Machines with MSU activity', Q(activity=True)) +STATUSES['activity'] = ('Machines with MSU activity', Q(activity__isnull=False)) STATUSES['sevendayactive'] = ('7 day active machines', Q(last_checkin__gte=WEEK_AGO)) STATUSES['thirtydayactive'] = ('30 day active machines', Q(last_checkin__gte=MONTH_AGO)) STATUSES['ninetydayactive'] = ('90 day active machines', Q(last_checkin__gte=THREE_MONTHS_AGO)) diff --git a/server/templates/plugins/pendingupdates.html b/server/templates/plugins/pendingupdates.html index 09ca446c..457ee57f 100644 --- a/server/templates/plugins/pendingupdates.html +++ b/server/templates/plugins/pendingupdates.html @@ -1,24 +1,25 @@ -{% load dashboard_extras %} -
{{ plugin.title }}
+ -
- - {% for item in data %} - - + + {% endfor %} +
- {% with item.update|cat:"--"|cat:item.update_version as data %} - - {% endwith %} +
+
+ + {% for item in data %} + + - - {% endfor %} -
+ {% with item.update|add:'--'|add:item.update_version as var %} + + {% endwith %} {{ item.display_name }} {{ item.update_version }} - {{ item.count }} - -
+ {{ item.count }} + +
+
diff --git a/server/templates/server/bu_dashboard.html b/server/templates/server/bu_dashboard.html index 8eb7a40c..273035e1 100755 --- a/server/templates/server/bu_dashboard.html +++ b/server/templates/server/bu_dashboard.html @@ -94,7 +94,43 @@

{{ business_unit.name }}

{{ widget.html|safe }} {% endfor %} +
+ {% if pending_updates|length %} +
+ Pending 3rd Party Updates + + {% for item in pending_updates|dictsort:'update' %} + + + + {% endfor %} +
+ + {{ item.display_name }} {{ item.update_version }} + {{ item.count }} + +
+
+ {% endif %} + {% if pending_apple_updates|length %} +
+ Pending Apple Updates + + {% for item in pending_apple_updates|dictsort:'update' %} + + + + {% endfor %} +
+ + {{ item.display_name }} {{ item.update_version }} + {{ item.count }} + +
+
+ {% endif %} +
{% else %}No Machine groups configured.{% endif %} {% endblock %} diff --git a/server/templates/server/machine_detail.html b/server/templates/server/machine_detail.html index 0b36bbb3..46b69b75 100644 --- a/server/templates/server/machine_detail.html +++ b/server/templates/server/machine_detail.html @@ -167,10 +167,12 @@

{{ machine.hostname }}

Free disk space:
{{ report.AvailableDiskSpace|humanreadablesize }} ({{ machine.hd_percent }}% used)
{% endif %} + {% if uptime_enabled %} {% if uptime %}
Uptime:
{{ uptime }}
{% endif %} + {% endif %} {% if report.ConsoleUser %}
Console user:
{{ report.ConsoleUser }}
diff --git a/server/text_utils.py b/server/text_utils.py index 19b7316a..02fd9c7e 100644 --- a/server/text_utils.py +++ b/server/text_utils.py @@ -35,33 +35,19 @@ def stringify(data): return str(data) -def decode_to_string(data, compression=''): - """Decodes a string optionally bz2 compressed and/or base64 encoded. - - Will decode base64 first if specified. Then decompress bz2 if - specified. - - Args: - data (str): Data to decode. May just be a regular string! - compression (str): Type of encoding and compression. Defaults - to empty str. Uses substrings to determine what to do: - 'base64' results in first base64 decoding. - 'bz2' results in bz2.decompression. - - Returns: - The decoded, decompressed string, or '' if there were - exceptions. - """ - if 'base64' in compression: +def decode_to_string(data, compression='base64bz2'): + '''Decodes a string that is optionally bz2 compressed and always base64 encoded.''' + if compression == 'base64bz2': try: - data = base64.b64decode(data) - except TypeError: - data = '' - - if 'bz2' in compression: + bz2data = base64.b64decode(data) + return bz2.decompress(bz2data) + except Exception: + return '' + elif compression == 'base64': try: - data = bz2.decompress(data) - except IOError: - data = '' + return base64.b64decode(data) + except Exception: + return + '' - return data + return '' diff --git a/server/urls.py b/server/urls.py index 7f3469cf..3dfc260f 100644 --- a/server/urls.py +++ b/server/urls.py @@ -18,11 +18,7 @@ # Checkin routes. url(r'^checkin/', checkin, name='checkin'), - # TODO: Remove after sal-scripts have reasonably been updated to - # not hit this endpoint. url(r'^installlog/hash/(?P.+)/$', install_log_hash, name='install_log_hash'), - # TODO: Remove after sal-scripts have reasonably been updated to - # not hit this endpoint. url(r'^installlog/submit/$', install_log_submit, name='install_log_submit'), url(r'^preflight/$', preflight, name='preflight'), url(r'^preflight-v2/get-script/(?P.+)/(?P.+)/$', diff --git a/server/utils.py b/server/utils.py index 3d7017c4..ca1e4764 100644 --- a/server/utils.py +++ b/server/utils.py @@ -352,14 +352,6 @@ def is_float(value): return False -def get_django_setting(name, default=None): - """Get a setting from the Django.conf.settings object - - In Sal, that's anything in the system_settings or settings files. - """ - return getattr(settings, name, default) - - # Plugin utilities def process_plugin_script(results, machine): @@ -392,7 +384,7 @@ def process_plugin_script(results, machine): else: plugin_row.save() - if is_postgres() and rows_to_create: + if is_postgres(): PluginScriptRow.objects.bulk_create(rows_to_create) @@ -639,8 +631,6 @@ def get_plugin_placeholder_markup(plugins, group_type='all', group_id=None): for enabled_plugin in display_plugins: name = enabled_plugin.name yapsy_plugin = manager.get_plugin_by_name(name) - if not yapsy_plugin: - continue # Skip this plugin if the group's members OS families aren't supported # ...but only if this group has any members (group_oses is not empty plugin_os_families = set(yapsy_plugin.get_supported_os_families()) diff --git a/server/views.py b/server/views.py index 89ea8b44..35652bed 100644 --- a/server/views.py +++ b/server/views.py @@ -390,20 +390,25 @@ def machine_detail(request, **kwargs): elif name not in report['RemovedItems']: report['RemovedItems'].append(item['name']) - if Plugin.objects.filter(name='Uptime'): + uptime_enabled = False + plugins = Plugin.objects.all() + for plugin in plugins: + if plugin.name == 'Uptime': + uptime_enabled = True + + if uptime_enabled: try: plugin_script_submission = PluginScriptSubmission.objects.get( - machine=machine, plugin='Uptime') + machine=machine, plugin__exact='Uptime') uptime_seconds = PluginScriptRow.objects.get( submission=plugin_script_submission, - pluginscript_name='UptimeSeconds').pluginscript_data + pluginscript_name__exact='UptimeSeconds').pluginscript_data except Exception: uptime_seconds = '0' - - uptime = utils.display_time(int(uptime_seconds)) else: - uptime = None + uptime_seconds = 0 + uptime = utils.display_time(int(uptime_seconds)) if 'managed_uninstalls_list' in report: report['managed_uninstalls_list'].sort() @@ -418,6 +423,7 @@ def machine_detail(request, **kwargs): 'removal_results': removal_results, 'machine': machine, 'ip_address': ip_address, + 'uptime_enabled': uptime_enabled, 'uptime': uptime, 'output': output} return render(request, 'server/machine_detail.html', context)