From 9974f5e00407569e3492b42c649a68a1cd131a71 Mon Sep 17 00:00:00 2001 From: Andrew Aikman Date: Fri, 16 Feb 2024 18:26:15 +0000 Subject: [PATCH] feat: RESTful endpoint for device module readings (#10) --- conftest.py | 7 + requirements.txt | 2 + shedpi_hub_dashboard/admin.py | 2 +- shedpi_hub_dashboard/forms/fields.py | 15 +- .../migrations/0001_initial.py | 4 +- shedpi_hub_dashboard/models.py | 14 +- shedpi_hub_dashboard/serlializers.py | 16 ++ .../shedpi_hub_dashboard/dummy_data.json | 10 - .../static/shedpi_hub_dashboard/js/index.js | 185 +++++++++++++++--- .../templates/shedpi_hub_dashboard/index.html | 36 ++-- shedpi_hub_dashboard/tests/test_endpoints.py | 99 ++++++++++ .../tests/test_json_schema.py | 24 +++ shedpi_hub_dashboard/views.py | 41 ++++ shedpi_hub_example_project/settings.py | 9 + shedpi_hub_example_project/urls.py | 9 + 15 files changed, 401 insertions(+), 72 deletions(-) create mode 100644 shedpi_hub_dashboard/serlializers.py delete mode 100644 shedpi_hub_dashboard/static/shedpi_hub_dashboard/dummy_data.json create mode 100644 shedpi_hub_dashboard/tests/test_endpoints.py diff --git a/conftest.py b/conftest.py index e69de29..ba3dfa6 100644 --- a/conftest.py +++ b/conftest.py @@ -0,0 +1,7 @@ +import pytest +from rest_framework.test import APIClient + + +@pytest.fixture +def client(): + return APIClient() diff --git a/requirements.txt b/requirements.txt index 4c91d69..a65f7cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ build==0.10.0 cfgv==3.4.0 distlib==0.3.8 Django==5.0.1 +djangorestframework==3.14.0 exceptiongroup==1.2.0 factory-boy==3.3.0 Faker==22.5.0 @@ -21,6 +22,7 @@ pyproject_hooks==1.0.0 pytest==7.4.4 pytest-django==4.7.0 python-dateutil==2.8.2 +pytz==2023.3.post1 PyYAML==6.0.1 referencing==0.32.1 rpds-py==0.17.1 diff --git a/shedpi_hub_dashboard/admin.py b/shedpi_hub_dashboard/admin.py index 66ff110..25e6ea9 100644 --- a/shedpi_hub_dashboard/admin.py +++ b/shedpi_hub_dashboard/admin.py @@ -15,4 +15,4 @@ class DeviceModuleAdmin(admin.ModelAdmin): @admin.register(DeviceModuleReading) class DeviceModuleReadingAdmin(admin.ModelAdmin): - pass + list_display = ("id", "device_module_id", "created_at") diff --git a/shedpi_hub_dashboard/forms/fields.py b/shedpi_hub_dashboard/forms/fields.py index c0bc3cd..c603025 100644 --- a/shedpi_hub_dashboard/forms/fields.py +++ b/shedpi_hub_dashboard/forms/fields.py @@ -1,13 +1,17 @@ import json +from django.db.models import JSONField from django.forms import JSONField as JSONFormField from django.forms import widgets class PrettyJSONWidget(widgets.Textarea): def format_value(self, value): - # Prettify the json - value = json.dumps(json.loads(value), indent=2, sort_keys=True) + try: + # Prettify the json + value = json.dumps(json.loads(value), indent=2, sort_keys=True) + except json.JSONDecodeError: + return super(PrettyJSONWidget, self).format_value(value) # Calculate the size of the contents row_lengths = [len(r) for r in value.split("\n")] @@ -23,3 +27,10 @@ def format_value(self, value): class PrettyJsonFormField(JSONFormField): widget = PrettyJSONWidget + + +class PrettyJsonField(JSONField): + def formfield(self, **kwargs): + defaults = {"form_class": PrettyJsonFormField} + defaults.update(kwargs) + return super().formfield(**defaults) diff --git a/shedpi_hub_dashboard/migrations/0001_initial.py b/shedpi_hub_dashboard/migrations/0001_initial.py index c1ab5af..eca88f5 100644 --- a/shedpi_hub_dashboard/migrations/0001_initial.py +++ b/shedpi_hub_dashboard/migrations/0001_initial.py @@ -44,7 +44,7 @@ class Migration(migrations.Migration): ("location", models.CharField(max_length=50)), ( "schema", - shedpi_hub_dashboard.models.PrettySONField(blank=True, null=True), + shedpi_hub_dashboard.models.PrettyJsonField(blank=True, null=True), ), ( "device", @@ -70,7 +70,7 @@ class Migration(migrations.Migration): ), ( "data", - shedpi_hub_dashboard.models.PrettySONField(blank=True, null=True), + shedpi_hub_dashboard.models.PrettyJsonField(blank=True, null=True), ), ("created_at", models.DateTimeField(auto_now_add=True)), ( diff --git a/shedpi_hub_dashboard/models.py b/shedpi_hub_dashboard/models.py index b2b3748..b2e0cff 100644 --- a/shedpi_hub_dashboard/models.py +++ b/shedpi_hub_dashboard/models.py @@ -1,17 +1,9 @@ import uuid from django.db import models -from django.db.models import JSONField from jsonschema import validate -from shedpi_hub_dashboard.forms.fields import PrettyJsonFormField - - -class PrettySONField(JSONField): - def formfield(self, **kwargs): - defaults = {"form_class": PrettyJsonFormField} - defaults.update(kwargs) - return super().formfield(**defaults) +from shedpi_hub_dashboard.forms.fields import PrettyJsonField class Device(models.Model): @@ -32,7 +24,7 @@ class DeviceModule(models.Model): ) name = models.CharField(max_length=20) location = models.CharField(max_length=50) - schema = PrettySONField(null=True, blank=True) + schema = PrettyJsonField(null=True, blank=True) def __str__(self): return self.name @@ -44,7 +36,7 @@ class DeviceModuleReading(models.Model): on_delete=models.CASCADE, help_text="A device whose readings were collected.", ) - data = PrettySONField(null=True, blank=True) + data = PrettyJsonField(null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) def validate_data(self) -> None: diff --git a/shedpi_hub_dashboard/serlializers.py b/shedpi_hub_dashboard/serlializers.py new file mode 100644 index 0000000..24b4302 --- /dev/null +++ b/shedpi_hub_dashboard/serlializers.py @@ -0,0 +1,16 @@ +from rest_framework import serializers + +from .models import DeviceModule, DeviceModuleReading + + +class DeviceModuleSerializer(serializers.ModelSerializer): + class Meta: + model = DeviceModule + fields = "__all__" + + +class DeviceModuleReadingSerializer(serializers.ModelSerializer): + class Meta: + model = DeviceModuleReading + fields = "__all__" + # extra_kwargs = {"device_module": {"required": True}} diff --git a/shedpi_hub_dashboard/static/shedpi_hub_dashboard/dummy_data.json b/shedpi_hub_dashboard/static/shedpi_hub_dashboard/dummy_data.json deleted file mode 100644 index eedab2e..0000000 --- a/shedpi_hub_dashboard/static/shedpi_hub_dashboard/dummy_data.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "headings": ["datetime", "temperature", "humidity"], - "readings": [ - { - "datetime": "2024-01-07 12:22:44", - "temperature": "12.345", - "humidity": "12" - } - ] -} diff --git a/shedpi_hub_dashboard/static/shedpi_hub_dashboard/js/index.js b/shedpi_hub_dashboard/static/shedpi_hub_dashboard/js/index.js index df7d187..26e8c7b 100644 --- a/shedpi_hub_dashboard/static/shedpi_hub_dashboard/js/index.js +++ b/shedpi_hub_dashboard/static/shedpi_hub_dashboard/js/index.js @@ -1,49 +1,178 @@ +const contents = document.getElementsByClassName("contents")[0]; +let section = contents -const contents = document.getElementsByClassName("contents"); -let section = contents[0] +let deviceModuleEndpoint = "" -const url = section.getAttribute("data-json-feed") -const myRequest = new Request(url); +// Global store for the device modules, with schema +let storeDeviceModules = [] +let deviceModuleSchemaMap = {} + +/* Drop down selection */ +// Create dropdown container +const deviceModuleSelectorContainer = document.createElement("div"); +section.append(deviceModuleSelectorContainer); + +const urlDeviceModule = "/api/v1/device-module/" +let endpointDeviceModule = new Request(urlDeviceModule); +response = fetch(endpointDeviceModule) + .then((response) => { + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + + return response.json(); + }) + .then((response) => { + storeDeviceModules = response + drawDropdown() + + // Build schema map + + }); + +let drawDropdown = function () { + let data = storeDeviceModules + let dropdown = document.createElement("select"); + + // Table Header + let emptySelector = document.createElement("option"); + emptySelector.textContent = "Please Select" + dropdown.append(emptySelector) + + dropdown.addEventListener('change', function (e) { + optionId = this.selectedOptions[0].id + + if (optionId) { + loadTableData(optionId) + } + }); + + for (let deviceModuleIndex in data) { + const deviceModule = data[deviceModuleIndex] + + let optionElement = document.createElement("option"); + optionElement.textContent = deviceModule.device + " - " + deviceModule.name + optionElement.id = deviceModule.id + + // Build schema map + deviceModuleSchemaMap[deviceModule.id] = deviceModule.schema + + dropdown.append(optionElement); + } + + // Add the drpdown to the page + deviceModuleSelectorContainer.append(dropdown); +}; + +/* Table visual */ + +// Create table container +const tableContainer = document.createElement("div"); +section.append(tableContainer); + +let loadTableData = function (deviceModuleId) { + + // const url = section.getAttribute("data-json-feed") + const url = "http://localhost:8000//api/v1/device-module-readings/" + const endpoint = new URL(url); + endpoint.searchParams.append("device_module", deviceModuleId); + + // FIXME: Need data output and need headings from Schema + + // const urlDeviceModuleReading = + let endpointDeviceModuleReading = new Request(endpoint); + + response = fetch(endpointDeviceModuleReading) + .then((response) => { + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + + return response.json(); + }) + .then((response) => { + drawTable(response, deviceModuleId) + }); +} + + +let drawTable = function (dataset, deviceModuleId) { + // First empty the table container + tableContainer.textContent = "" -let drawTable = function (data) { let table = document.createElement("table"); // Table Header let headerRow = document.createElement("tr"); - for(let heading in data.headings) { + // TODO: Build the header rows from the schema, or build a full list in th + // Could use the schema, what about historic data that may violate it, + // Built as a ist because the pagination would hammer the device modiule + const headingFields = dataset[0]; + + // TODO: Build the header rows from the schema, or build a full list in the backend and supply in the response + // Could use the schema, what about historic data that may violate it, + // Built as a ist because the pagination would hammer the device modiule + + schema = deviceModuleSchemaMap[deviceModuleId] + + let dataFields = [] + if (schema) { + extra_fields = Object.keys(schema.properties) + dataFields = [...dataFields, ...extra_fields]; + dataFields = [...new Set(dataFields)] + } + + // FIXME: Need human readable headings, probably needs to come from the BE to be + for (let heading in headingFields) { + + if (heading == "data") { - let headerItem = document.createElement("th"); - headerItem.textContent = data.headings[heading] - headerRow.append(headerItem); + for (let headingIndex in dataFields) { + const heading = dataFields[headingIndex] + let headerItem = document.createElement("th"); + headerItem.textContent = heading + headerRow.append(headerItem); + } + } else { + let headerItem = document.createElement("th"); + headerItem.textContent = heading + headerRow.append(headerItem); + } } table.append(headerRow); // Table Contents - for(let row in data.readings) { + for (let rowIndex in dataset) { + const row = dataset[rowIndex] let contentRow = document.createElement("tr"); - for(let reading in data.readings[row]) { - let contentItem = document.createElement("td"); - contentItem.textContent = data.readings[row][reading] - contentRow.append(contentItem); + for (let reading in row) { + const fieldValue = row[reading] + if (typeof fieldValue == "object") { + for (let dataFieldIndex in dataFields) { + let contentItem = document.createElement("td"); + const dataField = dataFields[dataFieldIndex] + + // FIXME: Need to change the null value in the project to be an empty object + let mydict = {} + if (fieldValue != null) { + if (fieldValue.hasOwnProperty(dataField)) { + contentItem.textContent = fieldValue[dataField] + } + } + + contentRow.append(contentItem); + } + } else { + let contentItem = document.createElement("td"); + contentItem.textContent = row[reading] + contentRow.append(contentItem); + } } table.append(contentRow); } // Add the table to the page - section.append(table); + tableContainer.append(table); } - - -response = fetch(myRequest) - .then((response) => { - if (!response.ok) { - throw new Error(`HTTP error! Status: ${response.status}`); - } - - return response.json(); - }) - .then((response) => { - drawTable(response) - }); diff --git a/shedpi_hub_dashboard/templates/shedpi_hub_dashboard/index.html b/shedpi_hub_dashboard/templates/shedpi_hub_dashboard/index.html index 9fd0115..f8489c1 100644 --- a/shedpi_hub_dashboard/templates/shedpi_hub_dashboard/index.html +++ b/shedpi_hub_dashboard/templates/shedpi_hub_dashboard/index.html @@ -1,23 +1,23 @@ - {% load static %} - - - - - ShedPi - - - -
-

Shed Pi data

-
-
-
- - - + + + + + ShedPi + + + +
+

Shed Pi data

+
+
+ +
+ + + diff --git a/shedpi_hub_dashboard/tests/test_endpoints.py b/shedpi_hub_dashboard/tests/test_endpoints.py new file mode 100644 index 0000000..0564935 --- /dev/null +++ b/shedpi_hub_dashboard/tests/test_endpoints.py @@ -0,0 +1,99 @@ +import json + +import pytest +from django.urls import reverse +from rest_framework import status + +from shedpi_hub_dashboard.tests.utils.factories import ( + DeviceModuleFactory, + DeviceModuleReadingFactory, +) + + +@pytest.mark.django_db +def test_device_module_list(client): + DeviceModuleFactory.create_batch(2) + + url = reverse("devicemodule-list") + response = client.get(url) + + assert response.status_code == status.HTTP_200_OK + assert len(response.data) == 2 + + +@pytest.mark.django_db +def test_device_module_readings_list(client): + """ + An individual device module readings are returned from the module readings endpoint + """ + schema = { + "$id": "https://example.com/person.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Reading", + "type": "object", + "properties": { + "temperature": {"type": "string", "description": "The Temperature"}, + }, + } + device_module = DeviceModuleFactory(schema=schema) + DeviceModuleReadingFactory(device_module=device_module, data={"temperature": "20"}) + DeviceModuleReadingFactory(device_module=device_module, data={"temperature": "22"}) + # Another modules readings that shouldn't be returned + DeviceModuleReadingFactory(data={"temperature": "10"}) + + # url = reverse("devicemodulereading-detail", kwargs={"pk": device_module.id}) + url = reverse("devicemodulereading-list") + response = client.get(url, data={"device_module": device_module.id}) + + assert response.status_code == status.HTTP_200_OK + assert len(response.data) == 2 + + +@pytest.mark.django_db +def test_device_module_readings_list_no_device_module_supplied(client): + """ """ + schema = { + "$id": "https://example.com/person.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Reading", + "type": "object", + "properties": { + "temperature": {"type": "string", "description": "The Temperature"}, + }, + } + device_module = DeviceModuleFactory(schema=schema) + DeviceModuleReadingFactory(device_module=device_module, data={"temperature": "20"}) + DeviceModuleReadingFactory(device_module=device_module, data={"temperature": "22"}) + # Another modules readings that shouldn't be returned + DeviceModuleReadingFactory(data={"temperature": "10"}) + + # url = reverse("devicemodulereading-detail", kwargs={"pk": device_module.id}) + url = reverse("devicemodulereading-list") + response = client.get(url, data={}) + + assert response.status_code == status.HTTP_200_OK + assert len(response.data) == 3 + + +@pytest.mark.django_db +def test_device_module_reading_submission(client): + schema = { + "$id": "https://example.com/person.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Reading", + "type": "object", + "properties": { + "temperature": {"type": "string", "description": "The Temperature"}, + }, + } + device_module = DeviceModuleFactory(schema=schema) + + # url = reverse("devicemodulereading-detail", kwargs={"pk": device_module.id}) + url = reverse("devicemodulereading-list") + data = {"temperature": "20.001"} + response = client.post( + url, data={"device_module": device_module.id, "data": json.dumps(data)} + ) + + assert response.status_code == status.HTTP_201_CREATED + assert response.data["data"] == data diff --git a/shedpi_hub_dashboard/tests/test_json_schema.py b/shedpi_hub_dashboard/tests/test_json_schema.py index 307218a..64d5df4 100644 --- a/shedpi_hub_dashboard/tests/test_json_schema.py +++ b/shedpi_hub_dashboard/tests/test_json_schema.py @@ -86,6 +86,30 @@ def test_json_schema_update_with_invalid_data(): reading.save() +@pytest.mark.django_db +def test_json_schema_update_with_more_fields_supplied(): + schema = { + "$id": "https://example.com/person.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Person", + "type": "object", + "properties": { + "age": { + "description": "Age in years which must be equal to or greater than zero.", + "type": "integer", + "minimum": 0, + } + }, + "unevaluatedProperties": False, + } + data = {"age": 1, "someOtherField": "Some value here"} + device_module = DeviceModuleFactory(schema=schema) + reading = DeviceModuleReading(device_module=device_module, data=data) + + with pytest.raises(ValidationError): + reading.save() + + @pytest.mark.django_db def test_json_schema_invalid_schema(): schema = {"type": 1234} diff --git a/shedpi_hub_dashboard/views.py b/shedpi_hub_dashboard/views.py index 2514c7f..6396412 100644 --- a/shedpi_hub_dashboard/views.py +++ b/shedpi_hub_dashboard/views.py @@ -1,6 +1,47 @@ from django.template.response import TemplateResponse +from rest_framework import viewsets + +from .models import DeviceModule, DeviceModuleReading +from .serlializers import DeviceModuleReadingSerializer, DeviceModuleSerializer def index(request): response = TemplateResponse(request, "shedpi_hub_dashboard/index.html", {}) return response + + +class DeviceModuleViewSet(viewsets.ModelViewSet): + queryset = DeviceModule.objects.all() + serializer_class = DeviceModuleSerializer + + +class DeviceModuleReadingViewSet(viewsets.ModelViewSet): + queryset = DeviceModuleReading.objects.all() + serializer_class = DeviceModuleReadingSerializer + + def get_queryset(self): + # FIXME: Validate that the user supplied this get param! + device_module_id = self.request.query_params.get("device_module") + + if device_module_id: + return self.queryset.filter(device_module=device_module_id) + + return self.queryset + + # def list(self, request): + # queryset = self.get_queryset() + # + # context = {"request": request} + # device_module_id = self.request.query_params.get("device_module") + # + # if device_module_id: + # queryset = queryset.filter(device_module=device_module_id) + # + # context["device_module"] = device_module_id + # + # context["queryset"] = queryset + # + # serializer = self.get_serializer(data=request.data, context=context) + # serializer.is_valid(raise_exception=True) + # + # return Response(serializer.data) diff --git a/shedpi_hub_example_project/settings.py b/shedpi_hub_example_project/settings.py index a57faf0..a0dc765 100644 --- a/shedpi_hub_example_project/settings.py +++ b/shedpi_hub_example_project/settings.py @@ -38,6 +38,7 @@ "django.contrib.messages", "django.contrib.staticfiles", "shedpi_hub_dashboard.apps.ShedpiHubDashboardConfig", + "rest_framework", ] MIDDLEWARE = [ @@ -126,3 +127,11 @@ # https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +REST_FRAMEWORK = { + # Use Django's standard `django.contrib.auth` permissions, + # or allow read-only access for unauthenticated users. + # "DEFAULT_PERMISSION_CLASSES": [ + # "rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly" + # ] +} diff --git a/shedpi_hub_example_project/urls.py b/shedpi_hub_example_project/urls.py index 232dc15..679974d 100644 --- a/shedpi_hub_example_project/urls.py +++ b/shedpi_hub_example_project/urls.py @@ -2,11 +2,20 @@ from django.conf.urls.static import static from django.contrib import admin from django.urls import include, path +from rest_framework import routers + +from shedpi_hub_dashboard.views import DeviceModuleReadingViewSet, DeviceModuleViewSet + +router = routers.DefaultRouter() +router.register(r"device-module", DeviceModuleViewSet) +router.register(r"device-module-readings", DeviceModuleReadingViewSet) urlpatterns = [ *[ path("admin/", admin.site.urls), path("", include("shedpi_hub_dashboard.urls")), + path("api-auth/", include("rest_framework.urls")), + path("api/v1/", include(router.urls)), ], *static(settings.STATIC_URL, document_root=settings.STATIC_ROOT), ]