Onedrive_Backup (#968)

* initial version

* update requirements.txt

* draft 2

* retrieve scope from config

* prepare config

* save persistent token cahe

* save token cache

* auth flow, token cache

* minor flake8 fixes

* clean up comments

* flake8, refactor

* Update publish_docs_to_wiki.yml

* move logic into api.py

* formatting

* Revert "Update publish_docs_to_wiki.yml"

This reverts commit 19a9835.

* flake8

* add required module to workflow

* remove sharepoint-onedrive SDK

.github/workflows/github-actions-python.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flake8 pytest paho-mqtt requests-mock jq pyjwt==2.6.0 bs4 pkce typing_extensions python-dateutil==2.8.2
+          pip install flake8 pytest paho-mqtt requests-mock jq pyjwt==2.6.0 bs4 pkce typing_extensions python-dateutil==2.8.2 msal
       - name: Flake8 with annotations in packages folder
         uses: TrueBrain/actions-flake8@v2.1
         with:

packages/helpermodules/command.py

@@ -10,8 +10,10 @@
 import traceback
 from pathlib import Path
 import paho.mqtt.client as mqtt
+
 from control.chargepoint import chargepoint
 from control.chargepoint.chargepoint_template import get_autolock_plan_default, get_chargepoint_template_default
+from modules.backup_clouds.onedrive.api import generateMSALAuthCode, retrieveMSALTokens
 
 from helpermodules import measurement_log
 from helpermodules.broker import InternalBrokerClient
@@ -28,6 +30,7 @@
 import dataclass_utils
 from modules.common.configurable_vehicle import IntervalConfig
 
+
 log = logging.getLogger(__name__)
 
 
@@ -695,6 +698,30 @@ def restoreBackup(self, connection_id: str, payload: dict) -> None:
                              f'Restore-Status: {result.returncode}<br />Meldung: {result.stdout.decode("utf-8")}',
                              MessageType.ERROR)
 
+    def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None:
+        ''' fordert einen Authentifizierungscode für MSAL (Microsoft Authentication Library)
+        an um Onedrive Backup zu ermöglichen'''
+        cloudbackupconfig = SubData.system_data["system"].backup_cloud
+        if cloudbackupconfig is None:
+            pub_user_message(payload, connection_id,
+                             "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern "
+                             "und erneut versuchen.<br />", MessageType.WARNING)
+            return
+        result = generateMSALAuthCode(cloudbackupconfig.config)
+        pub_user_message(payload, connection_id, result["message"], result["MessageType"])
+
+    def retrieveMSALTokens(self, connection_id: str, payload: dict) -> None:
+        """ holt die Tokens für MSAL (Microsoft Authentication Library) um Onedrive Backup zu ermöglichen
+        """
+        cloudbackupconfig = SubData.system_data["system"].backup_cloud
+        if cloudbackupconfig is None:
+            pub_user_message(payload, connection_id,
+                             "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern "
+                             "und erneut versuchen.<br />", MessageType.WARNING)
+            return
+        result = retrieveMSALTokens(cloudbackupconfig.config)
+        pub_user_message(payload, connection_id, result["message"], result["MessageType"])
+
     def factoryReset(self, connection_id: str, payload: dict) -> None:
         Path(Path(__file__).resolve().parents[2] / 'data' / 'restore' / 'factory_reset').touch()
         pub_user_message(payload, connection_id,

packages/modules/backup_clouds/onedrive/api.py

@@ -0,0 +1,170 @@
+import logging
+import pickle
+import json
+import paho.mqtt.publish as publish
+import msal
+import base64
+
+from msal import PublicClientApplication
+from helpermodules.messaging import MessageType
+from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration
+
+
+log = logging.getLogger(__name__)
+
+
+def encode_str_base64(string: str) -> str:
+    string_bytes = string.encode("ascii")
+    string_base64_bytes = base64.b64encode(string_bytes)
+    string_base64_string = string_base64_bytes.decode("ascii")
+    return string_base64_string
+
+
+def save_tokencache(config: OneDriveBackupCloudConfiguration, cache: str) -> None:
+    # encode cache to base64 and save to config
+    log.debug("saving updated tokencache to config")
+    config.persistent_tokencache = encode_str_base64(cache)
+
+    # construct full configuartion object for cloud backup
+    backupcloud = OneDriveBackupCloud()
+    backupcloud.configuration = config
+    backupcloud_to_mqtt = json.dumps(backupcloud.__dict__, default=lambda o: o.__dict__)
+    log.debug("Config to MQTT:" + str(backupcloud_to_mqtt))
+
+    publish.single("openWB/set/system/backup_cloud/config", backupcloud_to_mqtt, retain=True, hostname="localhost")
+
+
+def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict:
+    result = None
+    cache = msal.SerializableTokenCache()
+
+    if config.persistent_tokencache:
+        cache.deserialize(base64.b64decode(config.persistent_tokencache))
+    else:
+        raise Exception("No tokencache found, please re-configure and re-authorize access Cloud backup settings.")
+
+    # Create a public client application with msal
+    log.debug("creating MSAL public client application")
+    app = msal.PublicClientApplication(client_id=config.clientID, authority=config.authority, token_cache=cache)
+
+    log.debug("getting accounts")
+    accounts = app.get_accounts()
+    if accounts:
+        chosen = accounts[0]  # assume that we only will have a single account in cache
+        log.debug("selected account " + str(chosen["username"]))
+        # Now let's try to find a token in cache for this account
+        result = app.acquire_token_silent(scopes=config.scope, account=chosen)
+    else:
+        raise Exception("No matching account found,please re-configure and re-authorize access Cloud backup settings.")
+
+    log.debug("done acquring tokens")
+    if not result:  # We have no token for this account, so the end user shall sign-in
+        raise Exception("No token found, please re-configure and re-authorize access Cloud backup settings.")
+
+    if "access_token" in result:
+        log.debug("access token retrieved")
+        save_tokencache(config=config, cache=cache.serialize())
+    else:
+        # Print the error
+        raise Exception("Error retrieving access token", result.get("error"), result.get("error_description"))
+    return result
+
+
+def generateMSALAuthCode(cloudbackup: OneDriveBackupCloud) -> dict:
+    """ startet den Authentifizierungsprozess für MSAL (Microsoft Authentication Library) für Onedrive Backup
+    und speichert den AuthCode in der Konfiguration"""
+    result = dict(
+        message="",
+        MessageType=MessageType.SUCCESS
+    )
+
+    if cloudbackup is None:
+        result["message"] = """Es ist keine Backup-Cloud konfiguriert.
+                            Bitte Konfiguration speichern und erneut versuchen.<br />"""
+        result["MessageType"] = MessageType.WARNING
+        return result
+
+    # Create a public client application with msal
+    app = PublicClientApplication(
+        client_id=cloudbackup.configuration.clientID,
+        authority=cloudbackup.configuration.authority
+        )
+
+    # create device flow to obtain auth code
+    flow = app.initiate_device_flow(cloudbackup.configuration.scope)
+    if "user_code" not in flow:
+        raise Exception(
+            "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4))
+
+    flow["expires_at"] = 0  # Mark it as expired immediately to prevent
+    pickleString = str(pickle.dumps(flow), encoding='latin1')
+
+    cloudbackup.configuration.flow = str(pickleString)
+    cloudbackup.configuration.authcode = flow["user_code"]
+    cloudbackup.configuration.authurl = flow["verification_uri"]
+    cloudbackupconfig_to_mqtt = json.dumps(cloudbackup.__dict__, default=lambda o: o.__dict__)
+
+    publish.single(
+        "openWB/set/system/backup_cloud/config", cloudbackupconfig_to_mqtt, retain=True, hostname="localhost"
+        )
+
+    result["message"] = """Authorisierung gestartet, bitte den Link öffen, Code eingeben,
+        und Zugang authorisieren. Anschließend Zugangsberechtigung abrufen."""
+    result["MessageType"] = MessageType.SUCCESS
+
+    return result
+
+
+def retrieveMSALTokens(cloudbackup: OneDriveBackupCloud) -> dict:
+    result = dict(
+        message="",
+        MessageType=MessageType.SUCCESS
+    )
+    if cloudbackup is None:
+        result["message"] = """Es ist keine Backup-Cloud konfiguriert.
+                            Bitte Konfiguration speichern und erneut versuchen.<br />"""
+        result["MessageType"] = MessageType.WARNING
+        return result
+
+    # Create a public client application with msal
+    tokens = None
+    cache = msal.SerializableTokenCache()
+    app = PublicClientApplication(client_id=cloudbackup.configuration.clientID,
+                                  authority=cloudbackup.configuration.authority, token_cache=cache)
+
+    f = cloudbackup.configuration.flow
+    if f is None:
+        result["message"] = """Es ist wurde kein Auth-Code erstellt.
+                            Bitte zunächst Auth-Code erstellen und den Authorisierungsprozess beenden.<br />"""
+        result["MessageType"] = MessageType.WARNING
+        return result
+    flow = pickle.loads(bytes(f, encoding='latin1'))
+
+    tokens = app.acquire_token_by_device_flow(flow)
+    # https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_by_device_flow
+    # https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-device-code
+    # Check if the token was obtained successfully
+    if "access_token" in tokens:
+        log.debug("retrieved access token")
+
+        # Tokens retrieved, remove auth codes as they are single use only.
+        cloudbackup.configuration.flow = None
+        cloudbackup.configuration.authcode = None
+        cloudbackup.configuration.authurl = None
+
+        # save tokens
+        save_tokencache(config=cloudbackup.configuration, cache=cache.serialize())
+        result["message"] = """Zugangsberechtigung erfolgreich abgerufen."""
+        result["MessageType"] = MessageType.SUCCESS
+        return result
+
+    else:
+        result["message"] = """"Es konnten keine Tokens abgerufen werden:
+                            %s <br> %s""" % (tokens.get("error"), tokens.get("error_description"))
+        result["MessageType"] = MessageType.WARNING
+        '''pub_user_message(payload, connection_id,
+                            "Es konnten keine Tokens abgerufen werden: %s <br> %s"
+                            % (result.get("error"), result.get("error_description")), MessageType.WARNING
+                            )
+        '''
+        return result

packages/modules/backup_clouds/onedrive/backup_cloud.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import logging
+import os
+import pathlib
+
+from modules.backup_clouds.onedrive.msdrive.onedrive import OneDrive
+from modules.backup_clouds.onedrive.api import get_tokens
+from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration
+from modules.common.abstract_device import DeviceDescriptor
+from modules.common.configurable_backup_cloud import ConfigurableBackupCloud
+
+
+log = logging.getLogger(__name__)
+
+
+def upload_backup(config: OneDriveBackupCloudConfiguration, backup_filename: str, backup_file: bytes) -> None:
+    # upload a single file to onedrive useing credentials from OneDriveBackupCloudConfiguration
+    # https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online
+    tokens = get_tokens(config)  # type: ignore
+    log.debug("token object retrieved, access_token: %s", tokens.__len__)
+    log.debug("instantiate OneDrive connection")
+    onedrive = OneDrive(access_token=tokens["access_token"])
+
+    localbackup = os.path.join(pathlib.Path().resolve(), 'data', 'backup', backup_filename)
+    remote_filename = backup_filename.replace(':', '-')  # file won't upload when name contains ':'
+
+    if not config.backuppath.endswith("/"):
+        log.debug("fixing  missing ending slash in backuppath: " + config.backuppath)
+        config.backuppath = config.backuppath + "/"
+
+    log.debug("uploading file %s to OneDrive", backup_filename)
+    onedrive.upload_item(item_path=(config.backuppath+remote_filename), file_path=localbackup,
+                         conflict_behavior="replace")
+
+
+def create_backup_cloud(config: OneDriveBackupCloud):
+    def updater(backup_filename: str, backup_file: bytes):
+        upload_backup(config.configuration, backup_filename, backup_file)
+    return ConfigurableBackupCloud(config=config, component_updater=updater)
+
+
+device_descriptor = DeviceDescriptor(configuration_factory=OneDriveBackupCloud)

packages/modules/backup_clouds/onedrive/config.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+
+class OneDriveBackupCloudConfiguration:
+    def __init__(self, backuppath: str = "/openWB/Backup/",
+                 persistent_tokencache: Optional[str] = None,
+                 authurl: Optional[str] = None,
+                 authcode: Optional[str] = None,
+                 scope: Optional[list] = ["https://graph.microsoft.com/Files.ReadWrite"],
+                 authority: Optional[str] = "https://login.microsoftonline.com/consumers/",
+                 clientID: Optional[str] = "e529d8d2-3b0f-4ae4-b2ba-2d9a2bba55b2",
+                 flow: Optional[str] = None) -> None:
+        self.backuppath = backuppath
+        self.persistent_tokencache = persistent_tokencache
+        self.authurl = authurl
+        self.authcode = authcode
+        self.scope = scope
+        self.authority = authority
+        self.clientID = clientID
+        self.flow = flow
+
+
+class OneDriveBackupCloud:
+    def __init__(self,
+                 name: str = "OneDrive",
+                 type: str = "onedrive",
+                 configuration: OneDriveBackupCloudConfiguration = None) -> None:
+        self.name = name
+        self.type = type
+        self.configuration = configuration or OneDriveBackupCloudConfiguration()

packages/modules/backup_clouds/onedrive/msdrive/constants.py

@@ -0,0 +1,3 @@
+BASE_GRAPH_URL = "https://graph.microsoft.com/v1.0"
+SIMPLE_UPLOAD_MAX_SIZE = 4000000  # 4MB
+CHUNK_UPLOAD_MAX_SIZE = 3276800  # ~3MB must be divisible by 327680 bytes