NAS-131562 / 25.04 / Bring back OneDrive

docs/source/middleware/plugins/cloudsync.rst

@@ -0,0 +1,18 @@
+`cloudsync` plugin: Cloud Sync
+==============================
+
+.. contents:: Table of Contents
+    :depth: 3
+
+OAuth
+-----
+
+Some cloud providers offer the option to configure access using the standard OAuth flow through the user's browser.
+The process works as follows:
+
+* The UI opens a pop-up window with the URL: `https://truenas.com/oauth/<provider>?origin=<NAS IP>`
+* The `truenas.com` web server proxies this request to an installation of the
+  `oauth-portal <https://github.com/ixsystems/oauth-portal>`_.
+* The OAuth portal forwards the request to the corresponding OAuth provider, retrieves the tokens, and returns them to
+  the UI using `window.opener.postMessage`.
+* The UI receives the tokens and populates the corresponding Cloud Credentials configuration form.

docs/source/middleware/plugins/index.rst

@@ -6,4 +6,5 @@ Middleware Plugins
     :caption: Contents:
 
     alert.rst
+    cloudsync.rst
     migration.rst

docs/source/simulating/index.rst

@@ -1,4 +1,4 @@
-Simulating real-world conditions
+Simulating Real-World Conditions
 ================================
 
 .. toctree::

src/middlewared/middlewared/alembic/versions/23.10/2023-12-22_11-29_delete_onedrive_cloud_sync_tasks.py

@@ -17,10 +17,7 @@
 
 
 def upgrade():
-    conn = op.get_bind()
-    conn.execute("DELETE FROM tasks_cloudsync WHERE credential_id in (SELECT id FROM system_cloudcredentials WHERE "
-                 "provider = 'ONEDRIVE')")
-    conn.execute("DELETE FROM system_cloudcredentials WHERE provider = 'ONEDRIVE'")
+    pass
 
 
 def downgrade():

src/middlewared/middlewared/api/v25_04_0/cloud_sync.py

@@ -1,12 +1,17 @@
-from middlewared.api.base import (BaseModel, Excluded, excluded_field, ForUpdateMetaclass, NonEmptyString,
-                                  single_argument_result)
+from typing import Literal
+
+from pydantic import Secret
+
+from middlewared.api.base import (BaseModel, Excluded, excluded_field, ForUpdateMetaclass, LongNonEmptyString,
+                                  NonEmptyString, single_argument_args, single_argument_result)
 from .cloud_sync_providers import CloudCredentialProvider
 
 __all__ = ["CloudCredentialEntry",
            "CloudCredentialCreateArgs", "CloudCredentialCreateResult",
            "CloudCredentialUpdateArgs", "CloudCredentialUpdateResult",
            "CloudCredentialDeleteArgs", "CloudCredentialDeleteResult",
-           "CloudCredentialVerifyArgs", "CloudCredentialVerifyResult"]
+           "CloudCredentialVerifyArgs", "CloudCredentialVerifyResult",
+           "CloudSyncOneDriveListDrivesArgs", "CloudSyncOneDriveListDrivesResult"]
 
 
 class CloudCredentialEntry(BaseModel):
@@ -83,3 +88,19 @@ class CloudCredentialVerifyResult(BaseModel):
     valid: bool
     error: str | None = None
     excerpt: str | None = None
+
+
+@single_argument_args("onedrive_list_drives")
+class CloudSyncOneDriveListDrivesArgs(BaseModel):
+    client_id: Secret[str] = ""
+    client_secret: Secret[str] = ""
+    token: Secret[LongNonEmptyString]
+
+
+class CloudSyncOneDriveListDrivesResult(BaseModel):
+    result: list["CloudSyncOneDriveListDrivesDrive"]
+
+
+class CloudSyncOneDriveListDrivesDrive(BaseModel):
+    drive_id: str
+    drive_type: Literal["PERSONAL", "BUSINESS", "DOCUMENT_LIBRARY"]

src/middlewared/middlewared/api/v25_04_0/cloud_sync_providers.py

@@ -87,6 +87,15 @@ class MegaCredentialsModel(BaseModel):
     pass_: Secret[NonEmptyString] = Field(alias="pass")
 
 
+class OneDriveCredentialsModel(BaseModel):
+    type: Literal["ONEDRIVE"]
+    client_id: Secret[str] = ""
+    client_secret: Secret[str] = ""
+    token: Secret[LongNonEmptyString]
+    drive_type: Secret[Literal["PERSONAL", "BUSINESS", "DOCUMENT_LIBRARY"]]
+    drive_id: Secret[str]
+
+
 class PCloudCredentialsModel(BaseModel):
     type: Literal["PCLOUD"]
     client_id: Secret[str] = ""
@@ -169,6 +178,7 @@ class YandexCredentialsModel(BaseModel):
         HTTPCredentialsModel,
         HubicCredentialsModel,
         MegaCredentialsModel,
+        OneDriveCredentialsModel,
         PCloudCredentialsModel,
         S3CredentialsModel,
         SFTPCredentialsModel,

src/middlewared/middlewared/plugins/mail_/outlook.py

@@ -6,6 +6,7 @@
 import requests
 
 from middlewared.service import CallError, private, Service
+from middlewared.utils.microsoft import get_microsoft_access_token
 
 
 @dataclass
@@ -29,18 +30,12 @@ def outlook_xoauth2(self, server: SMTP, config: dict):
             self.logger.warning("Outlook XOAUTH2 failed: %r %r. Refreshing access token", code, response)
 
         self.logger.debug("Requesting Outlook access token")
-        r = requests.post(
-            "https://login.microsoftonline.com/common/oauth2/v2.0/token",
-            data={
-                "grant_type": "refresh_token",
-                "client_id": config["oauth"]["client_id"],
-                "client_secret": config["oauth"]["client_secret"],
-                "refresh_token": config["oauth"]["refresh_token"],
-                "scope": "https://outlook.office.com/SMTP.Send openid offline_access",
-            }
+        response = get_microsoft_access_token(
+            config["oauth"]["client_id"],
+            config["oauth"]["client_secret"],
+            config["oauth"]["refresh_token"],
+            "https://outlook.office.com/SMTP.Send openid offline_access",
         )
-        r.raise_for_status()
-        response = r.json()
 
         token = response["access_token"]
         self._set_outlook_token(config["fromemail"], config["oauth"]["refresh_token"], token, response["expires_in"])

src/middlewared/middlewared/rclone/remote/onedrive.py

@@ -0,0 +1,90 @@
+import json
+
+import requests
+
+from middlewared.api import api_method
+from middlewared.api.current import CloudSyncOneDriveListDrivesArgs, CloudSyncOneDriveListDrivesResult
+from middlewared.rclone.base import BaseRcloneRemote
+from middlewared.utils.microsoft import get_microsoft_access_token
+
+DRIVES_TYPES = {
+    "PERSONAL": "personal",
+    "BUSINESS": "business",
+    "DOCUMENT_LIBRARY": "documentLibrary",
+}
+DRIVES_TYPES_INV = {v: k for k, v in DRIVES_TYPES.items()}
+
+
+class OneDriveRcloneRemote(BaseRcloneRemote):
+    name = "ONEDRIVE"
+    title = "Microsoft OneDrive"
+
+    rclone_type = "onedrive"
+
+    credentials_oauth = True
+    refresh_credentials = ["token"]
+
+    extra_methods = ["list_drives"]
+
+    async def get_task_extra(self, task):
+        return {
+            "drive_type": DRIVES_TYPES.get(task["credentials"]["attributes"]["drive_type"], ""),
+            # Subject to change as Microsoft changes rate limits; please watch `forum.rclone.org`
+            "checkers": "1",
+            "tpslimit": "10",
+        }
+
+    @api_method(CloudSyncOneDriveListDrivesArgs, CloudSyncOneDriveListDrivesResult, roles=["CLOUD_SYNC_WRITE"])
+    def list_drives(self, credentials):
+        """
+        Lists all available drives and their types for given Microsoft OneDrive credentials.
+        """
+        self.middleware.call_sync("network.general.will_perform_activity", "cloud_sync")
+
+        if not credentials["client_id"]:
+            credentials["client_id"] = "b15665d9-eda6-4092-8539-0eec376afd59"
+        if not credentials["client_secret"]:
+            credentials["client_secret"] = "qtyfaBBYA403=unZUP40~_#"
+
+        token = json.loads(credentials["token"])
+
+        r = requests.get(
+            "https://graph.microsoft.com/v1.0/me/drives",
+            headers={"Authorization": f"Bearer {token['access_token']}"},
+            timeout=10,
+        )
+        if r.status_code == 401:
+            token = get_microsoft_access_token(
+                credentials["client_id"],
+                credentials["client_secret"],
+                token["refresh_token"],
+                "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access",
+            )
+            r = requests.get(
+                "https://graph.microsoft.com/v1.0/me/drives",
+                headers={"Authorization": f"Bearer {token['access_token']}"},
+                timeout=10,
+            )
+        r.raise_for_status()
+
+        def process_drive(drive):
+            return {
+                "drive_type": DRIVES_TYPES_INV.get(drive["driveType"], ""),
+                "drive_id": drive["id"],
+            }
+
+        result = []
+        for drive in r.json()["value"]:
+            result.append(process_drive(drive))
+        # Also call /me/drive as sometimes /me/drives doesn't return it
+        # see https://github.com/rclone/rclone/issues/4068
+        r = requests.get(
+            "https://graph.microsoft.com/v1.0/me/drive",
+            headers={"Authorization": f"Bearer {token['access_token']}"},
+            timeout=10,
+        )
+        r.raise_for_status()
+        me_drive = process_drive(r.json())
+        if me_drive not in result:
+            result.insert(0, me_drive)
+        return result

src/middlewared/middlewared/utils/microsoft.py

@@ -0,0 +1,17 @@
+import requests
+
+
+def get_microsoft_access_token(client_id: str, client_secret: str, refresh_token: str, scope: str) -> dict:
+    r = requests.post(
+        "https://login.microsoftonline.com/common/oauth2/v2.0/token",
+        data={
+            "grant_type": "refresh_token",
+            "client_id": client_id,
+            "client_secret": client_secret,
+            "refresh_token": refresh_token,
+            "scope": scope,
+        },
+        timeout=10,
+    )
+    r.raise_for_status()
+    return r.json()