From 0566abca00fa7209ce9e38d407d8b1f681559af2 Mon Sep 17 00:00:00 2001 From: themylogin Date: Fri, 10 Jan 2025 16:33:28 +0100 Subject: [PATCH] Bring back OneDrive --- docs/source/middleware/plugins/cloudsync.rst | 18 ++++ docs/source/middleware/plugins/index.rst | 1 + docs/source/simulating/index.rst | 2 +- ..._11-29_delete_onedrive_cloud_sync_tasks.py | 5 +- .../middlewared/api/v25_04_0/cloud_sync.py | 27 +++++- .../api/v25_04_0/cloud_sync_providers.py | 10 +++ .../middlewared/plugins/mail_/outlook.py | 17 ++-- .../middlewared/rclone/remote/onedrive.py | 90 +++++++++++++++++++ .../middlewared/utils/microsoft.py | 17 ++++ 9 files changed, 168 insertions(+), 19 deletions(-) create mode 100644 docs/source/middleware/plugins/cloudsync.rst create mode 100644 src/middlewared/middlewared/rclone/remote/onedrive.py create mode 100644 src/middlewared/middlewared/utils/microsoft.py diff --git a/docs/source/middleware/plugins/cloudsync.rst b/docs/source/middleware/plugins/cloudsync.rst new file mode 100644 index 0000000000000..7153cf6e32724 --- /dev/null +++ b/docs/source/middleware/plugins/cloudsync.rst @@ -0,0 +1,18 @@ +`cloudsync` plugin: Cloud Sync +============================== + +.. contents:: Table of Contents + :depth: 3 + +OAuth +----- + +Some cloud providers offer the option to configure access using the standard OAuth flow through the user's browser. +The process works as follows: + +* The UI opens a pop-up window with the URL: `https://truenas.com/oauth/?origin=` +* The `truenas.com` web server proxies this request to an installation of the + `oauth-portal `_. +* The OAuth portal forwards the request to the corresponding OAuth provider, retrieves the tokens, and returns them to + the UI using `window.opener.postMessage`. +* The UI receives the tokens and populates the corresponding Cloud Credentials configuration form. diff --git a/docs/source/middleware/plugins/index.rst b/docs/source/middleware/plugins/index.rst index 9ac2ed29f682a..8b28dbe975046 100644 --- a/docs/source/middleware/plugins/index.rst +++ b/docs/source/middleware/plugins/index.rst @@ -6,4 +6,5 @@ Middleware Plugins :caption: Contents: alert.rst + cloudsync.rst migration.rst diff --git a/docs/source/simulating/index.rst b/docs/source/simulating/index.rst index 5955614b61e1e..36e0a7b87e292 100644 --- a/docs/source/simulating/index.rst +++ b/docs/source/simulating/index.rst @@ -1,4 +1,4 @@ -Simulating real-world conditions +Simulating Real-World Conditions ================================ .. toctree:: diff --git a/src/middlewared/middlewared/alembic/versions/23.10/2023-12-22_11-29_delete_onedrive_cloud_sync_tasks.py b/src/middlewared/middlewared/alembic/versions/23.10/2023-12-22_11-29_delete_onedrive_cloud_sync_tasks.py index a751e91140e0c..a19f8eb275305 100644 --- a/src/middlewared/middlewared/alembic/versions/23.10/2023-12-22_11-29_delete_onedrive_cloud_sync_tasks.py +++ b/src/middlewared/middlewared/alembic/versions/23.10/2023-12-22_11-29_delete_onedrive_cloud_sync_tasks.py @@ -17,10 +17,7 @@ def upgrade(): - conn = op.get_bind() - conn.execute("DELETE FROM tasks_cloudsync WHERE credential_id in (SELECT id FROM system_cloudcredentials WHERE " - "provider = 'ONEDRIVE')") - conn.execute("DELETE FROM system_cloudcredentials WHERE provider = 'ONEDRIVE'") + pass def downgrade(): diff --git a/src/middlewared/middlewared/api/v25_04_0/cloud_sync.py b/src/middlewared/middlewared/api/v25_04_0/cloud_sync.py index 586296a3d596c..e3ef04d45a786 100644 --- a/src/middlewared/middlewared/api/v25_04_0/cloud_sync.py +++ b/src/middlewared/middlewared/api/v25_04_0/cloud_sync.py @@ -1,12 +1,17 @@ -from middlewared.api.base import (BaseModel, Excluded, excluded_field, ForUpdateMetaclass, NonEmptyString, - single_argument_result) +from typing import Literal + +from pydantic import Secret + +from middlewared.api.base import (BaseModel, Excluded, excluded_field, ForUpdateMetaclass, LongNonEmptyString, + NonEmptyString, single_argument_args, single_argument_result) from .cloud_sync_providers import CloudCredentialProvider __all__ = ["CloudCredentialEntry", "CloudCredentialCreateArgs", "CloudCredentialCreateResult", "CloudCredentialUpdateArgs", "CloudCredentialUpdateResult", "CloudCredentialDeleteArgs", "CloudCredentialDeleteResult", - "CloudCredentialVerifyArgs", "CloudCredentialVerifyResult"] + "CloudCredentialVerifyArgs", "CloudCredentialVerifyResult", + "CloudSyncOneDriveListDrivesArgs", "CloudSyncOneDriveListDrivesResult"] class CloudCredentialEntry(BaseModel): @@ -83,3 +88,19 @@ class CloudCredentialVerifyResult(BaseModel): valid: bool error: str | None = None excerpt: str | None = None + + +@single_argument_args("onedrive_list_drives") +class CloudSyncOneDriveListDrivesArgs(BaseModel): + client_id: Secret[str] = "" + client_secret: Secret[str] = "" + token: Secret[LongNonEmptyString] + + +class CloudSyncOneDriveListDrivesResult(BaseModel): + result: list["CloudSyncOneDriveListDrivesDrive"] + + +class CloudSyncOneDriveListDrivesDrive(BaseModel): + drive_id: str + drive_type: Literal["PERSONAL", "BUSINESS", "DOCUMENT_LIBRARY"] diff --git a/src/middlewared/middlewared/api/v25_04_0/cloud_sync_providers.py b/src/middlewared/middlewared/api/v25_04_0/cloud_sync_providers.py index 659738426c22d..73e2d183bf7c0 100644 --- a/src/middlewared/middlewared/api/v25_04_0/cloud_sync_providers.py +++ b/src/middlewared/middlewared/api/v25_04_0/cloud_sync_providers.py @@ -87,6 +87,15 @@ class MegaCredentialsModel(BaseModel): pass_: Secret[NonEmptyString] = Field(alias="pass") +class OneDriveCredentialsModel(BaseModel): + type: Literal["ONEDRIVE"] + client_id: Secret[str] = "" + client_secret: Secret[str] = "" + token: Secret[LongNonEmptyString] + drive_type: Secret[Literal["PERSONAL", "BUSINESS", "DOCUMENT_LIBRARY"]] + drive_id: Secret[str] + + class PCloudCredentialsModel(BaseModel): type: Literal["PCLOUD"] client_id: Secret[str] = "" @@ -169,6 +178,7 @@ class YandexCredentialsModel(BaseModel): HTTPCredentialsModel, HubicCredentialsModel, MegaCredentialsModel, + OneDriveCredentialsModel, PCloudCredentialsModel, S3CredentialsModel, SFTPCredentialsModel, diff --git a/src/middlewared/middlewared/plugins/mail_/outlook.py b/src/middlewared/middlewared/plugins/mail_/outlook.py index 40c97e951c9ae..ef2a81fc6e44e 100644 --- a/src/middlewared/middlewared/plugins/mail_/outlook.py +++ b/src/middlewared/middlewared/plugins/mail_/outlook.py @@ -6,6 +6,7 @@ import requests from middlewared.service import CallError, private, Service +from middlewared.utils.microsoft import get_microsoft_access_token @dataclass @@ -29,18 +30,12 @@ def outlook_xoauth2(self, server: SMTP, config: dict): self.logger.warning("Outlook XOAUTH2 failed: %r %r. Refreshing access token", code, response) self.logger.debug("Requesting Outlook access token") - r = requests.post( - "https://login.microsoftonline.com/common/oauth2/v2.0/token", - data={ - "grant_type": "refresh_token", - "client_id": config["oauth"]["client_id"], - "client_secret": config["oauth"]["client_secret"], - "refresh_token": config["oauth"]["refresh_token"], - "scope": "https://outlook.office.com/SMTP.Send openid offline_access", - } + response = get_microsoft_access_token( + config["oauth"]["client_id"], + config["oauth"]["client_secret"], + config["oauth"]["refresh_token"], + "https://outlook.office.com/SMTP.Send openid offline_access", ) - r.raise_for_status() - response = r.json() token = response["access_token"] self._set_outlook_token(config["fromemail"], config["oauth"]["refresh_token"], token, response["expires_in"]) diff --git a/src/middlewared/middlewared/rclone/remote/onedrive.py b/src/middlewared/middlewared/rclone/remote/onedrive.py new file mode 100644 index 0000000000000..d41f874067fa8 --- /dev/null +++ b/src/middlewared/middlewared/rclone/remote/onedrive.py @@ -0,0 +1,90 @@ +import json + +import requests + +from middlewared.api import api_method +from middlewared.api.current import CloudSyncOneDriveListDrivesArgs, CloudSyncOneDriveListDrivesResult +from middlewared.rclone.base import BaseRcloneRemote +from middlewared.utils.microsoft import get_microsoft_access_token + +DRIVES_TYPES = { + "PERSONAL": "personal", + "BUSINESS": "business", + "DOCUMENT_LIBRARY": "documentLibrary", +} +DRIVES_TYPES_INV = {v: k for k, v in DRIVES_TYPES.items()} + + +class OneDriveRcloneRemote(BaseRcloneRemote): + name = "ONEDRIVE" + title = "Microsoft OneDrive" + + rclone_type = "onedrive" + + credentials_oauth = True + refresh_credentials = ["token"] + + extra_methods = ["list_drives"] + + async def get_task_extra(self, task): + return { + "drive_type": DRIVES_TYPES.get(task["credentials"]["attributes"]["drive_type"], ""), + # Subject to change as Microsoft changes rate limits; please watch `forum.rclone.org` + "checkers": "1", + "tpslimit": "10", + } + + @api_method(CloudSyncOneDriveListDrivesArgs, CloudSyncOneDriveListDrivesResult, roles=["CLOUD_SYNC_WRITE"]) + def list_drives(self, credentials): + """ + Lists all available drives and their types for given Microsoft OneDrive credentials. + """ + self.middleware.call_sync("network.general.will_perform_activity", "cloud_sync") + + if not credentials["client_id"]: + credentials["client_id"] = "b15665d9-eda6-4092-8539-0eec376afd59" + if not credentials["client_secret"]: + credentials["client_secret"] = "qtyfaBBYA403=unZUP40~_#" + + token = json.loads(credentials["token"]) + + r = requests.get( + "https://graph.microsoft.com/v1.0/me/drives", + headers={"Authorization": f"Bearer {token['access_token']}"}, + timeout=10, + ) + if r.status_code == 401: + token = get_microsoft_access_token( + credentials["client_id"], + credentials["client_secret"], + token["refresh_token"], + "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access", + ) + r = requests.get( + "https://graph.microsoft.com/v1.0/me/drives", + headers={"Authorization": f"Bearer {token['access_token']}"}, + timeout=10, + ) + r.raise_for_status() + + def process_drive(drive): + return { + "drive_type": DRIVES_TYPES_INV.get(drive["driveType"], ""), + "drive_id": drive["id"], + } + + result = [] + for drive in r.json()["value"]: + result.append(process_drive(drive)) + # Also call /me/drive as sometimes /me/drives doesn't return it + # see https://github.com/rclone/rclone/issues/4068 + r = requests.get( + "https://graph.microsoft.com/v1.0/me/drive", + headers={"Authorization": f"Bearer {token['access_token']}"}, + timeout=10, + ) + r.raise_for_status() + me_drive = process_drive(r.json()) + if me_drive not in result: + result.insert(0, me_drive) + return result diff --git a/src/middlewared/middlewared/utils/microsoft.py b/src/middlewared/middlewared/utils/microsoft.py new file mode 100644 index 0000000000000..093e2dbf72974 --- /dev/null +++ b/src/middlewared/middlewared/utils/microsoft.py @@ -0,0 +1,17 @@ +import requests + + +def get_microsoft_access_token(client_id: str, client_secret: str, refresh_token: str, scope: str) -> dict: + r = requests.post( + "https://login.microsoftonline.com/common/oauth2/v2.0/token", + data={ + "grant_type": "refresh_token", + "client_id": client_id, + "client_secret": client_secret, + "refresh_token": refresh_token, + "scope": scope, + }, + timeout=10, + ) + r.raise_for_status() + return r.json()