From 2dfbeee7cac5c3a936f1325cdd5b7259f332f483 Mon Sep 17 00:00:00 2001 From: Henri Dickson <90480431+alphatownsman@users.noreply.github.com> Date: Thu, 31 Aug 2023 19:56:12 -0400 Subject: [PATCH] add relay support --- activities/models/fan_out.py | 6 +- activities/models/post.py | 7 ++ activities/models/post_interaction.py | 8 +- activities/views/posts.py | 4 +- core/ld.py | 22 +++- core/models/config.py | 2 +- takahe/urls.py | 6 + templates/admin/_menu.html | 4 + templates/admin/identities.html | 1 + templates/admin/relays.html | 69 +++++++++++ users/apps.py | 3 +- users/models/__init__.py | 1 + users/models/follow.py | 8 +- users/models/identity.py | 5 + users/models/inbox_message.py | 5 + users/models/relay_actor.py | 164 ++++++++++++++++++++++++++ users/services/identity.py | 26 +++- users/views/activitypub.py | 18 ++- users/views/admin/__init__.py | 1 + users/views/admin/relays.py | 37 ++++++ 20 files changed, 377 insertions(+), 20 deletions(-) create mode 100644 templates/admin/relays.html create mode 100644 users/models/relay_actor.py create mode 100644 users/views/admin/relays.py diff --git a/activities/models/fan_out.py b/activities/models/fan_out.py index 9b3a169d0..0036de12f 100644 --- a/activities/models/fan_out.py +++ b/activities/models/fan_out.py @@ -82,7 +82,7 @@ def handle_new(cls, instance: "FanOut"): instance.identity.shared_inbox_uri or instance.identity.inbox_uri ), - body=canonicalise(post.to_create_ap()), + body=canonicalise(post.to_create_ap(), outbound_compat=True), ) except httpx.RequestError: return @@ -98,7 +98,7 @@ def handle_new(cls, instance: "FanOut"): instance.identity.shared_inbox_uri or instance.identity.inbox_uri ), - body=canonicalise(post.to_update_ap()), + body=canonicalise(post.to_update_ap(), outbound_compat=True), ) except httpx.RequestError: return @@ -124,7 +124,7 @@ def handle_new(cls, instance: "FanOut"): instance.identity.shared_inbox_uri or instance.identity.inbox_uri ), - body=canonicalise(post.to_delete_ap()), + body=canonicalise(post.to_delete_ap(), outbound_compat=True), ) except httpx.RequestError: return diff --git a/activities/models/post.py b/activities/models/post.py index 2cb3be7ae..3ec47577d 100644 --- a/activities/models/post.py +++ b/activities/models/post.py @@ -42,6 +42,7 @@ from users.models.hashtag_follow import HashtagFollow from users.models.identity import Identity, IdentityStates from users.models.inbox_message import InboxMessage +from users.models.relay_actor import RelayActor from users.models.system_actor import SystemActor @@ -779,6 +780,12 @@ def get_targets(self) -> Iterable[Identity]: targets.remove(block.target) except KeyError: pass + # send local-created public posts to relays + if self.local and self.visibility in [ + Post.Visibilities.public, + Post.Visibilities.unlisted, + ]: + targets.update(set(RelayActor.get_relays())) # Now dedupe the targets based on shared inboxes (we only keep one per # shared inbox) deduped_targets = set() diff --git a/activities/models/post_interaction.py b/activities/models/post_interaction.py index 13ea8242b..67709cba7 100644 --- a/activities/models/post_interaction.py +++ b/activities/models/post_interaction.py @@ -10,6 +10,7 @@ from core.snowflake import Snowflake from stator.models import State, StateField, StateGraph, StatorModel from users.models.identity import Identity +from users.models.relay_actor import RelayActor class PostInteractionStates(StateGraph): @@ -298,6 +299,11 @@ def to_ap(self) -> dict: "object": self.post.object_uri, "to": "as:Public", } + if self.identity.is_local_relay: + # if boost to relay, set "to" to remote relays instead of Public + value["to"] = list( + RelayActor.get_relays().values_list("actor_uri", flat=True) + ) elif self.type == self.Types.like: value = { "type": "Like", @@ -315,7 +321,7 @@ def to_ap(self) -> dict: "inReplyTo": self.post.object_uri, "attributedTo": self.identity.actor_uri, } - elif self.type == self.Types.pin: + else: raise ValueError("Cannot turn into AP") return value diff --git a/activities/views/posts.py b/activities/views/posts.py index 797a6df32..63d8618ad 100644 --- a/activities/views/posts.py +++ b/activities/views/posts.py @@ -67,6 +67,8 @@ def serve_object(self): if not self.post_obj.local: return redirect(self.post_obj.object_uri) return JsonResponse( - canonicalise(self.post_obj.to_ap(), include_security=True), + canonicalise( + self.post_obj.to_ap(), include_security=True, outbound_compat=True + ), content_type="application/activity+json", ) diff --git a/core/ld.py b/core/ld.py index f127f74ae..839500ff2 100644 --- a/core/ld.py +++ b/core/ld.py @@ -87,7 +87,7 @@ "attachment": {"@id": "as:attachment", "@type": "@id"}, "bcc": {"@id": "as:bcc", "@type": "@id"}, "bto": {"@id": "as:bto", "@type": "@id"}, - "cc": {"@id": "as:cc", "@type": "@id"}, + "cc": {"@id": "as:cc", "@type": "@id", "@container": "@set"}, "context": {"@id": "as:context", "@type": "@id"}, "current": {"@id": "as:current", "@type": "@id"}, "first": {"@id": "as:first", "@type": "@id"}, @@ -119,7 +119,7 @@ "partOf": {"@id": "as:partOf", "@type": "@id"}, "tag": {"@id": "as:tag", "@type": "@id"}, "target": {"@id": "as:target", "@type": "@id"}, - "to": {"@id": "as:to", "@type": "@id"}, + "to": {"@id": "as:to", "@type": "@id", "@container": "@set"}, "url": {"@id": "as:url", "@type": "@id"}, "altitude": {"@id": "as:altitude", "@type": "xsd:float"}, "content": "as:content", @@ -594,7 +594,9 @@ def builtin_document_loader(url: str, options={}): ) -def canonicalise(json_data: dict, include_security: bool = False) -> dict: +def canonicalise( + json_data: dict, include_security: bool = False, outbound_compat: bool = False +) -> dict: """ Given an ActivityPub JSON-LD document, round-trips it through the LD systems to end up in a canonicalised, compacted format. @@ -632,8 +634,18 @@ def canonicalise(json_data: dict, include_security: bool = False) -> dict: context.append("https://w3id.org/security/v1") json_data["@context"] = context - - return jsonld.compact(jsonld.expand(json_data), context) + j = jsonld.compact(jsonld.expand(json_data), context) + if outbound_compat: + # patch outbound json to make it compatible with various implementations + for k in ["to", "cc"]: + if j.get(k): + j[k] = [ + x + if x != "as:Public" + else "https://www.w3.org/ns/activitystreams#Public" + for x in j[k] + ] + return j def get_list(container, key) -> list: diff --git a/core/models/config.py b/core/models/config.py index 5dbff4443..6b4b49c35 100644 --- a/core/models/config.py +++ b/core/models/config.py @@ -230,7 +230,7 @@ class SystemOptions(pydantic.BaseModel): cache_timeout_page_post: int = 60 * 2 cache_timeout_identity_feed: int = 60 * 5 - restricted_usernames: str = "admin\nadmins\nadministrator\nadministrators\nsystem\nroot\nannounce\nannouncement\nannouncements" + restricted_usernames: str = "__system__\n__relay__\nadmin\nadmins\nadministrator\nadministrators\nsystem\nroot\nannounce\nannouncement\nannouncements" custom_head: str | None diff --git a/takahe/urls.py b/takahe/urls.py index cb833ae92..2dbae1aa8 100644 --- a/takahe/urls.py +++ b/takahe/urls.py @@ -138,6 +138,11 @@ "admin/domains//delete/", admin.DomainDelete.as_view(), ), + path( + "admin/relays/", + admin.RelayRoot.as_view(), + name="admin_relays", + ), path( "admin/federation/", admin.FederationRoot.as_view(), @@ -342,6 +347,7 @@ path("actor/inbox/", activitypub.Inbox.as_view()), path("actor/outbox/", activitypub.EmptyOutbox.as_view()), path("inbox/", activitypub.Inbox.as_view(), name="shared_inbox"), + path("relay", activitypub.RelayActorView.as_view()), # API/Oauth path("api/", include("api.urls")), path("oauth/authorize", oauth.AuthorizationView.as_view()), diff --git a/templates/admin/_menu.html b/templates/admin/_menu.html index 0b0ec10f3..07fe06aef 100644 --- a/templates/admin/_menu.html +++ b/templates/admin/_menu.html @@ -45,6 +45,10 @@

Administration

Federation + + + Relays + Users diff --git a/templates/admin/identities.html b/templates/admin/identities.html index 15bbea2b4..3cad8782c 100644 --- a/templates/admin/identities.html +++ b/templates/admin/identities.html @@ -52,6 +52,7 @@ Remote {{ identity.followers_count }} local follower{{ identity.followers_count|pluralize }} {% endif %} + {{ identity.actor_type }} diff --git a/templates/admin/relays.html b/templates/admin/relays.html new file mode 100644 index 000000000..7f54fc45d --- /dev/null +++ b/templates/admin/relays.html @@ -0,0 +1,69 @@ +{% extends "admin/base_main.html" %} +{% load activity_tags %} + +{% block subtitle %}Relay{% endblock %} + +{% block settings_content %} +
+ + {% csrf_token %} + +
+ + {% for identity in page_obj %} + + + + + + + + + {% empty %} + + + + {% endfor %} +
+ + + {{ identity.actor_uri }} + {{ identity.domain.nodeinfo.metadata.nodeName }} + {% if identity.domain.nodeinfo.software %} + {{ identity.domain.nodeinfo.software.name }} / {{ identity.domain.nodeinfo.software.version }} + {% endif %} + + {% if identity.restriction == 1 %} + Limited + {% elif identity.restriction == 2 %} + Blocked + {% endif %} + + {% if identity.follow_state == 'accepting' or identity.follow_state == 'accepted' %} + ACTIVE + {% elif identity.follow_state == 'unrequested' or identity.follow_state == 'pending_approval' %} + ACTIVATING + {% else %} + DEACTIVATING + {% endif %} + ({{ identity.follow_state }}) + +
+ + {% csrf_token %} + +
+ 		+
+ + {% csrf_token %} + +
+
+ There are no relay yet. +
+
+   invalid actor uri will not show in this list; (un)subscribing may take a while; use remove only when it's stuck in (DE)ACTIVATING state for a long time. +
+ {% include "admin/_pagination.html" with nouns="relay,relays" %} +{% endblock %} diff --git a/users/apps.py b/users/apps.py index 67183adfa..cc59009e1 100644 --- a/users/apps.py +++ b/users/apps.py @@ -12,9 +12,10 @@ def data_init(self, **kwargs): boot (or post upgrade). """ # Generate the server actor keypair if needed - from users.models import SystemActor + from users.models import RelayActor, SystemActor SystemActor.generate_keys_if_needed() + RelayActor.initialize_if_needed() def ready(self) -> None: post_migrate.connect(self.data_init, sender=self) diff --git a/users/models/__init__.py b/users/models/__init__.py index 8396e424e..c0795ce25 100644 --- a/users/models/__init__.py +++ b/users/models/__init__.py @@ -8,6 +8,7 @@ from .inbox_message import InboxMessage, InboxMessageStates # noqa from .invite import Invite # noqa from .password_reset import PasswordReset # noqa +from .relay_actor import RelayActor # noqa from .report import Report # noqa from .system_actor import SystemActor # noqa from .user import User # noqa diff --git a/users/models/follow.py b/users/models/follow.py index cd1d0bad9..fde6aeb53 100644 --- a/users/models/follow.py +++ b/users/models/follow.py @@ -74,7 +74,7 @@ def handle_unrequested(cls, instance: "Follow"): instance.source.signed_request( method="post", uri=instance.target.inbox_uri, - body=canonicalise(instance.to_ap()), + body=canonicalise(instance.to_ap(), outbound_compat=True), ) except httpx.RequestError: return @@ -242,7 +242,11 @@ def create_local(cls, source, target, boosts=True): uri="", state=FollowStates.unrequested, ) - follow.uri = source.actor_uri + f"follow/{follow.pk}/" + follow.uri = ( + source.actor_uri + + ("/" if source.actor_uri[-1] != "/" else "") + + f"follow/{follow.pk}/" + ) follow.save() return follow diff --git a/users/models/identity.py b/users/models/identity.py index 41c2f4272..469dd0f9a 100644 --- a/users/models/identity.py +++ b/users/models/identity.py @@ -32,6 +32,7 @@ ) from stator.models import State, StateField, StateGraph, StatorModel from users.models.domain import Domain +from users.models.relay_actor import RelayActor from users.models.system_actor import SystemActor @@ -337,6 +338,10 @@ def safe_metadata(self): for data in self.metadata ] + @property + def is_local_relay(self): + return self.local and self.actor_uri == RelayActor.actor_uri + def ensure_uris(self): """ Ensures that local identities have all the URIs populated on their fields diff --git a/users/models/inbox_message.py b/users/models/inbox_message.py index 9dba3dba9..ac745f37f 100644 --- a/users/models/inbox_message.py +++ b/users/models/inbox_message.py @@ -3,6 +3,7 @@ from core.exceptions import ActivityPubError from stator.models import State, StateField, StateGraph, StatorModel +from users.models.relay_actor import RelayActor class InboxMessageStates(StateGraph): @@ -141,6 +142,10 @@ def handle_received(cls, instance: "InboxMessage"): IdentityService.handle_internal_add_follow( instance.message["object"] ) + case "unfollowrelay": + RelayActor.handle_internal_unfollow( + instance.message["object"] + ) case unknown: return cls.errored case unknown: diff --git a/users/models/relay_actor.py b/users/models/relay_actor.py new file mode 100644 index 000000000..3edd9101a --- /dev/null +++ b/users/models/relay_actor.py @@ -0,0 +1,164 @@ +from django.conf import settings + +from core.signatures import RsaKeys + + +class RelayActor: + """ + Pesudo actor for subscribing relay services from local instance + + Relay services typically supports two modes: Mastodon and LitePub. + + in Mastodon mode: + instance handshake with relay by sending Follow[1] to relay's inbox, relay won't follow back + instance publishes its activities by sending Create/Update/Delete to relay's inbox + relay publishes remote activities by sending Announce to instance's inbox + + in LitePub mode (also known as Pleroma mode): + instance queries relay server's actor uri, sends Follow to relay's inbox and gets Follow back + instance publishes its activities by sending Announce[2] to relay's inbox + relay publishes remote activities by sending Announce to instance's inbox + + Our implementation do initial handshake in LitePub mode, then send activities in Mastodon mode; + Most if not all modern relay implementations seem fine with this mixed mode + + [1] in this Follow and its Accept/Reject, target actor uri is unknown, "object" has to be + "https://www.w3.org/ns/activitystreams#Public" + [2] in this Announce, "to" has to be the relay's actor uri + [3] various implementation has strict validations, e.g. + instance actor uri has to ends with "/relay" + instance actor type must be Application + "to":"as:Public" must become "to":["https://www.w3.org/ns/activitystreams#Public"] + """ + + actor_uri = f"https://{settings.MAIN_DOMAIN}/relay" + inbox_uri = f"https://{settings.MAIN_DOMAIN}/inbox/" + handle = f"__relay__@{settings.MAIN_DOMAIN}" + _private_key = None + _public_key = None + + @classmethod + def subscribe(cls, relay_uri): + from .inbox_message import InboxMessage + + InboxMessage.create_internal( + { + "type": "AddFollow", + "source": cls.get_identity().pk, + "target_actor": relay_uri, + "boosts": False, + } + ) + + @classmethod + def unsubscribe(cls, relay_uri): + from .inbox_message import InboxMessage + + InboxMessage.create_internal( + { + "type": "UnfollowRelay", + "actor_uri": relay_uri, + } + ) + + @classmethod + def remove(cls, relay_uri): + from .follow import Follow + + Follow.objects.filter( + source__actor_uri=cls.actor_uri, target__actor_uri=relay_uri + ).delete() + Follow.objects.filter( + target__actor_uri=cls.actor_uri, source__actor_uri=relay_uri + ).delete() + + @classmethod + def handle_internal_unfollow(cls, payload): + """ + Handles actual unfollow from relay and remove queued fanout + """ + from activities.models import FanOut, FanOutStates + + from ..services.identity import IdentityService + from .identity import Identity + + relay_uri = payload["actor_uri"] + relay = Identity.objects.get(actor_uri=relay_uri) + svc = IdentityService(cls.get_identity()) + svc.unfollow(relay) + svc.reject_follow_request(relay) + FanOut.transition_perform_queryset( + FanOut.objects.filter(identity=relay), FanOutStates.skipped + ) + + @classmethod + def get_identity(cls): + from users.models import Identity + + return Identity.objects.get(actor_uri=cls.actor_uri) + + @classmethod + def get_relays(cls): + from .follow import FollowStates + from .identity import Identity + + return Identity.objects.not_deleted().filter( + inbound_follows__source=cls.get_identity(), + inbound_follows__state=FollowStates.accepted, + ) + + @classmethod + def initialize_if_needed(cls): + from users.models import Identity + + if not Identity.objects.filter(actor_uri=cls.actor_uri).exists(): + _private_key, _public_key = RsaKeys.generate_keypair() + Identity.objects.create( + username="__relay__", + # domain_id= settings.MAIN_DOMAIN, + name="System Relay Actor", + actor_uri=cls.actor_uri, + actor_type="Application", + local=True, + discoverable=False, + manually_approves_followers=False, + private_key=_private_key, + public_key=_public_key, + public_key_id=cls.actor_uri + "#main-key", + inbox_uri=cls.inbox_uri, + shared_inbox_uri=cls.inbox_uri, + ) + + @classmethod + def to_ap(cls): + identity = cls.get_identity() + return { + "id": identity.actor_uri, + "type": identity.actor_type, + "inbox": identity.shared_inbox_uri, + # "outbox": identity.outbox_uri, + "endpoints": { + "sharedInbox": identity.shared_inbox_uri, + }, + "preferredUsername": identity.username, + "name": identity.name, + "manuallyApprovesFollowers": identity.manually_approves_followers, + "toot:discoverable": identity.discoverable, + "publicKey": { + "id": identity.public_key_id, + "owner": identity.actor_uri, + "publicKeyPem": identity.public_key, + }, + } + + def to_webfinger(self): + return { + "subject": f"acct:{self.handle}", + "links": [ + { + "rel": "self", + "type": "application/activity+json", + "href": self.actor_uri, + }, + ], + } diff --git a/users/services/identity.py b/users/services/identity.py index 2a29171a7..9e9cbb17d 100644 --- a/users/services/identity.py +++ b/users/services/identity.py @@ -300,22 +300,38 @@ def set_image(self, file): @classmethod def handle_internal_add_follow(cls, payload): """ - Handles an inbox message saying we need to follow a handle + Handles an inbox message saying we need to follow a handle or actor Message format: { "type": "AddFollow", "source": "90310938129083", "target_handle": "andrew@aeracode.org", + "target_actor": "https://aeracode.org/@andrew" "boosts": true, } """ # Retrieve ourselves self = cls(Identity.objects.get(pk=payload["source"])) # Get the remote end (may need a fetch) - username, domain = payload["target_handle"].split("@") - target_identity = Identity.by_username_and_domain(username, domain, fetch=True) - if target_identity is None: - raise ValueError(f"Cannot find identity to follow: {target_identity}") + if payload.get("target_actor"): + target_identity = Identity.by_actor_uri( + payload["target_actor"], create=True, transient=True + ) + if not target_identity.public_key: + target_identity.fetch_actor() + if not target_identity.public_key: + raise ValueError( + f"Cannot fetch actor to follow: {payload['target_actor']}" + ) + else: + username, domain = payload["target_handle"].split("@") + target_identity = Identity.by_username_and_domain( + username, domain, fetch=True + ) + if target_identity is None: + raise ValueError( + f"Cannot find identity to follow: {payload['target_handle']}" + ) # Follow! self.follow(target_identity=target_identity, boosts=payload.get("boosts", True)) diff --git a/users/views/activitypub.py b/users/views/activitypub.py index 461840ee6..351da12a9 100644 --- a/users/views/activitypub.py +++ b/users/views/activitypub.py @@ -21,7 +21,7 @@ ) from core.views import StaticContentView from takahe import __version__ -from users.models import Identity, InboxMessage, SystemActor +from users.models import Identity, InboxMessage, RelayActor, SystemActor from users.shortcuts import by_handle_or_404 @@ -118,6 +118,9 @@ def get(self, request): if handle.startswith("__system__@"): # They are trying to webfinger the system actor actor = SystemActor() + elif handle.startswith("__relay__@"): + # They are trying to webfinger the system actor + actor = RelayActor() else: actor = by_handle_or_404(request, handle) @@ -296,3 +299,16 @@ def get_static_content(self) -> str | bytes: include_security=True, ) ) + + +@method_decorator(cache_control(max_age=60 * 15), name="dispatch") +class RelayActorView(StaticContentView): + content_type: str = "application/activity+json" + + def get_static_content(self) -> str | bytes: + return json.dumps( + canonicalise( + RelayActor.to_ap(), + include_security=True, + ) + ) diff --git a/users/views/admin/__init__.py b/users/views/admin/__init__.py index 3330b1bf8..e4305ab69 100644 --- a/users/views/admin/__init__.py +++ b/users/views/admin/__init__.py @@ -31,6 +31,7 @@ from users.views.admin.hashtags import HashtagEdit, HashtagEnable, Hashtags # noqa from users.views.admin.identities import IdentitiesRoot, IdentityEdit # noqa from users.views.admin.invites import InviteCreate, InvitesRoot, InviteView # noqa +from users.views.admin.relays import RelayRoot # noqa from users.views.admin.reports import ReportsRoot, ReportView # noqa from users.views.admin.settings import ( # noqa BasicSettings, diff --git a/users/views/admin/relays.py b/users/views/admin/relays.py new file mode 100644 index 000000000..bce676a1c --- /dev/null +++ b/users/views/admin/relays.py @@ -0,0 +1,37 @@ +from django.db import models +from django.shortcuts import redirect +from django.utils.decorators import method_decorator +from django.views.generic import ListView + +from users.decorators import admin_required +from users.models import Identity, RelayActor + + +@method_decorator(admin_required, name="dispatch") +class RelayRoot(ListView): + template_name = "admin/relays.html" + paginate_by = 30 + + def get(self, request, *args, **kwargs): + self.extra_context = { + "section": "relays", + } + return super().get(request, *args, **kwargs) + + def get_queryset(self): + identities = ( + Identity.objects.filter(inbound_follows__source=RelayActor.get_identity()) + .annotate(follow_state=models.F("inbound_follows__state")) + .order_by("-created") + ) + return identities + + def post(self, request, *args, **kwargs): + actor_uri = request.POST.get("actor_uri") + if "subscribe" in request.GET: + RelayActor.subscribe(actor_uri) + elif "unsubscribe" in request.GET: + RelayActor.unsubscribe(actor_uri) + elif "remove" in request.GET: + RelayActor.remove(actor_uri) + return redirect(".")