From 29eabba5a0ce4124058abba14313e897b3207609 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 1 Aug 2024 19:22:37 +0200 Subject: [PATCH 01/66] fix links --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbee7112..ee9675f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,8 @@ - fix metadata dictproxy which would confuse transactions resulting in missed notifications and other issues. - ([#393](https://github.com/deltachat/chatmail/pull/388)) - ([#394](https://github.com/deltachat/chatmail/pull/389)) + ([#393](https://github.com/deltachat/chatmail/pull/393)) + ([#394](https://github.com/deltachat/chatmail/pull/394)) - add optional "imap_rawlog" config option. If true, .in/.out files are created in user home dirs From effd5bc6e98ad375020ae6a53047fae279a4322f Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 2 Aug 2024 01:03:28 +0200 Subject: [PATCH 02/66] upgrade debian packages on "cmdeploy run" --- CHANGELOG.md | 3 +++ cmdeploy/src/cmdeploy/__init__.py | 1 + 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee9675f3..ec3d0ad3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## untagged +- trigger "apt upgrade" during "cmdeploy run" + ([#398](https://github.com/deltachat/chatmail/pull/398)) + ## 1.4.1 2024-07-31 - fix metadata dictproxy which would confuse transactions diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index 0fdc3e0a..ace9e47b 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -489,6 +489,7 @@ def deploy_chatmail(config_path: Path) -> None: ) apt.update(name="apt update", cache_time=24 * 3600) + apt.upgrade(name="upgrade apt packages", auto_remove=True) apt.packages( name="Install rsync", From dee36638cf1991af41093e2cfcf685661a373490 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 6 Aug 2024 10:24:28 +0200 Subject: [PATCH 03/66] fix #399 --- CHANGELOG.md | 2 ++ cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec3d0ad3..befe59ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## untagged +- avoid nginx listening on ipv6 if v6 is dsiabled + ([#402](https://github.com/deltachat/chatmail/pull/402)) - trigger "apt upgrade" during "cmdeploy run" ([#398](https://github.com/deltachat/chatmail/pull/398)) diff --git a/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 b/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 index 5c36a15f..cb53a1b8 100644 --- a/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 +++ b/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 @@ -19,7 +19,9 @@ stream { server { listen 443; + {% if not disable_ipv6 %} listen [::]:443; + {% endif %} proxy_pass $proxy; ssl_preread on; } From 7aa876a0bb8fad6be0acd03476d9badee772ca2a Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 6 Aug 2024 08:44:14 +0200 Subject: [PATCH 04/66] remove dysfunct hispanilandia ref --- CHANGELOG.md | 4 ++++ chatmaild/src/chatmaild/ini/chatmail.ini.f | 3 ++- chatmaild/src/chatmaild/ini/override-testrun.ini | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index befe59ef..94ea4ddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ - trigger "apt upgrade" during "cmdeploy run" ([#398](https://github.com/deltachat/chatmail/pull/398)) +- drop hispanilandia passthrough address + ([#401](https://github.com/deltachat/chatmail/pull/401)) + + ## 1.4.1 2024-07-31 - fix metadata dictproxy which would confuse transactions diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index 03d844bc..9bccdcd4 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -39,7 +39,8 @@ passthrough_senders = # list of e-mail recipients for which to accept outbound un-encrypted mails -passthrough_recipients = xstore@testrun.org groupsbot@hispanilandia.net +# (space-separated) +passthrough_recipients = xstore@testrun.org # # Deployment Details diff --git a/chatmaild/src/chatmaild/ini/override-testrun.ini b/chatmaild/src/chatmaild/ini/override-testrun.ini index a95a2895..b3c32f17 100644 --- a/chatmaild/src/chatmaild/ini/override-testrun.ini +++ b/chatmaild/src/chatmaild/ini/override-testrun.ini @@ -1,7 +1,7 @@ [privacy] -passthrough_recipients = privacy@testrun.org xstore@testrun.org groupsbot@hispanilandia.net +passthrough_recipients = privacy@testrun.org xstore@testrun.org privacy_postal = Merlinux GmbH, Represented by the managing director H. Krekel, From a1e80fdca16f9f35e3292c48f0b0888fa6a7b577 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 23 Aug 2024 11:57:47 +0000 Subject: [PATCH 05/66] Fix ruff warnings --- chatmaild/src/chatmaild/tests/plugin.py | 1 + chatmaild/src/chatmaild/tests/test_config.py | 1 + chatmaild/src/chatmaild/tests/test_doveauth.py | 3 ++- chatmaild/src/chatmaild/tests/test_filtermail.py | 1 + chatmaild/src/chatmaild/tests/test_metadata.py | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/chatmaild/src/chatmaild/tests/plugin.py b/chatmaild/src/chatmaild/tests/plugin.py index c61fd9b1..00b4c34a 100644 --- a/chatmaild/src/chatmaild/tests/plugin.py +++ b/chatmaild/src/chatmaild/tests/plugin.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest + from chatmaild.config import read_config, write_initial_config diff --git a/chatmaild/src/chatmaild/tests/test_config.py b/chatmaild/src/chatmaild/tests/test_config.py index 2ab96344..63d237cb 100644 --- a/chatmaild/src/chatmaild/tests/test_config.py +++ b/chatmaild/src/chatmaild/tests/test_config.py @@ -1,4 +1,5 @@ import pytest + from chatmaild.config import read_config diff --git a/chatmaild/src/chatmaild/tests/test_doveauth.py b/chatmaild/src/chatmaild/tests/test_doveauth.py index b6c828f5..078bec42 100644 --- a/chatmaild/src/chatmaild/tests/test_doveauth.py +++ b/chatmaild/src/chatmaild/tests/test_doveauth.py @@ -4,8 +4,9 @@ import threading import traceback -import chatmaild.doveauth import pytest + +import chatmaild.doveauth from chatmaild.doveauth import ( AuthDictProxy, is_allowed_to_create, diff --git a/chatmaild/src/chatmaild/tests/test_filtermail.py b/chatmaild/src/chatmaild/tests/test_filtermail.py index 74c28bb2..6072d656 100644 --- a/chatmaild/src/chatmaild/tests/test_filtermail.py +++ b/chatmaild/src/chatmaild/tests/test_filtermail.py @@ -1,4 +1,5 @@ import pytest + from chatmaild.filtermail import ( BeforeQueueHandler, SendRateLimiter, diff --git a/chatmaild/src/chatmaild/tests/test_metadata.py b/chatmaild/src/chatmaild/tests/test_metadata.py index d7778a34..3b2da96a 100644 --- a/chatmaild/src/chatmaild/tests/test_metadata.py +++ b/chatmaild/src/chatmaild/tests/test_metadata.py @@ -3,6 +3,7 @@ import pytest import requests + from chatmaild.metadata import ( Metadata, MetadataDictProxy, From cdfce25494e87d0d81a5768fc6294f9b71553565 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 2 Sep 2024 18:35:56 +0200 Subject: [PATCH 06/66] add a note on deletion of accounts --- chatmaild/src/chatmaild/ini/chatmail.ini.f | 4 ++-- www/src/info.md | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index 9bccdcd4..b2a5ff12 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -23,8 +23,8 @@ # days after which mails are unconditionally deleted delete_mails_after = 20 -# days after which users without a login are deleted (database and mails) -delete_inactive_users_after = 100 +# days after which users without a successful login are deleted (database and mails) +delete_inactive_users_after = 90 # minimum length a username must have username_min_length = 9 diff --git a/www/src/info.md b/www/src/info.md index 4a8126b2..3556249f 100644 --- a/www/src/info.md +++ b/www/src/info.md @@ -43,6 +43,20 @@ The first login sets your password. - You can store up to [{{ config.max_mailbox_size }} messages on the server](https://delta.chat/en/help#what-happens-if-i-turn-on-delete-old-messages-from-server). +### Account deletion + +If you remove a {{ config.mail_domain }} profile from within the Delta Chat app, +then the according account on the server, along with all associated data, +is automatically deleted {{ config.delete_inactive_users_after }} days afterwards. + +If you use multiple devices +then you need to remove the according chat profile from each device +in order for all account data to be removed on the server side. + +If you have any further questions or requests regarding account deletion +please send a message from your account to {{ config.privacy_mail }}. + + ### Who are the operators? Which software is running? This chatmail provider is run by a small voluntary group of devs and sysadmins, From e973bc1f41245326658187b87dd8629675e4828f Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 1 Aug 2024 19:02:41 +0200 Subject: [PATCH 07/66] organize remotely executing functions in "cmdeploy.remote" sub package --- CHANGELOG.md | 4 ++ chatmaild/src/chatmaild/__init__.py | 1 + cmdeploy/src/cmdeploy/cmdeploy.py | 6 +-- cmdeploy/src/cmdeploy/dns.py | 6 +-- cmdeploy/src/cmdeploy/remote/__init__.py | 12 +++++ .../src/cmdeploy/remote/_sshexec_bootstrap.py | 34 ++++++++++++ .../{remote_funcs.py => remote/rdns.py} | 53 +------------------ cmdeploy/src/cmdeploy/remote/rshell.py | 17 ++++++ cmdeploy/src/cmdeploy/sshexec.py | 41 ++++++++++++-- .../src/cmdeploy/tests/online/test_1_basic.py | 20 +++---- cmdeploy/src/cmdeploy/tests/test_dns.py | 14 ++--- 11 files changed, 131 insertions(+), 77 deletions(-) create mode 100644 cmdeploy/src/cmdeploy/remote/__init__.py create mode 100644 cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py rename cmdeploy/src/cmdeploy/{remote_funcs.py => remote/rdns.py} (67%) create mode 100644 cmdeploy/src/cmdeploy/remote/rshell.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ea4ddc..17630aff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ - avoid nginx listening on ipv6 if v6 is dsiabled ([#402](https://github.com/deltachat/chatmail/pull/402)) +- refactor ssh-based execution to allow organizing remote functions in + modules. + ([#396](https://github.com/deltachat/chatmail/pull/396)) + - trigger "apt upgrade" during "cmdeploy run" ([#398](https://github.com/deltachat/chatmail/pull/398)) diff --git a/chatmaild/src/chatmaild/__init__.py b/chatmaild/src/chatmaild/__init__.py index e69de29b..8b137891 100644 --- a/chatmaild/src/chatmaild/__init__.py +++ b/chatmaild/src/chatmaild/__init__.py @@ -0,0 +1 @@ + diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index 67d038eb..cd992a2b 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -18,7 +18,7 @@ from packaging import version from termcolor import colored -from . import dns, remote_funcs +from . import dns, remote from .sshexec import SSHExec # @@ -132,7 +132,7 @@ def status_cmd(args, out): else: out.red("no privacy settings") - for line in sshexec(remote_funcs.get_systemd_running): + for line in sshexec(remote.rshell.get_systemd_running): print(line) @@ -313,7 +313,7 @@ def main(args=None): def get_sshexec(): print(f"[ssh] login to {args.config.mail_domain}") - return SSHExec(args.config.mail_domain, remote_funcs, verbose=args.verbose) + return SSHExec(args.config.mail_domain, verbose=args.verbose) args.get_sshexec = get_sshexec diff --git a/cmdeploy/src/cmdeploy/dns.py b/cmdeploy/src/cmdeploy/dns.py index 7d2e9c43..e4b95f90 100644 --- a/cmdeploy/src/cmdeploy/dns.py +++ b/cmdeploy/src/cmdeploy/dns.py @@ -3,12 +3,12 @@ from jinja2 import Template -from . import remote_funcs +from . import remote def get_initial_remote_data(sshexec, mail_domain): return sshexec.logged( - call=remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=mail_domain) + call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain) ) @@ -42,7 +42,7 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int: and return (exitcode, remote_data) tuple.""" required_diff, recommended_diff = sshexec.logged( - remote_funcs.check_zonefile, kwargs=dict(zonefile=zonefile) + remote.rdns.check_zonefile, kwargs=dict(zonefile=zonefile) ) if required_diff: diff --git a/cmdeploy/src/cmdeploy/remote/__init__.py b/cmdeploy/src/cmdeploy/remote/__init__.py new file mode 100644 index 00000000..c300b44d --- /dev/null +++ b/cmdeploy/src/cmdeploy/remote/__init__.py @@ -0,0 +1,12 @@ +""" + +The 'cmdeploy.remote' sub package contains modules with remotely executing functions. + +Its "_sshexec_bootstrap" module is executed remotely through `SSHExec` +and its main() loop there stays connected via a command channel, +ready to receive function invocations ("command") and return results. +""" + +from . import rdns, rshell + +__all__ = ["rdns", "rshell"] diff --git a/cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py b/cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py new file mode 100644 index 00000000..f5b4c083 --- /dev/null +++ b/cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py @@ -0,0 +1,34 @@ +import builtins +import importlib +import traceback + +## Function Execution server + + +def _run_loop(cmd_channel): + while 1: + cmd = cmd_channel.receive() + if cmd is None: + break + + cmd_channel.send(_handle_one_request(cmd)) + + +def _handle_one_request(cmd): + pymod_path, func_name, kwargs = cmd + try: + mod = importlib.import_module(pymod_path) + func = getattr(mod, func_name) + res = func(**kwargs) + return ("finish", res) + except: + data = traceback.format_exc() + return ("error", data) + + +def main(channel): + # enable simple "print" logging + + builtins.print = lambda x="": channel.send(("log", x)) + + _run_loop(channel) diff --git a/cmdeploy/src/cmdeploy/remote_funcs.py b/cmdeploy/src/cmdeploy/remote/rdns.py similarity index 67% rename from cmdeploy/src/cmdeploy/remote_funcs.py rename to cmdeploy/src/cmdeploy/remote/rdns.py index 12ff3ff4..3402f53f 100644 --- a/cmdeploy/src/cmdeploy/remote_funcs.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -11,23 +11,8 @@ """ import re -import traceback -from subprocess import CalledProcessError, check_output - -def shell(command, fail_ok=False): - print(f"$ {command}") - try: - return check_output(command, shell=True).decode().rstrip() - except CalledProcessError: - if not fail_ok: - raise - return "" - - -def get_systemd_running(): - lines = shell("systemctl --type=service --state=running").split("\n") - return [line for line in lines if line.startswith(" ")] +from .rshell import ShellError, shell def perform_initial_checks(mail_domain): @@ -59,7 +44,7 @@ def get_dkim_entry(mail_domain, dkim_selector): f"openssl rsa -in /etc/dkimkeys/{dkim_selector}.private " "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'" ) - except CalledProcessError: + except ShellError: return dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s" dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw)) @@ -99,37 +84,3 @@ def check_zonefile(zonefile): recommended_diff.append(zf_line) return required_diff, recommended_diff - - -## Function Execution server - - -def _run_loop(cmd_channel): - while 1: - cmd = cmd_channel.receive() - if cmd is None: - break - - cmd_channel.send(_handle_one_request(cmd)) - - -def _handle_one_request(cmd): - func_name, kwargs = cmd - try: - res = globals()[func_name](**kwargs) - return ("finish", res) - except: - data = traceback.format_exc() - return ("error", data) - - -# check if this module is executed remotely -# and setup a simple serialized function-execution loop - -if __name__ == "__channelexec__": - channel = channel # noqa (channel object gets injected) - - # enable simple "print" logging for anyone changing this module - globals()["print"] = lambda x="": channel.send(("log", x)) - - _run_loop(channel) diff --git a/cmdeploy/src/cmdeploy/remote/rshell.py b/cmdeploy/src/cmdeploy/remote/rshell.py new file mode 100644 index 00000000..994223bc --- /dev/null +++ b/cmdeploy/src/cmdeploy/remote/rshell.py @@ -0,0 +1,17 @@ +from subprocess import CalledProcessError as ShellError +from subprocess import check_output + + +def shell(command, fail_ok=False): + print(f"$ {command}") + try: + return check_output(command, shell=True).decode().rstrip() + except ShellError: + if not fail_ok: + raise + return "" + + +def get_systemd_running(): + lines = shell("systemctl --type=service --state=running").split("\n") + return [line for line in lines if line.startswith(" ")] diff --git a/cmdeploy/src/cmdeploy/sshexec.py b/cmdeploy/src/cmdeploy/sshexec.py index 474c04db..8a87e781 100644 --- a/cmdeploy/src/cmdeploy/sshexec.py +++ b/cmdeploy/src/cmdeploy/sshexec.py @@ -1,12 +1,45 @@ +import inspect +import os import sys +from queue import Queue import execnet +from . import remote + class FuncError(Exception): pass +def bootstrap_remote(gateway, remote=remote): + """Return a command channel which can execute remote functions.""" + source_init_path = inspect.getfile(remote) + basedir = os.path.dirname(source_init_path) + name = os.path.basename(basedir) + + # rsync sourcedir to remote host + remote_pkg_path = f"/root/from-cmdeploy/{name}" + q = Queue() + finish = lambda: q.put(None) + rsync = execnet.RSync(sourcedir=basedir, verbose=False) + rsync.add_target(gateway, remote_pkg_path, finishedcallback=finish, delete=True) + rsync.send() + q.get() + + # start sshexec bootstrap and return its command channel + remote_sys_path = os.path.dirname(remote_pkg_path) + channel = gateway.remote_exec( + f""" + import sys + sys.path.insert(0, {remote_sys_path!r}) + from remote._sshexec_bootstrap import main + main(channel) + """ + ) + return channel + + def print_stderr(item="", end="\n"): print(item, file=sys.stderr, end=end) @@ -15,16 +48,18 @@ class SSHExec: RemoteError = execnet.RemoteError FuncError = FuncError - def __init__(self, host, remote_funcs, verbose=False, python="python3", timeout=60): + def __init__(self, host, verbose=False, python="python3", timeout=60): self.gateway = execnet.makegateway(f"ssh=root@{host}//python={python}") - self._remote_cmdloop_channel = self.gateway.remote_exec(remote_funcs) + self._remote_cmdloop_channel = bootstrap_remote(self.gateway, remote) self.timeout = timeout self.verbose = verbose def __call__(self, call, kwargs=None, log_callback=None): if kwargs is None: kwargs = {} - self._remote_cmdloop_channel.send((call.__name__, kwargs)) + assert call.__module__.startswith("cmdeploy.remote") + modname = call.__module__.replace("cmdeploy.", "") + self._remote_cmdloop_channel.send((modname, call.__name__, kwargs)) while 1: code, data = self._remote_cmdloop_channel.receive(timeout=self.timeout) if log_callback is not None and code == "log": diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index 0d3ecb8f..f5f2c023 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -2,29 +2,29 @@ import pytest -from cmdeploy import remote_funcs +from cmdeploy import remote from cmdeploy.sshexec import SSHExec class TestSSHExecutor: @pytest.fixture(scope="class") def sshexec(self, sshdomain): - return SSHExec(sshdomain, remote_funcs) + return SSHExec(sshdomain) def test_ls(self, sshexec): - out = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls")) - out2 = sshexec(call=remote_funcs.shell, kwargs=dict(command="ls")) + out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls")) + out2 = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls")) assert out == out2 def test_perform_initial(self, sshexec, maildomain): res = sshexec( - remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain) + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) ) assert res["A"] or res["AAAA"] def test_logged(self, sshexec, maildomain, capsys): sshexec.logged( - remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain) + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) ) out, err = capsys.readouterr() assert err.startswith("Collecting") @@ -33,21 +33,21 @@ def test_logged(self, sshexec, maildomain, capsys): sshexec.verbose = True sshexec.logged( - remote_funcs.perform_initial_checks, kwargs=dict(mail_domain=maildomain) + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) ) out, err = capsys.readouterr() lines = err.split("\n") assert len(lines) > 4 - assert remote_funcs.perform_initial_checks.__doc__ in lines[0] + assert remote.rdns.perform_initial_checks.__doc__ in lines[0] def test_exception(self, sshexec, capsys): try: sshexec.logged( - remote_funcs.perform_initial_checks, + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=None), ) except sshexec.FuncError as e: - assert "remote_funcs.py" in str(e) + assert "rdns.py" in str(e) assert "AssertionError" in str(e) else: pytest.fail("didn't raise exception") diff --git a/cmdeploy/src/cmdeploy/tests/test_dns.py b/cmdeploy/src/cmdeploy/tests/test_dns.py index eba0e904..71b1baa5 100644 --- a/cmdeploy/src/cmdeploy/tests/test_dns.py +++ b/cmdeploy/src/cmdeploy/tests/test_dns.py @@ -1,6 +1,6 @@ import pytest -from cmdeploy import remote_funcs +from cmdeploy import remote from cmdeploy.dns import check_full_zone, check_initial_remote_data @@ -14,7 +14,7 @@ def query_dns(typ, domain): except KeyError: return "" - monkeypatch.setattr(remote_funcs, query_dns.__name__, query_dns) + monkeypatch.setattr(remote.rdns, query_dns.__name__, query_dns) return qdict @@ -32,13 +32,13 @@ def mockdns(mockdns_base): class TestPerformInitialChecks: def test_perform_initial_checks_ok1(self, mockdns): - remote_data = remote_funcs.perform_initial_checks("some.domain") + remote_data = remote.rdns.perform_initial_checks("some.domain") assert len(remote_data) == 7 @pytest.mark.parametrize("drop", ["A", "AAAA"]) def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop): del mockdns[drop] - remote_data = remote_funcs.perform_initial_checks("some.domain") + remote_data = remote.rdns.perform_initial_checks("some.domain") assert len(remote_data) == 7 assert not remote_data[drop] @@ -49,7 +49,7 @@ def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop): def test_perform_initial_checks_no_mta_sts(self, mockdns): del mockdns["CNAME"] - remote_data = remote_funcs.perform_initial_checks("some.domain") + remote_data = remote.rdns.perform_initial_checks("some.domain") assert len(remote_data) == 4 assert not remote_data["MTA_STS"] @@ -85,14 +85,14 @@ class TestZonefileChecks: def test_check_zonefile_all_ok(self, cm_data, mockdns_base): zonefile = cm_data.get("zftest.zone") parse_zonefile_into_dict(zonefile, mockdns_base) - required_diff, recommended_diff = remote_funcs.check_zonefile(zonefile) + required_diff, recommended_diff = remote.rdns.check_zonefile(zonefile) assert not required_diff and not recommended_diff def test_check_zonefile_recommended_not_set(self, cm_data, mockdns_base): zonefile = cm_data.get("zftest.zone") zonefile_mocked = zonefile.split("; Recommended")[0] parse_zonefile_into_dict(zonefile_mocked, mockdns_base) - required_diff, recommended_diff = remote_funcs.check_zonefile(zonefile) + required_diff, recommended_diff = remote.rdns.check_zonefile(zonefile) assert not required_diff assert len(recommended_diff) == 8 From e32d81520a29b62a1582ad96702a962c86d9f37f Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 13 Aug 2024 14:50:27 +0200 Subject: [PATCH 08/66] use "walrus" operator (didn't know about it, doh!) --- cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py b/cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py index f5b4c083..443dfdd1 100644 --- a/cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py +++ b/cmdeploy/src/cmdeploy/remote/_sshexec_bootstrap.py @@ -6,11 +6,7 @@ def _run_loop(cmd_channel): - while 1: - cmd = cmd_channel.receive() - if cmd is None: - break - + while cmd := cmd_channel.receive(): cmd_channel.send(_handle_one_request(cmd)) From 8d72d770a3f32a045f8e7e39e01b24fb1590df63 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 2 Sep 2024 18:25:34 +0200 Subject: [PATCH 09/66] don't rename import as link2xt prefers --- cmdeploy/src/cmdeploy/remote/rdns.py | 4 ++-- cmdeploy/src/cmdeploy/remote/rshell.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cmdeploy/src/cmdeploy/remote/rdns.py b/cmdeploy/src/cmdeploy/remote/rdns.py index 3402f53f..ea0ec3e9 100644 --- a/cmdeploy/src/cmdeploy/remote/rdns.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -12,7 +12,7 @@ import re -from .rshell import ShellError, shell +from .rshell import CalledProcessError, shell def perform_initial_checks(mail_domain): @@ -44,7 +44,7 @@ def get_dkim_entry(mail_domain, dkim_selector): f"openssl rsa -in /etc/dkimkeys/{dkim_selector}.private " "-pubout 2>/dev/null | awk '/-/{next}{printf(\"%s\",$0)}'" ) - except ShellError: + except CalledProcessError: return dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s" dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw)) diff --git a/cmdeploy/src/cmdeploy/remote/rshell.py b/cmdeploy/src/cmdeploy/remote/rshell.py index 994223bc..ffc1ab66 100644 --- a/cmdeploy/src/cmdeploy/remote/rshell.py +++ b/cmdeploy/src/cmdeploy/remote/rshell.py @@ -1,12 +1,11 @@ -from subprocess import CalledProcessError as ShellError -from subprocess import check_output +from subprocess import CalledProcessError, check_output def shell(command, fail_ok=False): print(f"$ {command}") try: return check_output(command, shell=True).decode().rstrip() - except ShellError: + except CalledProcessError: if not fail_ok: raise return "" From 3ef45c2ffd35802bb4cc3ed963e2a64846c0be85 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 2 Sep 2024 23:02:34 +0200 Subject: [PATCH 10/66] add changelog entry for #405 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17630aff..d8bb2131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## untagged +- add a paragraph about "account deletion" to info page + ([#405](https://github.com/deltachat/chatmail/pull/405)) + - avoid nginx listening on ipv6 if v6 is dsiabled ([#402](https://github.com/deltachat/chatmail/pull/402)) From ba811c2e1c51319b8e241348efcbcd142fe91fd6 Mon Sep 17 00:00:00 2001 From: missytake Date: Fri, 13 Sep 2024 21:55:54 +0200 Subject: [PATCH 11/66] DNS: fix checking for required DNS records (#412) * Improve README for first setup * DNS: fix flushing DNS when requesting records * DNS: actually check whether mta-sts record is set correctly * DNS: add changelog * DNS: also check for www CNAME record * DNS: fix tests * lint: update ruff to 0.6.5 locally --- CHANGELOG.md | 3 +++ README.md | 9 +++++---- cmdeploy/src/cmdeploy/dns.py | 10 +++++++--- cmdeploy/src/cmdeploy/remote/rdns.py | 14 ++++++++------ cmdeploy/src/cmdeploy/tests/test_dns.py | 24 +++++++++++++++++------- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8bb2131..24c1fd9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## untagged +- fix checking for required DNS records + ([#412](https://github.com/deltachat/chatmail/pull/412)) + - add a paragraph about "account deletion" to info page ([#405](https://github.com/deltachat/chatmail/pull/405)) diff --git a/README.md b/README.md index c837761f..8fc4a3a5 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,8 @@ Please substitute it with your own domain. scripts/cmdeploy init chat.example.org # <-- use your domain ``` -3. Setup first DNS records for your chatmail domain, - according to the hints provided by `cmdeploy init`. +3. Point your domain to the server's IP address, + if you haven't done so already. Verify that SSH root login works: ``` @@ -47,7 +47,8 @@ Please substitute it with your own domain. ``` scripts/cmdeploy run ``` - This script will also show you additional DNS records + This script will check that you have all necessary DNS records. + If DNS records are missing, it will recommend which you should configure at your DNS provider (it can take some time until they are public). @@ -59,7 +60,7 @@ To check the status of your remotely running chatmail service: scripts/cmdeploy status ``` -To check whether your DNS records are correct: +To display and check all recommended DNS records: ``` scripts/cmdeploy dns diff --git a/cmdeploy/src/cmdeploy/dns.py b/cmdeploy/src/cmdeploy/dns.py index e4b95f90..c672da33 100644 --- a/cmdeploy/src/cmdeploy/dns.py +++ b/cmdeploy/src/cmdeploy/dns.py @@ -16,9 +16,12 @@ def check_initial_remote_data(remote_data, print=print): mail_domain = remote_data["mail_domain"] if not remote_data["A"] and not remote_data["AAAA"]: print(f"Missing A and/or AAAA DNS records for {mail_domain}!") - elif not remote_data["MTA_STS"]: + elif remote_data["MTA_STS"] != f"{mail_domain}.": print("Missing MTA-STS CNAME record:") - print(f"mta-sts.{mail_domain}. CNAME {mail_domain}") + print(f"mta-sts.{mail_domain}. CNAME {mail_domain}.") + elif remote_data["WWW"] != f"{mail_domain}.": + print("Missing www CNAME record:") + print(f"www.{mail_domain}. CNAME {mail_domain}.") else: return remote_data @@ -42,7 +45,8 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int: and return (exitcode, remote_data) tuple.""" required_diff, recommended_diff = sshexec.logged( - remote.rdns.check_zonefile, kwargs=dict(zonefile=zonefile) + remote.rdns.check_zonefile, + kwargs=dict(zonefile=zonefile, mail_domain=remote_data["mail_domain"]), ) if required_diff: diff --git a/cmdeploy/src/cmdeploy/remote/rdns.py b/cmdeploy/src/cmdeploy/remote/rdns.py index ea0ec3e9..3c9bdabb 100644 --- a/cmdeploy/src/cmdeploy/remote/rdns.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -18,18 +18,19 @@ def perform_initial_checks(mail_domain): """Collecting initial DNS settings.""" assert mail_domain + if not shell("dig", fail_ok=True): + shell("apt-get install -y dnsutils") + shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True) A = query_dns("A", mail_domain) AAAA = query_dns("AAAA", mail_domain) MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}") + WWW = query_dns("CNAME", f"www.{mail_domain}") - res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS) - if not MTA_STS or (not A and not AAAA): + res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW) + if not MTA_STS or not WWW or (not A and not AAAA): return res res["acme_account_url"] = shell("acmetool account-url", fail_ok=True) - if not shell("dig", fail_ok=True): - shell("apt-get install -y dnsutils") - shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True) res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim") # parse out sts-id if exists, example: "v=STSv1; id=2090123" @@ -59,8 +60,9 @@ def query_dns(typ, domain): return "" -def check_zonefile(zonefile): +def check_zonefile(zonefile, mail_domain): """Check expected zone file entries.""" + shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True) required = True required_diff = [] recommended_diff = [] diff --git a/cmdeploy/src/cmdeploy/tests/test_dns.py b/cmdeploy/src/cmdeploy/tests/test_dns.py index 71b1baa5..fd11095f 100644 --- a/cmdeploy/src/cmdeploy/tests/test_dns.py +++ b/cmdeploy/src/cmdeploy/tests/test_dns.py @@ -24,7 +24,10 @@ def mockdns(mockdns_base): { "A": {"some.domain": "1.1.1.1"}, "AAAA": {"some.domain": "fde5:cd7a:9e1c:3240:5a99:936f:cdac:53ae"}, - "CNAME": {"mta-sts.some.domain": "some.domain"}, + "CNAME": { + "mta-sts.some.domain": "some.domain.", + "www.some.domain": "some.domain.", + }, } ) return mockdns_base @@ -33,13 +36,15 @@ def mockdns(mockdns_base): class TestPerformInitialChecks: def test_perform_initial_checks_ok1(self, mockdns): remote_data = remote.rdns.perform_initial_checks("some.domain") - assert len(remote_data) == 7 + assert remote_data["A"] == mockdns["A"]["some.domain"] + assert remote_data["AAAA"] == mockdns["AAAA"]["some.domain"] + assert remote_data["MTA_STS"] == mockdns["CNAME"]["mta-sts.some.domain"] + assert remote_data["WWW"] == mockdns["CNAME"]["www.some.domain"] @pytest.mark.parametrize("drop", ["A", "AAAA"]) def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop): del mockdns[drop] remote_data = remote.rdns.perform_initial_checks("some.domain") - assert len(remote_data) == 7 assert not remote_data[drop] l = [] @@ -48,9 +53,8 @@ def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop): assert not l def test_perform_initial_checks_no_mta_sts(self, mockdns): - del mockdns["CNAME"] + del mockdns["CNAME"]["mta-sts.some.domain"] remote_data = remote.rdns.perform_initial_checks("some.domain") - assert len(remote_data) == 4 assert not remote_data["MTA_STS"] l = [] @@ -85,14 +89,18 @@ class TestZonefileChecks: def test_check_zonefile_all_ok(self, cm_data, mockdns_base): zonefile = cm_data.get("zftest.zone") parse_zonefile_into_dict(zonefile, mockdns_base) - required_diff, recommended_diff = remote.rdns.check_zonefile(zonefile) + required_diff, recommended_diff = remote.rdns.check_zonefile( + zonefile, "some.domain" + ) assert not required_diff and not recommended_diff def test_check_zonefile_recommended_not_set(self, cm_data, mockdns_base): zonefile = cm_data.get("zftest.zone") zonefile_mocked = zonefile.split("; Recommended")[0] parse_zonefile_into_dict(zonefile_mocked, mockdns_base) - required_diff, recommended_diff = remote.rdns.check_zonefile(zonefile) + required_diff, recommended_diff = remote.rdns.check_zonefile( + zonefile, "some.domain" + ) assert not required_diff assert len(recommended_diff) == 8 @@ -101,6 +109,7 @@ def test_check_zonefile_output_required_fine(self, cm_data, mockdns_base, mockou zonefile_mocked = zonefile.split("; Recommended")[0] parse_zonefile_into_dict(zonefile_mocked, mockdns_base, only_required=True) mssh = MockSSHExec() + mockdns_base["mail_domain"] = "some.domain" res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile) assert res == 0 assert "WARNING" in mockout.captured_plain[0] @@ -110,6 +119,7 @@ def test_check_zonefile_output_full(self, cm_data, mockdns_base, mockout): zonefile = cm_data.get("zftest.zone") parse_zonefile_into_dict(zonefile, mockdns_base) mssh = MockSSHExec() + mockdns_base["mail_domain"] = "some.domain" res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile) assert res == 0 assert not mockout.captured_red From a6bdbb748b2f735e391ed310bf272febd33d9005 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 13 Sep 2024 06:06:24 +0000 Subject: [PATCH 12/66] Set CAA record flags to 0 --- CHANGELOG.md | 2 ++ cmdeploy/src/cmdeploy/chatmail.zone.j2 | 2 +- cmdeploy/src/cmdeploy/tests/data/zftest.zone | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24c1fd9d..9c548fb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ - drop hispanilandia passthrough address ([#401](https://github.com/deltachat/chatmail/pull/401)) +- set CAA record flags to 0 + ## 1.4.1 2024-07-31 diff --git a/cmdeploy/src/cmdeploy/chatmail.zone.j2 b/cmdeploy/src/cmdeploy/chatmail.zone.j2 index 8844c1d7..f0ba176f 100644 --- a/cmdeploy/src/cmdeploy/chatmail.zone.j2 +++ b/cmdeploy/src/cmdeploy/chatmail.zone.j2 @@ -20,7 +20,7 @@ www.{{ mail_domain }}. CNAME {{ mail_domain }}. _dmarc.{{ mail_domain }}. TXT "v=DMARC1;p=reject;adkim=s;aspf=s" {% if acme_account_url %} -{{ mail_domain }}. CAA 128 issue "letsencrypt.org;accounturi={{ acme_account_url }}" +{{ mail_domain }}. CAA 0 issue "letsencrypt.org;accounturi={{ acme_account_url }}" {% endif %} _adsp._domainkey.{{ mail_domain }}. TXT "dkim=discardable" diff --git a/cmdeploy/src/cmdeploy/tests/data/zftest.zone b/cmdeploy/src/cmdeploy/tests/data/zftest.zone index 5be19373..4934c2b4 100644 --- a/cmdeploy/src/cmdeploy/tests/data/zftest.zone +++ b/cmdeploy/src/cmdeploy/tests/data/zftest.zone @@ -11,7 +11,7 @@ _submission._tcp.zftest.testrun.org. SRV 0 1 587 zftest.testrun.org. _submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org. _imap._tcp.zftest.testrun.org. SRV 0 1 143 zftest.testrun.org. _imaps._tcp.zftest.testrun.org. SRV 0 1 993 zftest.testrun.org. -zftest.testrun.org. CAA 128 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956" +zftest.testrun.org. CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956" zftest.testrun.org. TXT "v=spf1 a:zftest.testrun.org ~all" _dmarc.zftest.testrun.org. TXT "v=DMARC1;p=reject;adkim=s;aspf=s" _adsp._domainkey.zftest.testrun.org. TXT "dkim=discardable" From d0ed8830f7b886c89efd3921721078db0b46bb4a Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 22 Sep 2024 16:08:44 +0000 Subject: [PATCH 13/66] Add IMAP capabilities instead of overwriting them I wanted to add `COMPRESS=DEFLATE`, but it should be added only for sessions that are logged in because `COMPRESS` command does not work before logging in. Dovecot already does it correctly if we don't overwrite the capability string. --- CHANGELOG.md | 3 +++ cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 | 5 +---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c548fb5..551c29e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ - set CAA record flags to 0 +- add IMAP capabilities instead of overwriting them + ([#413](https://github.com/deltachat/chatmail/pull/413)) + ## 1.4.1 2024-07-31 diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index c3ec5c2a..87e3dea6 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -51,10 +51,7 @@ mail_server_comment = Chatmail server # mail_plugins = zlib quota -# these are the capabilities Delta Chat cares about actually -# so let's keep the network overhead per login small -# https://github.com/deltachat/deltachat-core-rust/blob/master/src/imap/capabilities.rs -imap_capability = IMAP4rev1 IDLE MOVE QUOTA CONDSTORE NOTIFY METADATA XDELTAPUSH XCHATMAIL +imap_capability = +XDELTAPUSH XCHATMAIL # Authentication for system users. From 5515607b63f54434d9a947bce7d361889569aeb7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 14 Oct 2024 09:18:35 +0000 Subject: [PATCH 14/66] Setup mtail (#388) Co-authored-by: holger krekel --- CHANGELOG.md | 10 +++ chatmaild/src/chatmaild/config.py | 1 + chatmaild/src/chatmaild/filtermail.py | 24 +++++-- chatmaild/src/chatmaild/ini/chatmail.ini.f | 16 +++++ cmdeploy/src/cmdeploy/__init__.py | 40 ++++++++++++ .../src/cmdeploy/mtail/delivered_mail.mtail | 64 +++++++++++++++++++ cmdeploy/src/cmdeploy/mtail/mtail.service.j2 | 10 +++ 7 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail create mode 100644 cmdeploy/src/cmdeploy/mtail/mtail.service.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 551c29e9..09ea6c83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## untagged +- add mtail support (new optional `mail_address` ini value) + This defines the address on which [`mtail`](https://google.github.io/mtail/) + exposes its metrics collected from the logs. + If you want to collect the metrics with Prometheus, + setup a private network (e.g. WireGuard interface) + and assign an IP address from this network to the host. + If you do not plan to collect metrics, + keep this setting unset. + ([#388](https://github.com/deltachat/chatmail/pull/388)) + - fix checking for required DNS records ([#412](https://github.com/deltachat/chatmail/pull/412)) diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index f8109520..53c83f94 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -30,6 +30,7 @@ def __init__(self, inipath, params): self.passthrough_recipients = params["passthrough_recipients"].split() self.filtermail_smtp_port = int(params["filtermail_smtp_port"]) self.postfix_reinject_port = int(params["postfix_reinject_port"]) + self.mtail_address = params.get("mtail_address") self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true" self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true" self.iroh_relay = params.get("iroh_relay") diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 140e7172..fb149446 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -183,15 +183,29 @@ def check_DATA(self, envelope): mail_encrypted = check_encrypted(message) _, from_addr = parseaddr(message.get("from").strip()) + envelope_from_domain = from_addr.split("@").pop() + logging.info(f"mime-from: {from_addr} envelope-from: {envelope.mail_from!r}") if envelope.mail_from.lower() != from_addr.lower(): return f"500 Invalid FROM <{from_addr!r}> for <{envelope.mail_from!r}>" + if mail_encrypted: + print("Filtering encrypted mail.", file=sys.stderr) + else: + print("Filtering unencrypted mail.", file=sys.stderr) + if envelope.mail_from in self.config.passthrough_senders: return passthrough_recipients = self.config.passthrough_recipients - envelope_from_domain = from_addr.split("@").pop() + + is_securejoin = message.get("secure-join") in [ + "vc-request", + "vg-request", + ] + if is_securejoin: + return + for recipient in envelope.rcpt_tos: if envelope.mail_from == recipient: # Always allow sending emails to self. @@ -205,12 +219,8 @@ def check_DATA(self, envelope): is_outgoing = recipient_domain != envelope_from_domain if is_outgoing and not mail_encrypted: - is_securejoin = message.get("secure-join") in [ - "vc-request", - "vg-request", - ] - if not is_securejoin: - return f"500 Invalid unencrypted mail to <{recipient}>" + print("Rejected unencrypted mail.", file=sys.stderr) + return f"500 Invalid unencrypted mail to <{recipient}>" class SendRateLimiter: diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index b2a5ff12..e8e8ef5d 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -55,6 +55,22 @@ # if set to "True" IPv6 is disabled disable_ipv6 = False +# Address on which `mtail` listens, +# e.g. 127.0.0.1 or some private network +# address like 192.168.10.1. +# You can point Prometheus +# or some other OpenMetrics-compatible +# collector to +# http://{{mtail_address}}:3903/metrics +# and display collected metrics with Grafana. +# +# WARNING: do not expose this service +# to the public IP address. +# +# `mtail is not running if the setting is not set. + +# mtail_address = 127.0.0.1 + # # Debugging options # diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index ace9e47b..226d17d2 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -441,6 +441,44 @@ def check_config(config): return config +def deploy_mtail(config): + apt.packages( + name="Install mtail", + packages=["mtail"], + ) + + # Using our own systemd unit instead of `/usr/lib/systemd/system/mtail.service`. + # This allows to read from journalctl instead of log files. + files.template( + src=importlib.resources.files(__package__).joinpath("mtail/mtail.service.j2"), + dest="/etc/systemd/system/mtail.service", + user="root", + group="root", + mode="644", + address=config.mtail_address or "127.0.0.1", + port=3903, + ) + + mtail_conf = files.put( + name="Mtail configuration", + src=importlib.resources.files(__package__).joinpath( + "mtail/delivered_mail.mtail" + ), + dest="/etc/mtail/delivered_mail.mtail", + user="root", + group="root", + mode="644", + ) + + systemd.service( + name="Start and enable mtail", + service="mtail.service", + running=bool(config.mtail_address), + enabled=bool(config.mtail_address), + restarted=mtail_conf.changed, + ) + + def deploy_chatmail(config_path: Path) -> None: """Deploy a chat-mail instance. @@ -636,3 +674,5 @@ def deploy_chatmail(config_path: Path) -> None: name="Ensure cron is installed", packages=["cron"], ) + + deploy_mtail(config) diff --git a/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail b/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail new file mode 100644 index 00000000..dc55e340 --- /dev/null +++ b/cmdeploy/src/cmdeploy/mtail/delivered_mail.mtail @@ -0,0 +1,64 @@ +counter delivered_mail +/saved mail to INBOX$/ { + delivered_mail++ +} + +counter quota_exceeded +/Quota exceeded \(mailbox for user is full\)$/ { + quota_exceeded++ +} + +# Essentially the number of outgoing messages. +counter dkim_signed +/DKIM-Signature field added/ { + dkim_signed++ +} + +counter created_accounts +counter created_ci_accounts +counter created_nonci_accounts + +/: Created address: (?P.*)$/ { + created_accounts++ + + $addr =~ /ci-/ { + created_ci_accounts++ + } else { + created_nonci_accounts++ + } +} + +counter postfix_timeouts +/timeout after DATA/ { + postfix_timeouts++ +} + +counter postfix_noqueue +/postfix\/.*NOQUEUE/ { + postfix_noqueue++ +} + +counter warning_count +/warning/ { + warning_count++ +} + + +counter filtered_mail_count + +counter encrypted_mail_count +/Filtering encrypted mail\./ { + encrypted_mail_count++ + filtered_mail_count++ +} + +counter unencrypted_mail_count +/Filtering unencrypted mail\./ { + unencrypted_mail_count++ + filtered_mail_count++ +} + +counter rejected_unencrypted_mail_count +/Rejected unencrypted mail\./ { + rejected_unencrypted_mail_count++ +} diff --git a/cmdeploy/src/cmdeploy/mtail/mtail.service.j2 b/cmdeploy/src/cmdeploy/mtail/mtail.service.j2 new file mode 100644 index 00000000..97d209d1 --- /dev/null +++ b/cmdeploy/src/cmdeploy/mtail/mtail.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=mtail + +[Service] +Type=simple +ExecStart=/bin/sh -c "journalctl -f -o short-iso -n 0 | /usr/bin/mtail --address={{ address }} --port={{ port }} --progs /etc/mtail --logtostderr --logs -" +Restart=on-failure + +[Install] +WantedBy=multi-user.target From 46297d48391dec1b9a109894b6025e57a8ee477a Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 14 Oct 2024 06:40:47 +0000 Subject: [PATCH 15/66] Document setting up DNAT --- README.md | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/README.md b/README.md index 8fc4a3a5..4c620d5f 100644 --- a/README.md +++ b/README.md @@ -187,3 +187,115 @@ to MAIL FROM with and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch`](reject_sender_login_mismatch) policy. `From:` header must correspond to envelope MAIL FROM, this is ensured by `filtermail` proxy. + +## Setting up a reverse proxy + +A chatmail server does not depend on the client IP address +for its operation, so it can be run behind a reverse proxy. +This will not even affect incoming mail authentication +as DKIM only checks the cryptographic signature +of the message and does not use the IP address as the input. + +For example, you may want to self-host your chatmail server +and only use hosted VPS to provide a public IP address +for client connections and incoming mail. +You can also setup multiple reverse proxies +for your chatmail server in different networks +to ensure your server is reachable even when +one of the IPs becomes inaccessible due to +hosting or routing problems. + +Note that your server still needs +to be able to make outgoing connections on port 25 +to send messages outside. + +To setup a reverse proxy +(or rather Destination NAT, DNAT) +for your chatmail server, +put the following configuration in `/etc/nftables.conf`: +``` +#!/usr/sbin/nft -f + +flush ruleset + +define wan = eth0 + +# Which ports to proxy. +# +# Note that SSH is not proxied +# so it is possible to log into the proxy server +# and not the original one. +define ports = { smtp, http, https, imap, imaps, submission, submissions } + +# The host we want to proxy to. +define ipv4_address = AAA.BBB.CCC.DDD +define ipv6_address = [XXX::1] + +table ip nat { + chain prerouting { + type nat hook prerouting priority dstnat; policy accept; + iif $wan tcp dport $ports dnat to $ipv4_address + } + + chain postrouting { + type nat hook postrouting priority 0; + + oifname $wan masquerade + } +} + +table ip6 nat { + chain prerouting { + type nat hook prerouting priority dstnat; policy accept; + iif $wan tcp dport $ports dnat to $ipv6_address + } + + chain postrouting { + type nat hook postrouting priority 0; + + oifname $wan masquerade + } +} + +table inet filter { + chain input { + type filter hook input priority filter; policy drop; + + # Accept ICMP. + # It is especially important to accept ICMPv6 ND messages, + # otherwise IPv6 connectivity breaks. + icmp type { echo-request } accept + icmpv6 type { echo-request, nd-neighbor-solicit, nd-router-advert, nd-neighbor-advert } accept + + # Allow incoming SSH connections. + tcp dport { ssh } accept + + ct state established accept + } + chain forward { + type filter hook forward priority filter; policy drop; + + ct state established accept + ip daddr $ipv4_address counter accept + ip6 daddr $ipv6_address counter accept + } + chain output { + type filter hook output priority filter; + } +} +``` + +Run `systemctl enable nftables.service` +to ensure configuration is reloaded when the proxy server reboots. + +Uncomment in `/etc/sysctl.conf` the following two lines: + +``` +net.ipv4.ip_forward=1 +net.ipv6.conf.all.forwarding=1 +``` + +Then reboot the server or do `sysctl -p` and `nft -f /etc/nftables.conf`. + +Once proxy server is set up, +you can add its IP address to the DNS. From 7573ef928f0a5d874d500d504fb1f2840db9a4d6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 14 Oct 2024 10:11:17 +0000 Subject: [PATCH 16/66] mention wireguard --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 4c620d5f..e841fab6 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,12 @@ of the message and does not use the IP address as the input. For example, you may want to self-host your chatmail server and only use hosted VPS to provide a public IP address for client connections and incoming mail. +You can connect chatmail server to VPS +using a tunnel protocol +such as [WireGuard](https://www.wireguard.com/) +and setup a reverse proxy on a VPS +to forward connections to the chatmail server +over the tunnel. You can also setup multiple reverse proxies for your chatmail server in different networks to ensure your server is reachable even when From a2f2e04ff9f64b4d5861d3260875a2859550e106 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 15 Oct 2024 14:36:41 +0000 Subject: [PATCH 17/66] fix: set acme_account_url even if some DNS records are not set perform_initial_checks may exit early and not add `acme_account_url` if required DNS records are not found. In this case other `cmdeploy run` fails with KeyError. To avoid this, `acme_account_url` should always be set. Unlike DNS checks, running acmetool may not fail due to network errors, so it is more reliable and should be checked first. --- cmdeploy/src/cmdeploy/remote/rdns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmdeploy/src/cmdeploy/remote/rdns.py b/cmdeploy/src/cmdeploy/remote/rdns.py index 3c9bdabb..d378990a 100644 --- a/cmdeploy/src/cmdeploy/remote/rdns.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -27,12 +27,12 @@ def perform_initial_checks(mail_domain): WWW = query_dns("CNAME", f"www.{mail_domain}") res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW) - if not MTA_STS or not WWW or (not A and not AAAA): - return res - res["acme_account_url"] = shell("acmetool account-url", fail_ok=True) res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim") + if not MTA_STS or not WWW or (not A and not AAAA): + return res + # parse out sts-id if exists, example: "v=STSv1; id=2090123" parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=") res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else "" From 20fa5d9656889433be0c2f4e974565629d533ca6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 15 Oct 2024 21:47:13 +0000 Subject: [PATCH 18/66] Query autoritative nameserver directly to bypass DNS cache unbound-control is not installed out of the box and even once installed `flush_zone` does not seem to work reliably. Instead of trying to flush the cache from unbound, we now query authoritative nameserver directly using `dig`. --- CHANGELOG.md | 3 +++ cmdeploy/src/cmdeploy/remote/rdns.py | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09ea6c83..032605a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## untagged +- query autoritative nameserver to bypass DNS cache + ([#424](https://github.com/deltachat/chatmail/pull/424)) + - add mtail support (new optional `mail_address` ini value) This defines the address on which [`mtail`](https://google.github.io/mtail/) exposes its metrics collected from the logs. diff --git a/cmdeploy/src/cmdeploy/remote/rdns.py b/cmdeploy/src/cmdeploy/remote/rdns.py index d378990a..77093503 100644 --- a/cmdeploy/src/cmdeploy/remote/rdns.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -20,7 +20,6 @@ def perform_initial_checks(mail_domain): assert mail_domain if not shell("dig", fail_ok=True): shell("apt-get install -y dnsutils") - shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True) A = query_dns("A", mail_domain) AAAA = query_dns("AAAA", mail_domain) MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}") @@ -53,8 +52,20 @@ def get_dkim_entry(mail_domain, dkim_selector): def query_dns(typ, domain): - res = shell(f"dig -r -q {domain} -t {typ} +short") - print(res) + # Get autoritative nameserver from the SOA record. + soa_answers = [ + x.split() + for x in shell(f"dig -r -q {domain} -t SOA +noall +authority +answer").split( + "\n" + ) + ] + soa = [a for a in soa_answers if len(a) >= 3 and a[3] == "SOA"] + if not soa: + return + ns = soa[0][4] + + # Query authoritative nameserver directly to bypass DNS cache. + res = shell(f"dig @{ns} -r -q {domain} -t {typ} +short") if res: return res.split("\n")[0] return "" @@ -62,7 +73,6 @@ def query_dns(typ, domain): def check_zonefile(zonefile, mail_domain): """Check expected zone file entries.""" - shell(f"unbound-control flush_zone {mail_domain}", fail_ok=True) required = True required_diff = [] recommended_diff = [] From 737ab54bf21fc145594cb6b79f2affea1cbd2642 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 15 Oct 2024 23:08:43 +0000 Subject: [PATCH 19/66] ci: test `cmdeploy dns` only once It should be reliable. --- .github/workflows/test-and-deploy-ipv4only.yaml | 4 ++-- .github/workflows/test-and-deploy.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-and-deploy-ipv4only.yaml b/.github/workflows/test-and-deploy-ipv4only.yaml index e2866a25..fe1046b7 100644 --- a/.github/workflows/test-and-deploy-ipv4only.yaml +++ b/.github/workflows/test-and-deploy-ipv4only.yaml @@ -93,6 +93,6 @@ jobs: - name: cmdeploy test run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow - - name: cmdeploy dns (try 3 times) - run: cmdeploy dns || cmdeploy dns || cmdeploy dns + - name: cmdeploy dns + run: cmdeploy dns -v diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index 72a757ca..686f77d0 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -91,6 +91,6 @@ jobs: - name: cmdeploy test run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow - - name: cmdeploy dns (try 3 times) - run: cmdeploy dns -v || cmdeploy dns -v || cmdeploy dns -v + - name: cmdeploy dns + run: cmdeploy dns -v From 15f30d88411256084cf24b5aa1d3ebe630467924 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 16 Oct 2024 11:21:09 +0200 Subject: [PATCH 20/66] cmdeploy: flag to disable postfix + dovecot for migration --- cmdeploy/src/cmdeploy/__init__.py | 19 ++++++++++--------- cmdeploy/src/cmdeploy/cmdeploy.py | 7 +++++++ cmdeploy/src/cmdeploy/deploy.py | 3 ++- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index 226d17d2..283e268b 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -479,10 +479,11 @@ def deploy_mtail(config): ) -def deploy_chatmail(config_path: Path) -> None: +def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: """Deploy a chat-mail instance. :param config_path: path to chatmail.ini + :param disable_mail: whether to disable postfix & dovecot """ config = read_config(config_path) check_config(config) @@ -624,19 +625,19 @@ def deploy_chatmail(config_path: Path) -> None: # because it creates authentication socket # required by Postfix. systemd.service( - name="Start and enable Dovecot", + name="disable dovecot for now" if disable_mail else "Start and enable Dovecot", service="dovecot.service", - running=True, - enabled=True, - restarted=dovecot_need_restart, + running=False if disable_mail else True, + enabled=False if disable_mail else True, + restarted=dovecot_need_restart if not disable_mail else False, ) systemd.service( - name="Start and enable Postfix", + name="disable postfix for now" if disable_mail else "Start and enable Postfix", service="postfix.service", - running=True, - enabled=True, - restarted=postfix_need_restart, + running=False if disable_mail else True, + enabled=False if disable_mail else True, + restarted=postfix_need_restart if not disable_mail else False, ) systemd.service( diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index cd992a2b..cb744b37 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -52,6 +52,12 @@ def run_cmd_options(parser): action="store_true", help="don't actually modify the server", ) + parser.add_argument( + "--disable-mail", + dest="disable_mail", + action="store_true", + help="install/upgrade the server, but disable postfix & dovecot for now" + ) def run_cmd(args, out): @@ -64,6 +70,7 @@ def run_cmd(args, out): env = os.environ.copy() env["CHATMAIL_INI"] = args.inipath + env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else "" deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve() pyinf = "pyinfra --dry" if args.dry_run else "pyinfra" cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path} -y" diff --git a/cmdeploy/src/cmdeploy/deploy.py b/cmdeploy/src/cmdeploy/deploy.py index ef259ad0..0ea153d7 100644 --- a/cmdeploy/src/cmdeploy/deploy.py +++ b/cmdeploy/src/cmdeploy/deploy.py @@ -11,8 +11,9 @@ def main(): "CHATMAIL_INI", importlib.resources.files("cmdeploy").joinpath("../../../chatmail.ini"), ) + disable_mail = bool(os.environ.get('CHATMAIL_DISABLE_MAIL')) - deploy_chatmail(config_path) + deploy_chatmail(config_path, disable_mail) if pyinfra.is_cli: From babdff361cdf529fc4beebf3a3278a27c61b9bc7 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 15 Oct 2024 14:54:51 +0200 Subject: [PATCH 21/66] docs: more details for the repo overview #419 --- README.md | 80 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index e841fab6..35261f57 100644 --- a/README.md +++ b/README.md @@ -80,31 +80,81 @@ scripts/cmdeploy bench ## Overview of this repository -This repository drives the development of chatmail services, -comprised of minimal setups of +This repository has four directories: + +- [cmdeploy](https://github.com/deltachat/chatmail/tree/main/cmdeploy) + is a collection of configuration files + and a [pyinfra](https://pyinfra.com)-based deployment script. +- [chatmaild](https://github.com/deltachat/chatmail/tree/main/chatmaild) + is a python package containing several small services + which handle authentication, + trigger push notifications on new messages, + ensure that outbound mails are encrypted, + delete inactive users, + and some other minor things. + chatmaild can also be installed as a stand-alone python package. +- [www](https://github.com/deltachat/chatmail/tree/main/www) + contains the html, css, and markdown files + which make up a chatmail server's web page. + Edit them before deploying to make your chatmail server stand out. +- [scripts](https://github.com/deltachat/chatmail/tree/main/scripts) + offers two convenience tools for beginners; + `initenv.sh` installs the necessary dependencies to a local virtual environment, + and the `scripts/cmdeploy` script enables you + to run the `cmdeploy` command line tool in the local virtual environment. + +### cmdeploy + +The `cmdeploy/src/cmdeploy/cmdeploy.py` command line tool +helps with setting up and managing the chatmail service. +`cmdeploy run` uses [pyinfra-based scripting](https://pyinfra.com/) +in `cmdeploy/src/cmdeploy/__init__.py` +to automatically install all chatmail components on a server: -- [postfix smtp server](https://www.postfix.org) -- [dovecot imap server](https://www.dovecot.org) +- [postfix smtp server](https://www.postfix.org) accepts sent messages (both from your users and from other servers) +- [dovecot imap server](https://www.dovecot.org) stores messages for your users until they download them +- [nginx](https://nginx.org/) shows the web page with your privacy policy and additional information +- [acmetool](https://hlandau.github.io/acmetool/) manages TLS certificates for dovecot, postfix, and nginx +- [opendkim](http://www.opendkim.org/) for signing messages with DKIM and rejecting inbound messages without DKIM +- [mtail](https://google.github.io/mtail/) for collecting anonymized metrics in case you have monitoring +- and the chatmaild services, explained in the next section: -as well as custom services that are integrated with these two: +### chatmaild -- `chatmaild/src/chatmaild/doveauth.py` implements +chatmaild offers several commands +which differentiate a *chatmail* server from a classic mail server. +If you deploy them with cmdeploy, +they are run by systemd services in the background. +A short overview: + +- [`doveauth`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/doveauth.py) implements create-on-login account creation semantics and is used by Dovecot during login authentication and by Postfix which in turn uses [Dovecot SASL](https://doc.dovecot.org/configuration_manual/authentication/dict/#complete-example-for-authenticating-via-a-unix-socket) to authenticate users to send mails for them. - -- `chatmaild/src/chatmaild/filtermail.py` prevents +- [`filtermail`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/filtermail.py) prevents unencrypted e-mail from leaving the chatmail service and is integrated into postfix's outbound mail pipelines. - -There is also the `cmdeploy/src/cmdeploy/cmdeploy.py` command line tool -which helps with setting up and managing the chatmail service. -`cmdeploy run` uses [pyinfra-based scripting](https://pyinfra.com/) -in `cmdeploy/src/cmdeploy/__init__.py` -to automatically install all chatmail components on a server. - +- [`chatmail-metadata`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/metadata.py) is contacted by a + [dovecot lua script](https://github.com/deltachat/chatmail/blob/main/cmdeploy/src/cmdeploy/dovecot/push_notification.lua) + to store user-specific server-side config. + On new messages, + it [passes the user's push notification token](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/notifier.py) + to [notifications.delta.chat](https://delta.chat/help#instant-delivery) + so the push notifications on the user's phone can be triggered + by Apple/Google. +- [`delete_inactive_users`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/delete_inactive_users.py) + deletes users if they have not logged in for a very long time. + The timeframe can be configured in `chatmail.ini`. +- [`lastlogin`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/lastlogin.py) + is contacted by dovecot when a user logs in + and stores the date of the login. +- [`echobot`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/echo.py) + is a small bot for test purposes. + It simply echoes back messages from users. +- [`chatmail-metrics`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/metrics.py) + collects some metrics and displays them at `https://example.org/metrics`. ### Home page and getting started for users From 80cbdda772dff46d5e3e1539f6ebedb239fcecd7 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 15 Oct 2024 15:02:14 +0200 Subject: [PATCH 22/66] docs: mention the chatmail.ini in the cmdeploy description --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 35261f57..cee68936 100644 --- a/README.md +++ b/README.md @@ -107,9 +107,12 @@ This repository has four directories: The `cmdeploy/src/cmdeploy/cmdeploy.py` command line tool helps with setting up and managing the chatmail service. -`cmdeploy run` uses [pyinfra-based scripting](https://pyinfra.com/) -in `cmdeploy/src/cmdeploy/__init__.py` -to automatically install all chatmail components on a server: +`cmdeploy init` creates the `chatmail.ini` config file. +`cmdeploy run` uses a [pyinfra](https://pyinfra.com/)-based [script](`cmdeploy/src/cmdeploy/__init__.py`) +to automatically install or upgrade all chatmail components on a server, +according to the `chatmail.ini` config. + +The components of chatmail are: - [postfix smtp server](https://www.postfix.org) accepts sent messages (both from your users and from other servers) - [dovecot imap server](https://www.dovecot.org) stores messages for your users until they download them From bbf508d95e7c8bf6f711b133b6c0e4dcd63623e7 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 16 Oct 2024 16:44:47 +0200 Subject: [PATCH 23/66] docs: nicer linebreaks --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index cee68936..c9cc13a7 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ This repository has four directories: - [cmdeploy](https://github.com/deltachat/chatmail/tree/main/cmdeploy) is a collection of configuration files and a [pyinfra](https://pyinfra.com)-based deployment script. + - [chatmaild](https://github.com/deltachat/chatmail/tree/main/chatmaild) is a python package containing several small services which handle authentication, @@ -93,10 +94,12 @@ This repository has four directories: delete inactive users, and some other minor things. chatmaild can also be installed as a stand-alone python package. + - [www](https://github.com/deltachat/chatmail/tree/main/www) contains the html, css, and markdown files which make up a chatmail server's web page. Edit them before deploying to make your chatmail server stand out. + - [scripts](https://github.com/deltachat/chatmail/tree/main/scripts) offers two convenience tools for beginners; `initenv.sh` installs the necessary dependencies to a local virtual environment, @@ -115,11 +118,17 @@ according to the `chatmail.ini` config. The components of chatmail are: - [postfix smtp server](https://www.postfix.org) accepts sent messages (both from your users and from other servers) + - [dovecot imap server](https://www.dovecot.org) stores messages for your users until they download them + - [nginx](https://nginx.org/) shows the web page with your privacy policy and additional information + - [acmetool](https://hlandau.github.io/acmetool/) manages TLS certificates for dovecot, postfix, and nginx + - [opendkim](http://www.opendkim.org/) for signing messages with DKIM and rejecting inbound messages without DKIM + - [mtail](https://google.github.io/mtail/) for collecting anonymized metrics in case you have monitoring + - and the chatmaild services, explained in the next section: ### chatmaild @@ -136,9 +145,11 @@ A short overview: which in turn uses [Dovecot SASL](https://doc.dovecot.org/configuration_manual/authentication/dict/#complete-example-for-authenticating-via-a-unix-socket) to authenticate users to send mails for them. + - [`filtermail`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/filtermail.py) prevents unencrypted e-mail from leaving the chatmail service and is integrated into postfix's outbound mail pipelines. + - [`chatmail-metadata`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/metadata.py) is contacted by a [dovecot lua script](https://github.com/deltachat/chatmail/blob/main/cmdeploy/src/cmdeploy/dovecot/push_notification.lua) to store user-specific server-side config. @@ -147,15 +158,19 @@ A short overview: to [notifications.delta.chat](https://delta.chat/help#instant-delivery) so the push notifications on the user's phone can be triggered by Apple/Google. + - [`delete_inactive_users`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/delete_inactive_users.py) deletes users if they have not logged in for a very long time. The timeframe can be configured in `chatmail.ini`. + - [`lastlogin`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/lastlogin.py) is contacted by dovecot when a user logs in and stores the date of the login. + - [`echobot`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/echo.py) is a small bot for test purposes. It simply echoes back messages from users. + - [`chatmail-metrics`](https://github.com/deltachat/chatmail/blob/main/chatmaild/src/chatmaild/metrics.py) collects some metrics and displays them at `https://example.org/metrics`. From 5055434e48b22ed0a6763709413f70ec44492793 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 22 Oct 2024 18:00:34 +0000 Subject: [PATCH 24/66] Fix OpenPGP payload check Replace \r\r\n in literal.eml test with \r\n to make `test_filtermail_no_literal_packets` actually reach `check_openpgp_payload()` and make `check_openpgp_payload()` more strict. --- CHANGELOG.md | 3 + chatmaild/src/chatmaild/filtermail.py | 17 ++-- .../src/chatmaild/tests/mail-data/literal.eml | 86 +++++++++---------- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 032605a4..5a98ef43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,9 @@ - add IMAP capabilities instead of overwriting them ([#413](https://github.com/deltachat/chatmail/pull/413)) +- fix OpenPGP payload check + ([#435](https://github.com/deltachat/chatmail/pull/435)) + ## 1.4.1 2024-07-31 diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index fb149446..6e0dbf53 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -60,10 +60,11 @@ def check_openpgp_payload(payload: bytes): i += body_len if i == len(payload): - if packet_type_id == 18: - # Last packet should be - # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD) - return True + # Last packet should be + # Symmetrically Encrypted and Integrity Protected Data Packet (SEIPD) + # + # This is the only place where this function may return `True`. + return packet_type_id == 18 elif packet_type_id not in [1, 3]: # All packets except the last one must be either # Public-Key Encrypted Session Key Packet (PKESK) @@ -71,13 +72,7 @@ def check_openpgp_payload(payload: bytes): # Symmetric-Key Encrypted Session Key Packet (SKESK) return False - if i == 0: - return False - - if i > len(payload): - # Payload is truncated. - return False - return True + return False def check_armored_payload(payload: str): diff --git a/chatmaild/src/chatmaild/tests/mail-data/literal.eml b/chatmaild/src/chatmaild/tests/mail-data/literal.eml index b9ab6d2c..9b00947a 100644 --- a/chatmaild/src/chatmaild/tests/mail-data/literal.eml +++ b/chatmaild/src/chatmaild/tests/mail-data/literal.eml @@ -1,44 +1,44 @@ -From: {from_addr} -To: {to_addr} -Subject: ... -Date: Sun, 15 Oct 2023 16:43:21 +0000 -Message-ID: -In-Reply-To: -References: - -Chat-Version: 1.0 -Autocrypt: addr={from_addr}; prefer-encrypt=mutual; - keydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH - rNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID - FgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW - XQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK - KwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ - JlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP - IXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO -MIME-Version: 1.0 -Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; - boundary="YFrteb74qSXmggbOxZL9dRnhymywAi" - - ---YFrteb74qSXmggbOxZL9dRnhymywAi -Content-Description: PGP/MIME version identification -Content-Type: application/pgp-encrypted - -Version: 1 - - ---YFrteb74qSXmggbOxZL9dRnhymywAi -Content-Description: OpenPGP encrypted message -Content-Disposition: inline; filename="encrypted.asc"; -Content-Type: application/octet-stream; name="encrypted.asc" - ------BEGIN PGP MESSAGE----- - -yxJiAAAAAABIZWxsbyB3b3JsZCE= -=1I/B ------END PGP MESSAGE----- - - ---YFrteb74qSXmggbOxZL9dRnhymywAi-- - +From: {from_addr} +To: {to_addr} +Subject: ... +Date: Sun, 15 Oct 2023 16:43:21 +0000 +Message-ID: +In-Reply-To: +References: + +Chat-Version: 1.0 +Autocrypt: addr={from_addr}; prefer-encrypt=mutual; + keydata=xjMEZSwWjhYJKwYBBAHaRw8BAQdAQBEhqeJh0GueHB6kF/DUQqYCxARNBVokg/AzT+7LqH + rNFzxiYXJiYXpAYzIudGVzdHJ1bi5vcmc+wosEEBYIADMCGQEFAmUsFo4CGwMECwkIBwYVCAkKCwID + FgIBFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX9A4AEAnHWHp49eBCMHK5t66gYPiW + XQuB1mwUjzGfYWB+0RXUoA/0xcQ3FbUNlGKW7Blp6eMFfViv6Mv2d3kNSXACB6nmcMzjgEZSwWjhIK + KwYBBAGXVQEFAQEHQBpY5L2M1XHo0uxf8SX1wNLBp/OVvidoWHQF2Jz+kJsUAwEIB8J4BBgWCAAgBQ + JlLBaOAhsMFiEEFTfUNvVnY3b9F7yHnmme1PfUhX8ACgkQnmme1PfUhX/INgEA37AJaNvruYsJVanP + IXnYw4CKd55UAwl8Zcy+M2diAbkA/0fHHcGV4r78hpbbL1Os52DPOdqYQRauIeJUeG+G6bQO +MIME-Version: 1.0 +Content-Type: multipart/encrypted; protocol="application/pgp-encrypted"; + boundary="YFrteb74qSXmggbOxZL9dRnhymywAi" + + +--YFrteb74qSXmggbOxZL9dRnhymywAi +Content-Description: PGP/MIME version identification +Content-Type: application/pgp-encrypted + +Version: 1 + + +--YFrteb74qSXmggbOxZL9dRnhymywAi +Content-Description: OpenPGP encrypted message +Content-Disposition: inline; filename="encrypted.asc"; +Content-Type: application/octet-stream; name="encrypted.asc" + +-----BEGIN PGP MESSAGE----- + +yxJiAAAAAABIZWxsbyB3b3JsZCE= +=1I/B +-----END PGP MESSAGE----- + + +--YFrteb74qSXmggbOxZL9dRnhymywAi-- + From 48fdff670071696d3db066c0305973531e921232 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 16 Oct 2024 20:52:09 +0200 Subject: [PATCH 25/66] fix wrong ref in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a98ef43..f514f513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - query autoritative nameserver to bypass DNS cache ([#424](https://github.com/deltachat/chatmail/pull/424)) -- add mtail support (new optional `mail_address` ini value) +- add mtail support (new optional `mtail_address` ini value) This defines the address on which [`mtail`](https://google.github.io/mtail/) exposes its metrics collected from the logs. If you want to collect the metrics with Prometheus, From 8fe173439d2251d02cc61dca856e3f5679b6e112 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 26 Oct 2024 17:22:20 -0400 Subject: [PATCH 26/66] Dovecot quota_max_mail_size to use the Chatmail max_message_size value --- CHANGELOG.md | 2 ++ cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f514f513..760b9b2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ - fix OpenPGP payload check ([#435](https://github.com/deltachat/chatmail/pull/435)) +- fix Dovecot quota_max_mail_size to use max_message_size config value + ## 1.4.1 2024-07-31 diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index 87e3dea6..f4b68f9e 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -141,7 +141,7 @@ plugin { # for now we define static quota-rules for all users quota = maildir:User quota quota_rule = *:storage={{ config.max_mailbox_size }} - quota_max_mail_size=30M + quota_max_mail_size={{ config.max_message_size }} quota_grace = 0 # quota_over_flag_value = TRUE } From 3e646efee95d673d03dbbe600b47ec015ef00624 Mon Sep 17 00:00:00 2001 From: missytake Date: Sun, 27 Oct 2024 12:16:09 +0100 Subject: [PATCH 27/66] add PR link to CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 760b9b2e..d2aada52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ ([#435](https://github.com/deltachat/chatmail/pull/435)) - fix Dovecot quota_max_mail_size to use max_message_size config value + ([#438](https://github.com/deltachat/chatmail/pull/438)) ## 1.4.1 2024-07-31 From 7f3f69fa7224d740cfecd0e7f5255a2c0cc94136 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 26 Oct 2024 20:23:20 +0000 Subject: [PATCH 28/66] fix: increase `request_queue_size` for UNIX sockets to 1000 Default value is 5. This setting was lost during refactoring in commit bf0f6e230352b9a05fa5b2e4ce59e511731f412e --- CHANGELOG.md | 3 +++ chatmaild/src/chatmaild/dictproxy.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2aada52..071aa202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## untagged +- increase `request_queue_size` for UNIX sockets to 1000. + ([#437](https://github.com/deltachat/chatmail/pull/437)) + - query autoritative nameserver to bypass DNS cache ([#424](https://github.com/deltachat/chatmail/pull/424)) diff --git a/chatmaild/src/chatmaild/dictproxy.py b/chatmaild/src/chatmaild/dictproxy.py index 56f91c23..a0a7b654 100644 --- a/chatmaild/src/chatmaild/dictproxy.py +++ b/chatmaild/src/chatmaild/dictproxy.py @@ -87,8 +87,12 @@ def handle(self): except FileNotFoundError: pass - with ThreadingUnixStreamServer(socket, Handler) as server: + with CustomThreadingUnixStreamServer(socket, Handler) as server: try: server.serve_forever() except KeyboardInterrupt: pass + + +class CustomThreadingUnixStreamServer(ThreadingUnixStreamServer): + request_queue_size = 1000 From 30392df9011b65ea1eec360d4b640c4eeaa9b218 Mon Sep 17 00:00:00 2001 From: missytake Date: Sun, 27 Oct 2024 13:56:26 +0100 Subject: [PATCH 29/66] cmdeploy: add argument to specify different SSH host than mail_domain --- cmdeploy/src/cmdeploy/cmdeploy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index cb744b37..2d199fa0 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -58,6 +58,11 @@ def run_cmd_options(parser): action="store_true", help="install/upgrade the server, but disable postfix & dovecot for now" ) + parser.add_argument( + "--ssh-host", + dest="ssh_host", + help="specify an SSH host to deploy to; uses mail_domain from chatmail.ini by default" + ) def run_cmd(args, out): @@ -73,7 +78,8 @@ def run_cmd(args, out): env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else "" deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve() pyinf = "pyinfra --dry" if args.dry_run else "pyinfra" - cmd = f"{pyinf} --ssh-user root {args.config.mail_domain} {deploy_path} -y" + ssh_host = args.config.mail_domain if not args.ssh_host else args.ssh_host + cmd = f"{pyinf} --ssh-user root {ssh_host} {deploy_path} -y" if version.parse(pyinfra.__version__) < version.parse("3"): out.red("Please re-run scripts/initenv.sh to update pyinfra to version 3.") return 1 From 579e6fd1cd926cdc5ee1b79b7f98d71e3fb074b4 Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 28 Oct 2024 14:57:16 +0100 Subject: [PATCH 30/66] added changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 071aa202..5f35f925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ - increase `request_queue_size` for UNIX sockets to 1000. ([#437](https://github.com/deltachat/chatmail/pull/437)) +- add argument to `cmdeploy run` for specifying + a different SSH host than `mail_domain` + ([#439](https://github.com/deltachat/chatmail/pull/439)) + - query autoritative nameserver to bypass DNS cache ([#424](https://github.com/deltachat/chatmail/pull/424)) From 75f11e68dea56bd02f2057a0449d53aa48c7f1fe Mon Sep 17 00:00:00 2001 From: missytake Date: Sat, 26 Oct 2024 10:22:58 +0200 Subject: [PATCH 31/66] updated privacy policy to testrun UG --- www/src/privacy.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/src/privacy.md b/www/src/privacy.md index 5adc5b20..5b0ea01f 100644 --- a/www/src/privacy.md +++ b/www/src/privacy.md @@ -3,8 +3,8 @@ {% if config.mail_domain == "nine.testrun.org" %} Welcome to `{{config.mail_domain}}`, the default chatmail onboarding server for Delta Chat users. -It is operated on the side by a small sysops team employed by [merlinux](https://merlinux.eu), -an open-source R&D company also acting as the fiscal sponsor of Delta Chat app developments. +It is operated on the side by a small sysops team +on a voluntary basis. See [other chatmail servers](https://delta.chat/en/chatmail) for alternative server operators. {% endif %} @@ -253,7 +253,7 @@ is the `{{ config.privacy_supervisor }}`. ## 6. Validity of this privacy policy This data protection declaration is valid -as of *December 2023*. +as of *October 2024*. Due to the further development of our service and offers or due to changed legal or official requirements, it may become necessary to revise this data protection declaration from time to time. From 648bf53e83cea831cd6e29a0bda88200d66644cd Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 16 Oct 2024 11:59:04 +0200 Subject: [PATCH 32/66] Guide on how to migrate chatmail to a new host This guide doesn't require knowing about firewalls, but utilizes the `cmdeploy run --disable-mail` command from #428. supercedes #417 --- README.md | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/README.md b/README.md index c9cc13a7..8a538cc2 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,86 @@ and rejects incorrectly authenticated emails with [`reject_sender_login_mismatch `From:` header must correspond to envelope MAIL FROM, this is ensured by `filtermail` proxy. +## Migrating chatmail server to a new host + +If you want to migrate chatmail from an old machine +to a new machine, +you can use these steps. +They were tested with a linux laptop; +you might need to adjust some of the steps to your environment. + +Let's assume that your `mail_domain` is `mail.example.org`, +all involved machines run Debian 12, +your old server's IP address is `13.37.13.37`, +and your new server's IP address is `13.12.23.42`. + +1. First, copy `/var/lib/acme` to your local machine with `rsync -avz mail.example.org:/var/lib/acme .` + +2. Now, in your local `/etc/hosts`, point your domain to the new machine: `13.12.23.42 mail.example.org` + +3. You need to run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` so you can connect to the new machine via SSH. + +4. Upload /var/lib/acme to the new machine with `rsync -avz acme mail.example.org:/var/lib/`. + +5. On the server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. + +6. Run `cmdeploy run --disable-mail` to install chatmail on the new machine. + postfix and dovecot are disabled for now, + we will enable them later. + +7. Now, point DNS to the new IP addresses. + + You can already remove the old IP addresses from DNS. + Existing Delta Chat users will still be able to connect + to the old server, send and receive messages, + but new users will fail to create new profiles + with your chatmail server. + + If other servers try to deliver messages to your new server they will fail, + but normally email servers will retry delivering messages + for at least a week, so messages will not be lost. + +8. Then point the domain to the old machine in your local `/etc/hosts` again: `13.37.13.37 mail.example.org` + +9. And run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` again so you can connect to the new machine via SSH. + +10. Now you can run `cmdeploy run --disable-mail` to disable your old server. + + Now your users will notice the migration + and will not be able to send or receive messages + until the migration is completed. + +11. After everything is stopped, + you can copy the `/home/vmail/mail` directory to the new server. + It includes all user data, messages, password hashes, etc. + + If you have enough storage on your local machine, + you can simply download it with `rsync -avz mail.example.org:/home/vmail/mail .`, + change `/etc/hosts` and run `ssh-keygen` as in step 11 and 12, + and upload it again with `rsync -avz mail mail.example.org:/home/vmail/`. + + The other way would be copying it + from the old machine to the new machine directly, + which requires setting up an SSH connection + with a new SSH key. + + After this, your new server has all the necessary files to start operating :) + +12. If you haven't done this during the last step, + point your domain to the new machine in your `/etc/hosts` again: `13.12.23.42 mail.example.org` + +13. And run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` a final time + to make sure you can SSH-connect to the new machine. + +14. To be sure the permissions are still fine, + run `chown vmail: -R /home/vmail` on the new server. + +15. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. + Your users can continue using the chatmail server, + and messages which were sent after step 9 should arrive now. + +16. Voilà! Consider removing the entry in your local `/etc/hosts` to clean up. + ## Setting up a reverse proxy A chatmail server does not depend on the client IP address From ebed7ebf5eabdad30e2aa3424e7c79da2d70d854 Mon Sep 17 00:00:00 2001 From: missytake Date: Sun, 27 Oct 2024 14:04:05 +0100 Subject: [PATCH 33/66] doc: migration guide should use new --ssh-host command --- README.md | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 8a538cc2..7d2e3bf9 100644 --- a/README.md +++ b/README.md @@ -269,21 +269,21 @@ all involved machines run Debian 12, your old server's IP address is `13.37.13.37`, and your new server's IP address is `13.12.23.42`. -1. First, copy `/var/lib/acme` to your local machine with `rsync -avz mail.example.org:/var/lib/acme .` +During the guide, you might get a warning about changed SSH Host keys; +in this case, just run `ssh-keygen -R "mail.example.org"` as recommended +to make sure you can connect with SSH. -2. Now, in your local `/etc/hosts`, point your domain to the new machine: `13.12.23.42 mail.example.org` +1. First, copy `/var/lib/acme` to your local machine with `rsync -avz root@13.37.13.37:/var/lib/acme .` -3. You need to run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` so you can connect to the new machine via SSH. +2. Upload /var/lib/acme to the new machine with `rsync -avz acme root@13.12.23.42:/var/lib/`. -4. Upload /var/lib/acme to the new machine with `rsync -avz acme mail.example.org:/var/lib/`. +3. On the new server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. -5. On the server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. - -6. Run `cmdeploy run --disable-mail` to install chatmail on the new machine. +4. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. postfix and dovecot are disabled for now, we will enable them later. -7. Now, point DNS to the new IP addresses. +5. Now, point DNS to the new IP addresses. You can already remove the old IP addresses from DNS. Existing Delta Chat users will still be able to connect @@ -295,24 +295,19 @@ and your new server's IP address is `13.12.23.42`. but normally email servers will retry delivering messages for at least a week, so messages will not be lost. -8. Then point the domain to the old machine in your local `/etc/hosts` again: `13.37.13.37 mail.example.org` - -9. And run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` again so you can connect to the new machine via SSH. - -10. Now you can run `cmdeploy run --disable-mail` to disable your old server. +6. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. Now your users will notice the migration and will not be able to send or receive messages until the migration is completed. -11. After everything is stopped, +7. After everything is stopped, you can copy the `/home/vmail/mail` directory to the new server. It includes all user data, messages, password hashes, etc. If you have enough storage on your local machine, - you can simply download it with `rsync -avz mail.example.org:/home/vmail/mail .`, - change `/etc/hosts` and run `ssh-keygen` as in step 11 and 12, - and upload it again with `rsync -avz mail mail.example.org:/home/vmail/`. + you can simply download it with `rsync -avz 13.37.13.37:/home/vmail/mail .`, + and upload it again with `rsync -avz mail 13.12.23.42:/home/vmail/`. The other way would be copying it from the old machine to the new machine directly, @@ -321,20 +316,13 @@ and your new server's IP address is `13.12.23.42`. After this, your new server has all the necessary files to start operating :) -12. If you haven't done this during the last step, - point your domain to the new machine in your `/etc/hosts` again: `13.12.23.42 mail.example.org` - -13. And run `ssh-keygen -f "/home/$USER/.ssh/known_hosts" -R "mail.example.org"` a final time - to make sure you can SSH-connect to the new machine. - -14. To be sure the permissions are still fine, +8. To be sure the permissions are still fine, run `chown vmail: -R /home/vmail` on the new server. -15. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. +9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. Your users can continue using the chatmail server, and messages which were sent after step 9 should arrive now. - -16. Voilà! Consider removing the entry in your local `/etc/hosts` to clean up. + Voilà! ## Setting up a reverse proxy From 70f77a93ea8c77336e525257cd40482406aa0d6d Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 28 Oct 2024 14:59:28 +0100 Subject: [PATCH 34/66] doc: fix step 9 -> step 6 Co-authored-by: holger krekel --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d2e3bf9..cea1f38d 100644 --- a/README.md +++ b/README.md @@ -321,7 +321,7 @@ to make sure you can connect with SSH. 9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. Your users can continue using the chatmail server, - and messages which were sent after step 9 should arrive now. + and messages which were sent after step 6. should arrive now. Voilà! ## Setting up a reverse proxy From a9779d7e7cd5e357a11a077706422e90bb325274 Mon Sep 17 00:00:00 2001 From: missytake Date: Mon, 28 Oct 2024 15:00:42 +0100 Subject: [PATCH 35/66] add changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f35f925..493a2c22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## untagged +- add guide to migrate chatmail to a new server + ([#429](https://github.com/deltachat/chatmail/pull/429)) + - increase `request_queue_size` for UNIX sockets to 1000. ([#437](https://github.com/deltachat/chatmail/pull/437)) From c35c44ad8d644bffe47ff2c399a94d07622d96e1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 28 Oct 2024 18:16:01 +0000 Subject: [PATCH 36/66] Replace rsync with tar --- README.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index cea1f38d..485a7f30 100644 --- a/README.md +++ b/README.md @@ -273,17 +273,16 @@ During the guide, you might get a warning about changed SSH Host keys; in this case, just run `ssh-keygen -R "mail.example.org"` as recommended to make sure you can connect with SSH. -1. First, copy `/var/lib/acme` to your local machine with `rsync -avz root@13.37.13.37:/var/lib/acme .` +1. First, copy `/var/lib/acme` to your local machine with + `ssh root@13.37.13.37 tar c /var/lib/acme | ssh root@13.12.23.42 tar x -C /var/lib/`. -2. Upload /var/lib/acme to the new machine with `rsync -avz acme root@13.12.23.42:/var/lib/`. +2. On the new server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. -3. On the new server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. - -4. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. +3. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. postfix and dovecot are disabled for now, we will enable them later. -5. Now, point DNS to the new IP addresses. +4. Now, point DNS to the new IP addresses. You can already remove the old IP addresses from DNS. Existing Delta Chat users will still be able to connect @@ -295,13 +294,13 @@ to make sure you can connect with SSH. but normally email servers will retry delivering messages for at least a week, so messages will not be lost. -6. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. +5. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. Now your users will notice the migration and will not be able to send or receive messages until the migration is completed. -7. After everything is stopped, +6. After everything is stopped, you can copy the `/home/vmail/mail` directory to the new server. It includes all user data, messages, password hashes, etc. @@ -316,10 +315,10 @@ to make sure you can connect with SSH. After this, your new server has all the necessary files to start operating :) -8. To be sure the permissions are still fine, +7. To be sure the permissions are still fine, run `chown vmail: -R /home/vmail` on the new server. -9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. +8. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. Your users can continue using the chatmail server, and messages which were sent after step 6. should arrive now. Voilà! From b92d9c889bfa83fac9b2012e111fc8cdcdedd584 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 29 Oct 2024 16:42:35 +0100 Subject: [PATCH 37/66] doc: use ssh+tar to transfer vmail + dkimkeys as well --- README.md | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 485a7f30..eaf00981 100644 --- a/README.md +++ b/README.md @@ -273,16 +273,21 @@ During the guide, you might get a warning about changed SSH Host keys; in this case, just run `ssh-keygen -R "mail.example.org"` as recommended to make sure you can connect with SSH. -1. First, copy `/var/lib/acme` to your local machine with +1. First, copy `/var/lib/acme` to the new server with `ssh root@13.37.13.37 tar c /var/lib/acme | ssh root@13.12.23.42 tar x -C /var/lib/`. + This transfers your TLS certificate. -2. On the new server, run `chown root: -R /var/lib/acme` to make sure the permissions are correct. +2. You should also copy `/etc/dkimkeys` to the new server with + `ssh root@13.37.13.37 tar c /etc/dkimkeys | ssh root@13.12.23.42 tar x -C /etc/` + so the DKIM DNS record stays correct. -3. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. +3. On the new server, run `chown root: -R /var/lib/acme` and `chown root: -R /etc/dkimkeys` to make sure the permissions are correct. + +4. Run `cmdeploy run --disable-mail --ssh-host 13.12.23.42` to install chatmail on the new machine. postfix and dovecot are disabled for now, we will enable them later. -4. Now, point DNS to the new IP addresses. +5. Now, point DNS to the new IP addresses. You can already remove the old IP addresses from DNS. Existing Delta Chat users will still be able to connect @@ -294,31 +299,24 @@ to make sure you can connect with SSH. but normally email servers will retry delivering messages for at least a week, so messages will not be lost. -5. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. +6. Now you can run `cmdeploy run --disable-mail --ssh-host 13.37.13.37` to disable your old server. Now your users will notice the migration and will not be able to send or receive messages until the migration is completed. -6. After everything is stopped, +7. After everything is stopped, you can copy the `/home/vmail/mail` directory to the new server. It includes all user data, messages, password hashes, etc. - If you have enough storage on your local machine, - you can simply download it with `rsync -avz 13.37.13.37:/home/vmail/mail .`, - and upload it again with `rsync -avz mail 13.12.23.42:/home/vmail/`. - - The other way would be copying it - from the old machine to the new machine directly, - which requires setting up an SSH connection - with a new SSH key. + Just run: `ssh root@13.37.13.37 tar c /home/vmail/mail | ssh root@13.12.23.42 tar x -C /home/vmail/` After this, your new server has all the necessary files to start operating :) -7. To be sure the permissions are still fine, +8. To be sure the permissions are still fine, run `chown vmail: -R /home/vmail` on the new server. -8. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. +9. Finally, you can run `cmdeploy run` to turn on chatmail on the new server. Your users can continue using the chatmail server, and messages which were sent after step 6. should arrive now. Voilà! From 5048bde6d02a1cefc58aa0ca22fbdf9117a16ebc Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 22 Oct 2024 17:40:25 +0000 Subject: [PATCH 38/66] Deploy iroh relay --- .../staging-ipv4.testrun.org-default.zone | 1 + .../staging.testrun.org-default.zone | 1 + CHANGELOG.md | 3 + chatmaild/src/chatmaild/config.py | 7 ++- chatmaild/src/chatmaild/ini/chatmail.ini.f | 7 +++ cmdeploy/src/cmdeploy/__init__.py | 56 ++++++++++++++++++- cmdeploy/src/cmdeploy/cmdeploy.py | 8 ++- cmdeploy/src/cmdeploy/dns.py | 9 ++- cmdeploy/src/cmdeploy/iroh-relay.service | 12 ++++ cmdeploy/src/cmdeploy/iroh-relay.toml | 5 ++ cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 | 12 ++++ cmdeploy/src/cmdeploy/remote/rdns.py | 7 ++- .../src/cmdeploy/tests/online/test_1_basic.py | 8 +-- cmdeploy/src/cmdeploy/tests/test_dns.py | 12 ++-- 14 files changed, 127 insertions(+), 21 deletions(-) create mode 100644 cmdeploy/src/cmdeploy/iroh-relay.service create mode 100644 cmdeploy/src/cmdeploy/iroh-relay.toml diff --git a/.github/workflows/staging-ipv4.testrun.org-default.zone b/.github/workflows/staging-ipv4.testrun.org-default.zone index 785b71aa..5c7df6d8 100644 --- a/.github/workflows/staging-ipv4.testrun.org-default.zone +++ b/.github/workflows/staging-ipv4.testrun.org-default.zone @@ -17,4 +17,5 @@ $TTL 300 ;; DNS records. @ IN A 37.27.95.249 mta-sts.staging-ipv4.testrun.org. CNAME staging-ipv4.testrun.org. +iroh.staging-ipv4.testrun.org. CNAME staging-ipv4.testrun.org. www.staging-ipv4.testrun.org. CNAME staging-ipv4.testrun.org. diff --git a/.github/workflows/staging.testrun.org-default.zone b/.github/workflows/staging.testrun.org-default.zone index 444e4d86..311c95f5 100644 --- a/.github/workflows/staging.testrun.org-default.zone +++ b/.github/workflows/staging.testrun.org-default.zone @@ -17,5 +17,6 @@ $TTL 300 ;; DNS records. @ IN A 37.27.24.139 mta-sts.staging2.testrun.org. CNAME staging2.testrun.org. +iroh.staging2.testrun.org. CNAME staging2.testrun.org. www.staging2.testrun.org. CNAME staging2.testrun.org. diff --git a/CHANGELOG.md b/CHANGELOG.md index 493a2c22..50a646b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ - add guide to migrate chatmail to a new server ([#429](https://github.com/deltachat/chatmail/pull/429)) +- deploy `iroh-relay` (requires new "iroh.{mail_domain}" DNS entry) + ([#434](https://github.com/deltachat/chatmail/pull/434)) + - increase `request_queue_size` for UNIX sockets to 1000. ([#437](https://github.com/deltachat/chatmail/pull/437)) diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index 53c83f94..b453389c 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -33,7 +33,12 @@ def __init__(self, inipath, params): self.mtail_address = params.get("mtail_address") self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true" self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true" - self.iroh_relay = params.get("iroh_relay") + if "iroh_relay" not in params: + self.iroh_relay = "https://iroh." + params["mail_domain"] + self.enable_iroh_relay = True + else: + self.iroh_relay = params["iroh_relay"].strip() + self.enable_iroh_relay = False self.privacy_postal = params.get("privacy_postal") self.privacy_mail = params.get("privacy_mail") self.privacy_pdo = params.get("privacy_pdo") diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index e8e8ef5d..bde5bf5b 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -55,6 +55,13 @@ # if set to "True" IPv6 is disabled disable_ipv6 = False +# Defaults to https://iroh.{{mail_domain}} and running `iroh-relay` on the chatmail +# service. +# If you set it to anything else, the service will be disabled +# and users will be directed to use the given iroh relay URL. +# Set it to empty string if you want users to use their default iroh relay. +# iroh_relay = + # Address on which `mtail` listens, # e.g. 127.0.0.1 or some private network # address like 192.168.10.1. diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index 283e268b..ce361858 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -10,7 +10,7 @@ from pathlib import Path from chatmaild.config import Config, read_config -from pyinfra import host +from pyinfra import host, facts from pyinfra.facts.files import File from pyinfra.facts.systemd import SystemdEnabled from pyinfra.operations import apt, files, pip, server, systemd @@ -479,6 +479,55 @@ def deploy_mtail(config): ) +def deploy_iroh_relay(config) -> None: + (url, sha256sum) = { + "x86_64": ("https://github.com/n0-computer/iroh/releases/download/v0.27.0/iroh-relay-v0.27.0-x86_64-unknown-linux-musl.tar.gz", "8af7f6d29d17476ce5c3053c3161db5793cb2ac49057d0bcaf689436cdccbeab"), + "aarch64": ("https://github.com/n0-computer/iroh/releases/download/v0.27.0/iroh-relay-v0.27.0-aarch64-unknown-linux-musl.tar.gz", "18039f0d39df78922a5055a0d4a5a8fa98a2a0e19b1eaa4c3fe6db73b8698697") + }[host.get_fact(facts.server.Arch)] + + server.shell( + name="Download iroh-relay", + commands=[ + f"(echo '{sha256sum} /usr/local/bin/iroh-relay' | sha256sum -c) || curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay", + "chmod 755 /usr/local/bin/iroh-relay", + ], + ) + + need_restart = False + + systemd_unit = files.put( + name="Upload iroh-relay systemd unit", + src=importlib.resources.files(__package__).joinpath( + "iroh-relay.service" + ), + dest="/etc/systemd/system/iroh-relay.service", + user="root", + group="root", + mode="644", + ) + need_restart |= systemd_unit.changed + + iroh_config = files.put( + name=f"Upload iroh-relay config", + src=importlib.resources.files(__package__).joinpath( + "iroh-relay.toml" + ), + dest=f"/etc/iroh-relay.toml", + user="iroh", + group="iroh", + mode="600", + ) + need_restart |= iroh_config.changed + + systemd.service( + name="Start and enable iroh-relay", + service="iroh-relay.service", + running=True, + enabled=config.enable_iroh_relay, + restarted=need_restart, + ) + + def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: """Deploy a chat-mail instance. @@ -508,6 +557,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: system=True, ) server.user(name="Create echobot user", user="echobot", system=True) + server.user(name="Create iroh user", user="iroh", system=True) # Add our OBS repository for dovecot_no_delay files.put( @@ -556,9 +606,11 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: enabled=True, ) + deploy_iroh_relay(config) + # Deploy acmetool to have TLS certificates. deploy_acmetool( - domains=[mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"], + domains=[mail_domain, f"mta-sts.{mail_domain}", f"iroh.{mail_domain}", f"www.{mail_domain}"], ) apt.packages( diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index 2d199fa0..5a66fd52 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -69,8 +69,9 @@ def run_cmd(args, out): """Deploy chatmail services on the remote server.""" sshexec = args.get_sshexec() - remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain) - if not dns.check_initial_remote_data(remote_data, print=out.red): + require_iroh = args.config.enable_iroh_relay + remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain, require_iroh) + if not dns.check_initial_remote_data(remote_data, require_iroh, print=out.red): return 1 env = os.environ.copy() @@ -109,7 +110,8 @@ def dns_cmd_options(parser): def dns_cmd(args, out): """Check DNS entries and optionally generate dns zone file.""" sshexec = args.get_sshexec() - remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain) + require_iroh = args.config.enable_iroh_relay + remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain, require_iroh) if not remote_data: return 1 diff --git a/cmdeploy/src/cmdeploy/dns.py b/cmdeploy/src/cmdeploy/dns.py index c672da33..b8e05f21 100644 --- a/cmdeploy/src/cmdeploy/dns.py +++ b/cmdeploy/src/cmdeploy/dns.py @@ -6,19 +6,22 @@ from . import remote -def get_initial_remote_data(sshexec, mail_domain): +def get_initial_remote_data(sshexec, mail_domain, iroh_enabled): return sshexec.logged( - call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain) + call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain, iroh_enabled=iroh_enabled) ) -def check_initial_remote_data(remote_data, print=print): +def check_initial_remote_data(remote_data, require_iroh, *, print=print): mail_domain = remote_data["mail_domain"] if not remote_data["A"] and not remote_data["AAAA"]: print(f"Missing A and/or AAAA DNS records for {mail_domain}!") elif remote_data["MTA_STS"] != f"{mail_domain}.": print("Missing MTA-STS CNAME record:") print(f"mta-sts.{mail_domain}. CNAME {mail_domain}.") + elif require_iroh and remote_data["IROH"] != f"{mail_domain}.": + print("Missing iroh CNAME record:") + print(f"iroh.{mail_domain}. CNAME {mail_domain}.") elif remote_data["WWW"] != f"{mail_domain}.": print("Missing www CNAME record:") print(f"www.{mail_domain}. CNAME {mail_domain}.") diff --git a/cmdeploy/src/cmdeploy/iroh-relay.service b/cmdeploy/src/cmdeploy/iroh-relay.service new file mode 100644 index 00000000..004e8518 --- /dev/null +++ b/cmdeploy/src/cmdeploy/iroh-relay.service @@ -0,0 +1,12 @@ +[Unit] +Description=Iroh relay + +[Service] +ExecStart=/usr/local/bin/iroh-relay --config-path /etc/iroh-relay.toml +Restart=on-failure +RestartSec=5s +User=iroh +Group=iroh + +[Install] +WantedBy=multi-user.target diff --git a/cmdeploy/src/cmdeploy/iroh-relay.toml b/cmdeploy/src/cmdeploy/iroh-relay.toml new file mode 100644 index 00000000..35b2f4ab --- /dev/null +++ b/cmdeploy/src/cmdeploy/iroh-relay.toml @@ -0,0 +1,5 @@ +enable_relay = true +http_bind_addr = "[::]:3340" +enable_stun = true +enable_metrics = false +metrics_bind_addr = "127.0.0.1:9092" diff --git a/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 b/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 index cb53a1b8..5797b4c0 100644 --- a/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 +++ b/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 @@ -108,4 +108,16 @@ http { return 301 $scheme://{{ config.domain_name }}$request_uri; access_log syslog:server=unix:/dev/log,facility=local7; } + + # Pass iroh. to iroh-relay service. + server { + listen 8443 ssl; + {% if not disable_ipv6 %} + listen [::]:8443 ssl; + {% endif %} + server_name iroh.{{ config.domain_name }}; + location / { + proxy_pass http://127.0.0.1:3340; + } + } } diff --git a/cmdeploy/src/cmdeploy/remote/rdns.py b/cmdeploy/src/cmdeploy/remote/rdns.py index 77093503..107d7d23 100644 --- a/cmdeploy/src/cmdeploy/remote/rdns.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -15,7 +15,7 @@ from .rshell import CalledProcessError, shell -def perform_initial_checks(mail_domain): +def perform_initial_checks(mail_domain, iroh_enabled): """Collecting initial DNS settings.""" assert mail_domain if not shell("dig", fail_ok=True): @@ -23,13 +23,14 @@ def perform_initial_checks(mail_domain): A = query_dns("A", mail_domain) AAAA = query_dns("AAAA", mail_domain) MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}") + IROH = query_dns("CNAME", f"iroh.{mail_domain}") WWW = query_dns("CNAME", f"www.{mail_domain}") - res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW) + res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, IROH=IROH, WWW=WWW) res["acme_account_url"] = shell("acmetool account-url", fail_ok=True) res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim") - if not MTA_STS or not WWW or (not A and not AAAA): + if not MTA_STS or (not IROH and not iroh_enabled) or not WWW or (not A and not AAAA): return res # parse out sts-id if exists, example: "v=STSv1; id=2090123" diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index f5f2c023..13424ef3 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -18,13 +18,13 @@ def test_ls(self, sshexec): def test_perform_initial(self, sshexec, maildomain): res = sshexec( - remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain, iroh_enabled=True) ) assert res["A"] or res["AAAA"] def test_logged(self, sshexec, maildomain, capsys): sshexec.logged( - remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain, iroh_enabled=True) ) out, err = capsys.readouterr() assert err.startswith("Collecting") @@ -33,7 +33,7 @@ def test_logged(self, sshexec, maildomain, capsys): sshexec.verbose = True sshexec.logged( - remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain, iroh_enabled=True) ) out, err = capsys.readouterr() lines = err.split("\n") @@ -44,7 +44,7 @@ def test_exception(self, sshexec, capsys): try: sshexec.logged( remote.rdns.perform_initial_checks, - kwargs=dict(mail_domain=None), + kwargs=dict(mail_domain=None, iroh_enabled=True), ) except sshexec.FuncError as e: assert "rdns.py" in str(e) diff --git a/cmdeploy/src/cmdeploy/tests/test_dns.py b/cmdeploy/src/cmdeploy/tests/test_dns.py index fd11095f..a69a7f73 100644 --- a/cmdeploy/src/cmdeploy/tests/test_dns.py +++ b/cmdeploy/src/cmdeploy/tests/test_dns.py @@ -26,6 +26,7 @@ def mockdns(mockdns_base): "AAAA": {"some.domain": "fde5:cd7a:9e1c:3240:5a99:936f:cdac:53ae"}, "CNAME": { "mta-sts.some.domain": "some.domain.", + "iroh.some.domain": "some.domain.", "www.some.domain": "some.domain.", }, } @@ -35,30 +36,31 @@ def mockdns(mockdns_base): class TestPerformInitialChecks: def test_perform_initial_checks_ok1(self, mockdns): - remote_data = remote.rdns.perform_initial_checks("some.domain") + remote_data = remote.rdns.perform_initial_checks("some.domain", iroh_enabled=True) assert remote_data["A"] == mockdns["A"]["some.domain"] assert remote_data["AAAA"] == mockdns["AAAA"]["some.domain"] assert remote_data["MTA_STS"] == mockdns["CNAME"]["mta-sts.some.domain"] + assert remote_data["IROH"] == mockdns["CNAME"]["iroh.some.domain"] assert remote_data["WWW"] == mockdns["CNAME"]["www.some.domain"] @pytest.mark.parametrize("drop", ["A", "AAAA"]) def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop): del mockdns[drop] - remote_data = remote.rdns.perform_initial_checks("some.domain") + remote_data = remote.rdns.perform_initial_checks("some.domain", iroh_enabled=True) assert not remote_data[drop] l = [] - res = check_initial_remote_data(remote_data, print=l.append) + res = check_initial_remote_data(remote_data, require_iroh=True, print=l.append) assert res assert not l def test_perform_initial_checks_no_mta_sts(self, mockdns): del mockdns["CNAME"]["mta-sts.some.domain"] - remote_data = remote.rdns.perform_initial_checks("some.domain") + remote_data = remote.rdns.perform_initial_checks("some.domain", iroh_enabled=True) assert not remote_data["MTA_STS"] l = [] - res = check_initial_remote_data(remote_data, print=l.append) + res = check_initial_remote_data(remote_data, require_iroh=True, print=l.append) assert not res assert len(l) == 2 From aae05ac8326b63964f37906a8089d79b541f2021 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 30 Oct 2024 10:52:01 +0100 Subject: [PATCH 39/66] CI: set necessary DNS records before cmdeploy run, so it doesn't fail --- .github/workflows/test-and-deploy-ipv4only.yaml | 2 ++ .github/workflows/test-and-deploy.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/test-and-deploy-ipv4only.yaml b/.github/workflows/test-and-deploy-ipv4only.yaml index fe1046b7..fb9c449e 100644 --- a/.github/workflows/test-and-deploy-ipv4only.yaml +++ b/.github/workflows/test-and-deploy-ipv4only.yaml @@ -38,7 +38,9 @@ jobs: if [ -f dkimkeys-ipv4/dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys-ipv4 root@ns.testrun.org:/tmp/ || true; fi if [ "$(ls -A acme-ipv4/acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme-ipv4 root@ns.testrun.org:/tmp/ || true; fi # make sure CAA record isn't set + scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging-ipv4.testrun.org.zone + ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone ssh root@ns.testrun.org systemctl reload nsd - name: rebuild staging-ipv4.testrun.org to have a clean VPS diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index 686f77d0..cd4dd507 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -38,7 +38,9 @@ jobs: if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi # make sure CAA record isn't set + scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone + ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone ssh root@ns.testrun.org systemctl reload nsd - name: rebuild staging2.testrun.org to have a clean VPS From af17b459ba5b20ebd7c53655eeb1d5d3dca7487e Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 30 Oct 2024 12:23:09 +0100 Subject: [PATCH 40/66] also change privacy policy to circumscribe iroh-relay services --- CHANGELOG.md | 7 ++++--- www/src/privacy.md | 24 +++++++++++++++--------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50a646b0..5154275f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,13 @@ ## untagged +- deploy `iroh-relay` (requires new "iroh.{mail_domain}" DNS entry), + also update "realtime relay services" in privacy policy. + ([#434](https://github.com/deltachat/chatmail/pull/434)) + - add guide to migrate chatmail to a new server ([#429](https://github.com/deltachat/chatmail/pull/429)) -- deploy `iroh-relay` (requires new "iroh.{mail_domain}" DNS entry) - ([#434](https://github.com/deltachat/chatmail/pull/434)) - - increase `request_queue_size` for UNIX sockets to 1000. ([#437](https://github.com/deltachat/chatmail/pull/437)) diff --git a/www/src/privacy.md b/www/src/privacy.md index 5b0ea01f..c6e918dc 100644 --- a/www/src/privacy.md +++ b/www/src/privacy.md @@ -54,18 +54,18 @@ We have appointed a data protection officer: ## 2. Processing when using chat e-mail services -We provide e-mail services optimized for the use from [Delta Chat](https://delta.chat) apps +We provide services optimized for the use from [Delta Chat](https://delta.chat) apps and process only the data necessary -for the setup and technical execution of the e-mail dispatch. -The purpose of the processing is to -read, write, manage, delete, send, and receive emails. +for the setup and technical execution of message delivery. +The purpose of the processing is that users can +read, write, manage, delete, send, and receive chat messages. For this purpose, we operate server-side software -that enables us to send and receive e-mail messages. -Allowing the use of the e-mail service, -we process the following data and details: +that enables us to send and receive messages. -- Outgoing and incoming messages (SMTP) are stored for transit +We process the following data and details: + +- Outgoing and incoming messages (SMTP) are stored for transit on behalf of their users until the message can be delivered. - E-Mail-Messages are stored for the recipient and made accessible via IMAP protocols, @@ -74,9 +74,15 @@ we process the following data and details: - IMAP and SMTP protocols are password protected with unique credentials for each account. -- Users can retrieve or delete all stored messages +- Users can retrieve or delete all stored messages without intervention from the operators using standard IMAP client tools. +- Users can connect to a "realtime relay service" + to establish Peer-to-Peer connection between user devices, + allowing them to send and retrieve ephemeral messages + which are never stored on the chatmail server, also not in encrypted form. + + ### 2.1 Account setup Creating an account happens in one of two ways on our mail servers: From dfc1042a3f05d32fb42de124267e18071e4b2c7f Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 30 Oct 2024 12:42:41 +0100 Subject: [PATCH 41/66] CI: fix #422 nested acme&dkimkeys folders --- .github/workflows/test-and-deploy-ipv4only.yaml | 4 ++-- .github/workflows/test-and-deploy.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-and-deploy-ipv4only.yaml b/.github/workflows/test-and-deploy-ipv4only.yaml index fb9c449e..37d684fd 100644 --- a/.github/workflows/test-and-deploy-ipv4only.yaml +++ b/.github/workflows/test-and-deploy-ipv4only.yaml @@ -66,8 +66,8 @@ jobs: rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme-ipv4 acme-restore || true rsync -avz root@ns.testrun.org:/tmp/dkimkeys-ipv4 dkimkeys-restore || true # restore acme & dkim state to staging2.testrun.org - rsync -avz acme-restore/acme-ipv4/acme root@staging-ipv4.testrun.org:/var/lib/acme || true - rsync -avz dkimkeys-restore/dkimkeys-ipv4/dkimkeys root@staging-ipv4.testrun.org:/etc/dkimkeys || true + rsync -avz acme-restore/acme-ipv4/acme root@staging-ipv4.testrun.org:/var/lib/ || true + rsync -avz dkimkeys-restore/dkimkeys-ipv4/dkimkeys root@staging-ipv4.testrun.org:/etc/ || true ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown root:root -R /var/lib/acme || true - name: run formatting checks diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index cd4dd507..3cb4b26f 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -66,8 +66,8 @@ jobs: rsync -e "ssh -o StrictHostKeyChecking=accept-new" -avz root@ns.testrun.org:/tmp/acme acme-restore || true rsync -avz root@ns.testrun.org:/tmp/dkimkeys dkimkeys-restore || true # restore acme & dkim state to staging2.testrun.org - rsync -avz acme-restore/acme/ root@staging2.testrun.org:/var/lib/acme || true - rsync -avz dkimkeys-restore/dkimkeys/ root@staging2.testrun.org:/etc/dkimkeys || true + rsync -avz acme-restore/acme root@staging2.testrun.org:/var/lib/ || true + rsync -avz dkimkeys-restore/dkimkeys root@staging2.testrun.org:/etc/ || true ssh -o StrictHostKeyChecking=accept-new -v root@staging2.testrun.org chown root:root -R /var/lib/acme || true - name: run formatting checks From 3098afb342f7ad474912f072965d9e056a9877b4 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 30 Oct 2024 13:17:59 +0100 Subject: [PATCH 42/66] CI: fix accepting ns.testrun.org SSH Host Key --- .github/workflows/test-and-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index 3cb4b26f..53aa79a8 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -38,8 +38,8 @@ jobs: if [ -f dkimkeys/opendkim.private ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" dkimkeys root@ns.testrun.org:/tmp/ || true; fi if [ "$(ls -A acme/certs)" ]; then rsync -avz -e "ssh -o StrictHostKeyChecking=accept-new" acme root@ns.testrun.org:/tmp/ || true; fi # make sure CAA record isn't set - scp .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone - ssh -o StrictHostKeyChecking=accept-new root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone + scp -o StrictHostKeyChecking=accept-new .github/workflows/staging.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging2.testrun.org.zone + ssh root@ns.testrun.org sed -i '/CAA/d' /etc/nsd/staging2.testrun.org.zone ssh root@ns.testrun.org nsd-checkzone staging2.testrun.org /etc/nsd/staging2.testrun.org.zone ssh root@ns.testrun.org systemctl reload nsd From a5fd5cfb55b63bb20ae7ce1d6ef9cc6464444b7a Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 30 Oct 2024 15:31:25 +0100 Subject: [PATCH 43/66] dovecot: disable anvil authentication penalty fix #441 --- CHANGELOG.md | 3 +++ cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5154275f..aa10953c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ - add guide to migrate chatmail to a new server ([#429](https://github.com/deltachat/chatmail/pull/429)) +- disable anvil authentication penalty + ([#414](https://github.com/deltachat/chatmail/pull/444) + - increase `request_queue_size` for UNIX sockets to 1000. ([#437](https://github.com/deltachat/chatmail/pull/437)) diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index f4b68f9e..d8965343 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -194,6 +194,15 @@ service imap-login { process_min_avail = 10 } +service anvil { + # We are disabling anvil penalty on failed login attempts + # because it can only detect brute forcing by IP address + # not by username. As the correct IP address is not handed + # to dovecot anyway, it is more of hindrance than of use. + # See for details. + unix_listener anvil-auth-penalty { mode = 0 } +} + ssl = required ssl_cert = Date: Wed, 30 Oct 2024 15:41:35 +0100 Subject: [PATCH 44/66] dovecot: fix syntax error --- cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 index d8965343..1390e4db 100644 --- a/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 +++ b/cmdeploy/src/cmdeploy/dovecot/dovecot.conf.j2 @@ -200,7 +200,9 @@ service anvil { # not by username. As the correct IP address is not handed # to dovecot anyway, it is more of hindrance than of use. # See for details. - unix_listener anvil-auth-penalty { mode = 0 } + unix_listener anvil-auth-penalty { + mode = 0 + } } ssl = required From 72df078d0253ee2ef07bd200f9426f5cc3a90d97 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 6 Sep 2024 18:36:33 +0200 Subject: [PATCH 45/66] add support for specifying whole domains for passthrough --- CHANGELOG.md | 3 +++ chatmaild/src/chatmaild/filtermail.py | 11 ++++++++- chatmaild/src/chatmaild/ini/chatmail.ini.f | 2 +- .../src/chatmaild/tests/test_filtermail.py | 24 +++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa10953c..98e1b8a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ - fix checking for required DNS records ([#412](https://github.com/deltachat/chatmail/pull/412)) +- add support for specifying whole domains for recipient passthrough list + ([#408](https://github.com/deltachat/chatmail/pull/408)) + - add a paragraph about "account deletion" to info page ([#405](https://github.com/deltachat/chatmail/pull/405)) diff --git a/chatmaild/src/chatmaild/filtermail.py b/chatmaild/src/chatmaild/filtermail.py index 6e0dbf53..47ab791a 100644 --- a/chatmaild/src/chatmaild/filtermail.py +++ b/chatmaild/src/chatmaild/filtermail.py @@ -142,6 +142,15 @@ async def asyncmain_beforequeue(config): Controller(BeforeQueueHandler(config), hostname="127.0.0.1", port=port).start() +def recipient_matches_passthrough(recipient, passthrough_recipients): + for addr in passthrough_recipients: + if recipient == addr: + return True + if addr[0] == "@" and recipient.endswith(addr): + return True + return False + + class BeforeQueueHandler: def __init__(self, config): self.config = config @@ -205,7 +214,7 @@ def check_DATA(self, envelope): if envelope.mail_from == recipient: # Always allow sending emails to self. continue - if recipient in passthrough_recipients: + if recipient_matches_passthrough(recipient, passthrough_recipients): continue res = recipient.split("@") if len(res) != 2: diff --git a/chatmaild/src/chatmaild/ini/chatmail.ini.f b/chatmaild/src/chatmaild/ini/chatmail.ini.f index bde5bf5b..60fe1dcd 100644 --- a/chatmaild/src/chatmaild/ini/chatmail.ini.f +++ b/chatmaild/src/chatmaild/ini/chatmail.ini.f @@ -39,7 +39,7 @@ passthrough_senders = # list of e-mail recipients for which to accept outbound un-encrypted mails -# (space-separated) +# (space-separated, item may start with "@" to whitelist whole recipient domains) passthrough_recipients = xstore@testrun.org # diff --git a/chatmaild/src/chatmaild/tests/test_filtermail.py b/chatmaild/src/chatmaild/tests/test_filtermail.py index 6072d656..2bd5b2b1 100644 --- a/chatmaild/src/chatmaild/tests/test_filtermail.py +++ b/chatmaild/src/chatmaild/tests/test_filtermail.py @@ -121,6 +121,30 @@ class env2: assert "500" in handler.check_DATA(envelope=env2) +def test_passthrough_domains(maildata, gencreds, handler): + from_addr = gencreds()[0] + to_addr = "privacy@x.y.z" + handler.config.passthrough_recipients = ["@x.y.z"] + false_to = "something@x.y" + + msg = maildata("plain.eml", from_addr=from_addr, to_addr=to_addr) + + class env: + mail_from = from_addr + rcpt_tos = [to_addr] + content = msg.as_bytes() + + # assert that None/no error is returned + assert not handler.check_DATA(envelope=env) + + class env2: + mail_from = from_addr + rcpt_tos = [to_addr, false_to] + content = msg.as_bytes() + + assert "500" in handler.check_DATA(envelope=env2) + + def test_passthrough_senders(gencreds, handler, maildata): acc1 = gencreds()[0] to_addr = "recipient@something.org" From d3e71aa394c207846850c82c081ced8956949e5f Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 30 Oct 2024 12:30:57 +0100 Subject: [PATCH 46/66] streamline intro, mention IP addresses --- www/src/privacy.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/www/src/privacy.md b/www/src/privacy.md index c6e918dc..8e2514e0 100644 --- a/www/src/privacy.md +++ b/www/src/privacy.md @@ -23,18 +23,22 @@ A chatmail server behaves more like the Signal messaging server but does not know about phone numbers and securely and automatically interoperates with other chatmail and classic e-mail servers. -In particular, this chatmail server +Unlike classic e-mail servers, this chatmail server - unconditionally removes messages after {{ config.delete_mails_after }} days, - prohibits sending out un-encrypted messages, -- only has temporary log files used for debugging purposes. +- does not store or collect Internet addresses ("IP addresses"), + +- only uses ephemeral log files used for debugging purposes. + +Due to the lack of personal data processing +we may not need to provide a privacy policy. + +Nevertheless, we provide more legal details below to make life easier +for data protection specialists and lawyers scrutinizing chatmail operations. -Legally, authorities might still regard chatmail as a "classic e-mail" server -which collects and retains personal data. -We do not agree on this interpretation. Nevertheless, we provide more legal details below -to make life easier for data protection specialists and lawyers scrutinizing chatmail operations. ## 1. Name and contact information From 99fbe1d4c40cfe23c5774af585ca73eb9162477b Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 30 Oct 2024 17:12:30 +0100 Subject: [PATCH 47/66] Apply suggestions from code review Co-authored-by: missytake --- www/src/privacy.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/src/privacy.md b/www/src/privacy.md index 8e2514e0..1b5997fd 100644 --- a/www/src/privacy.md +++ b/www/src/privacy.md @@ -29,9 +29,9 @@ Unlike classic e-mail servers, this chatmail server - prohibits sending out un-encrypted messages, -- does not store or collect Internet addresses ("IP addresses"), +- does not store Internet addresses ("IP addresses"), -- only uses ephemeral log files used for debugging purposes. +- does not process IP addresses in ways which could relate them to email addresses. Due to the lack of personal data processing we may not need to provide a privacy policy. From fe51dbd844ce36b23cf30c5002732c0cf94b6802 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 30 Oct 2024 17:16:41 +0100 Subject: [PATCH 48/66] streamline --- www/src/privacy.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/www/src/privacy.md b/www/src/privacy.md index 1b5997fd..a6626960 100644 --- a/www/src/privacy.md +++ b/www/src/privacy.md @@ -31,13 +31,13 @@ Unlike classic e-mail servers, this chatmail server - does not store Internet addresses ("IP addresses"), -- does not process IP addresses in ways which could relate them to email addresses. +- does not process IP addresses in relation to email addresses. -Due to the lack of personal data processing -we may not need to provide a privacy policy. +Due to the resulting lack of personal data processing +this chatmail server may not require a privacy policy. -Nevertheless, we provide more legal details below to make life easier -for data protection specialists and lawyers scrutinizing chatmail operations. +Nevertheless, we provide legal details below to make life easier +for data protection specialists and lawyers scrutinizing chatmail operations. From 2c0b659893259d48c8d131e1ec208a7003d706e1 Mon Sep 17 00:00:00 2001 From: missytake Date: Thu, 31 Oct 2024 17:36:56 +0100 Subject: [PATCH 49/66] dns: add iroh CNAME to zonefile --- cmdeploy/src/cmdeploy/chatmail.zone.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/cmdeploy/src/cmdeploy/chatmail.zone.j2 b/cmdeploy/src/cmdeploy/chatmail.zone.j2 index f0ba176f..10b07758 100644 --- a/cmdeploy/src/cmdeploy/chatmail.zone.j2 +++ b/cmdeploy/src/cmdeploy/chatmail.zone.j2 @@ -11,6 +11,7 @@ _mta-sts.{{ mail_domain }}. TXT "v=STSv1; id={{ sts_id }}" mta-sts.{{ mail_domain }}. CNAME {{ mail_domain }}. www.{{ mail_domain }}. CNAME {{ mail_domain }}. +iroh.{{ mail_domain }}. CNAME {{ mail_domain }}. {{ dkim_entry }} ; From 35a254fc1cd1d2dba3594dfb71eebbccf3722e10 Mon Sep 17 00:00:00 2001 From: missytake Date: Thu, 31 Oct 2024 17:59:17 +0100 Subject: [PATCH 50/66] acmetool: only request iroh certificate if it's required --- cmdeploy/src/cmdeploy/__init__.py | 8 ++++++-- cmdeploy/src/cmdeploy/cmdeploy.py | 1 + cmdeploy/src/cmdeploy/deploy.py | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index ce361858..8ac0654f 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -528,11 +528,12 @@ def deploy_iroh_relay(config) -> None: ) -def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: +def deploy_chatmail(config_path: Path, disable_mail: bool, require_iroh: bool) -> None: """Deploy a chat-mail instance. :param config_path: path to chatmail.ini :param disable_mail: whether to disable postfix & dovecot + :param require_iroh: whether to request a TLS certificate for iroh.$mail_domain """ config = read_config(config_path) check_config(config) @@ -609,8 +610,11 @@ def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: deploy_iroh_relay(config) # Deploy acmetool to have TLS certificates. + tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"] + if require_iroh: + tls_domains.append(f"iroh.{mail_domain}") deploy_acmetool( - domains=[mail_domain, f"mta-sts.{mail_domain}", f"iroh.{mail_domain}", f"www.{mail_domain}"], + domains=tls_domains, ) apt.packages( diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index 5a66fd52..fe7970dc 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -77,6 +77,7 @@ def run_cmd(args, out): env = os.environ.copy() env["CHATMAIL_INI"] = args.inipath env["CHATMAIL_DISABLE_MAIL"] = "True" if args.disable_mail else "" + env["CHATMAIL_REQUIRE_IROH"] = "True" if require_iroh else "" deploy_path = importlib.resources.files(__package__).joinpath("deploy.py").resolve() pyinf = "pyinfra --dry" if args.dry_run else "pyinfra" ssh_host = args.config.mail_domain if not args.ssh_host else args.ssh_host diff --git a/cmdeploy/src/cmdeploy/deploy.py b/cmdeploy/src/cmdeploy/deploy.py index 0ea153d7..9c8e7400 100644 --- a/cmdeploy/src/cmdeploy/deploy.py +++ b/cmdeploy/src/cmdeploy/deploy.py @@ -12,8 +12,9 @@ def main(): importlib.resources.files("cmdeploy").joinpath("../../../chatmail.ini"), ) disable_mail = bool(os.environ.get('CHATMAIL_DISABLE_MAIL')) + require_iroh = bool(os.environ.get('CHATMAIL_REQUIRE_IROH')) - deploy_chatmail(config_path, disable_mail) + deploy_chatmail(config_path, disable_mail, require_iroh) if pyinfra.is_cli: From 12217437e32af01f81eee6461f682252dd46ce55 Mon Sep 17 00:00:00 2001 From: missytake Date: Sat, 2 Nov 2024 12:26:00 +0100 Subject: [PATCH 51/66] cmdeploy: install curl for downloading iroh --- cmdeploy/src/cmdeploy/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index 8ac0654f..120d0f4a 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -485,6 +485,11 @@ def deploy_iroh_relay(config) -> None: "aarch64": ("https://github.com/n0-computer/iroh/releases/download/v0.27.0/iroh-relay-v0.27.0-aarch64-unknown-linux-musl.tar.gz", "18039f0d39df78922a5055a0d4a5a8fa98a2a0e19b1eaa4c3fe6db73b8698697") }[host.get_fact(facts.server.Arch)] + apt.packages( + name="Install curl", + packages=["curl"], + ) + server.shell( name="Download iroh-relay", commands=[ From 95f8c4b269d2efe5225d163c56a7ac4279b2a998 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 5 Nov 2024 13:18:11 +0000 Subject: [PATCH 52/66] Update iroh and remove iroh. subdomain --- CHANGELOG.md | 4 +-- chatmaild/src/chatmaild/config.py | 2 +- cmdeploy/src/cmdeploy/__init__.py | 31 +++++++++--------- cmdeploy/src/cmdeploy/chatmail.zone.j2 | 1 - cmdeploy/src/cmdeploy/cmdeploy.py | 7 ++-- cmdeploy/src/cmdeploy/deploy.py | 3 +- cmdeploy/src/cmdeploy/dns.py | 9 ++---- cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 | 32 ++++++++++++------- cmdeploy/src/cmdeploy/remote/rdns.py | 7 ++-- .../src/cmdeploy/tests/online/test_1_basic.py | 8 ++--- cmdeploy/src/cmdeploy/tests/test_dns.py | 12 +++---- 11 files changed, 57 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98e1b8a9..097a8ad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## untagged -- deploy `iroh-relay` (requires new "iroh.{mail_domain}" DNS entry), - also update "realtime relay services" in privacy policy. +- deploy `iroh-relay` and also update "realtime relay services" in privacy policy. ([#434](https://github.com/deltachat/chatmail/pull/434)) + ([#451](https://github.com/deltachat/chatmail/pull/451)) - add guide to migrate chatmail to a new server ([#429](https://github.com/deltachat/chatmail/pull/429)) diff --git a/chatmaild/src/chatmaild/config.py b/chatmaild/src/chatmaild/config.py index b453389c..670b2042 100644 --- a/chatmaild/src/chatmaild/config.py +++ b/chatmaild/src/chatmaild/config.py @@ -34,7 +34,7 @@ def __init__(self, inipath, params): self.disable_ipv6 = params.get("disable_ipv6", "false").lower() == "true" self.imap_rawlog = params.get("imap_rawlog", "false").lower() == "true" if "iroh_relay" not in params: - self.iroh_relay = "https://iroh." + params["mail_domain"] + self.iroh_relay = "https://" + params["mail_domain"] self.enable_iroh_relay = True else: self.iroh_relay = params["iroh_relay"].strip() diff --git a/cmdeploy/src/cmdeploy/__init__.py b/cmdeploy/src/cmdeploy/__init__.py index 120d0f4a..df5bbaa2 100644 --- a/cmdeploy/src/cmdeploy/__init__.py +++ b/cmdeploy/src/cmdeploy/__init__.py @@ -481,8 +481,14 @@ def deploy_mtail(config): def deploy_iroh_relay(config) -> None: (url, sha256sum) = { - "x86_64": ("https://github.com/n0-computer/iroh/releases/download/v0.27.0/iroh-relay-v0.27.0-x86_64-unknown-linux-musl.tar.gz", "8af7f6d29d17476ce5c3053c3161db5793cb2ac49057d0bcaf689436cdccbeab"), - "aarch64": ("https://github.com/n0-computer/iroh/releases/download/v0.27.0/iroh-relay-v0.27.0-aarch64-unknown-linux-musl.tar.gz", "18039f0d39df78922a5055a0d4a5a8fa98a2a0e19b1eaa4c3fe6db73b8698697") + "x86_64": ( + "https://github.com/n0-computer/iroh/releases/download/v0.28.1/iroh-relay-v0.28.1-x86_64-unknown-linux-musl.tar.gz", + "2ffacf7c0622c26b67a5895ee8e07388769599f60e5f52a3bd40a3258db89b2c", + ), + "aarch64": ( + "https://github.com/n0-computer/iroh/releases/download/v0.28.1/iroh-relay-v0.28.1-aarch64-unknown-linux-musl.tar.gz", + "b915037bcc1ff1110cc9fcb5de4a17c00ff576fd2f568cd339b3b2d54c420dc4", + ), }[host.get_fact(facts.server.Arch)] apt.packages( @@ -493,7 +499,7 @@ def deploy_iroh_relay(config) -> None: server.shell( name="Download iroh-relay", commands=[ - f"(echo '{sha256sum} /usr/local/bin/iroh-relay' | sha256sum -c) || curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay", + f"(echo '{sha256sum} /usr/local/bin/iroh-relay' | sha256sum -c) || (curl -L {url} | gunzip | tar -x -f - ./iroh-relay -O >/usr/local/bin/iroh-relay.new && mv /usr/local/bin/iroh-relay.new /usr/local/bin/iroh-relay)", "chmod 755 /usr/local/bin/iroh-relay", ], ) @@ -502,9 +508,7 @@ def deploy_iroh_relay(config) -> None: systemd_unit = files.put( name="Upload iroh-relay systemd unit", - src=importlib.resources.files(__package__).joinpath( - "iroh-relay.service" - ), + src=importlib.resources.files(__package__).joinpath("iroh-relay.service"), dest="/etc/systemd/system/iroh-relay.service", user="root", group="root", @@ -514,13 +518,11 @@ def deploy_iroh_relay(config) -> None: iroh_config = files.put( name=f"Upload iroh-relay config", - src=importlib.resources.files(__package__).joinpath( - "iroh-relay.toml" - ), + src=importlib.resources.files(__package__).joinpath("iroh-relay.toml"), dest=f"/etc/iroh-relay.toml", - user="iroh", - group="iroh", - mode="600", + user="root", + group="root", + mode="644", ) need_restart |= iroh_config.changed @@ -533,12 +535,11 @@ def deploy_iroh_relay(config) -> None: ) -def deploy_chatmail(config_path: Path, disable_mail: bool, require_iroh: bool) -> None: +def deploy_chatmail(config_path: Path, disable_mail: bool) -> None: """Deploy a chat-mail instance. :param config_path: path to chatmail.ini :param disable_mail: whether to disable postfix & dovecot - :param require_iroh: whether to request a TLS certificate for iroh.$mail_domain """ config = read_config(config_path) check_config(config) @@ -616,8 +617,6 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, require_iroh: bool) - # Deploy acmetool to have TLS certificates. tls_domains = [mail_domain, f"mta-sts.{mail_domain}", f"www.{mail_domain}"] - if require_iroh: - tls_domains.append(f"iroh.{mail_domain}") deploy_acmetool( domains=tls_domains, ) diff --git a/cmdeploy/src/cmdeploy/chatmail.zone.j2 b/cmdeploy/src/cmdeploy/chatmail.zone.j2 index 10b07758..f0ba176f 100644 --- a/cmdeploy/src/cmdeploy/chatmail.zone.j2 +++ b/cmdeploy/src/cmdeploy/chatmail.zone.j2 @@ -11,7 +11,6 @@ _mta-sts.{{ mail_domain }}. TXT "v=STSv1; id={{ sts_id }}" mta-sts.{{ mail_domain }}. CNAME {{ mail_domain }}. www.{{ mail_domain }}. CNAME {{ mail_domain }}. -iroh.{{ mail_domain }}. CNAME {{ mail_domain }}. {{ dkim_entry }} ; diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index fe7970dc..b36b29f0 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -70,8 +70,8 @@ def run_cmd(args, out): sshexec = args.get_sshexec() require_iroh = args.config.enable_iroh_relay - remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain, require_iroh) - if not dns.check_initial_remote_data(remote_data, require_iroh, print=out.red): + remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain) + if not dns.check_initial_remote_data(remote_data, print=out.red): return 1 env = os.environ.copy() @@ -111,8 +111,7 @@ def dns_cmd_options(parser): def dns_cmd(args, out): """Check DNS entries and optionally generate dns zone file.""" sshexec = args.get_sshexec() - require_iroh = args.config.enable_iroh_relay - remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain, require_iroh) + remote_data = dns.get_initial_remote_data(sshexec, args.config.mail_domain) if not remote_data: return 1 diff --git a/cmdeploy/src/cmdeploy/deploy.py b/cmdeploy/src/cmdeploy/deploy.py index 9c8e7400..0ea153d7 100644 --- a/cmdeploy/src/cmdeploy/deploy.py +++ b/cmdeploy/src/cmdeploy/deploy.py @@ -12,9 +12,8 @@ def main(): importlib.resources.files("cmdeploy").joinpath("../../../chatmail.ini"), ) disable_mail = bool(os.environ.get('CHATMAIL_DISABLE_MAIL')) - require_iroh = bool(os.environ.get('CHATMAIL_REQUIRE_IROH')) - deploy_chatmail(config_path, disable_mail, require_iroh) + deploy_chatmail(config_path, disable_mail) if pyinfra.is_cli: diff --git a/cmdeploy/src/cmdeploy/dns.py b/cmdeploy/src/cmdeploy/dns.py index b8e05f21..d3c541bb 100644 --- a/cmdeploy/src/cmdeploy/dns.py +++ b/cmdeploy/src/cmdeploy/dns.py @@ -6,22 +6,19 @@ from . import remote -def get_initial_remote_data(sshexec, mail_domain, iroh_enabled): +def get_initial_remote_data(sshexec, mail_domain): return sshexec.logged( - call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain, iroh_enabled=iroh_enabled) + call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain) ) -def check_initial_remote_data(remote_data, require_iroh, *, print=print): +def check_initial_remote_data(remote_data, *, print=print): mail_domain = remote_data["mail_domain"] if not remote_data["A"] and not remote_data["AAAA"]: print(f"Missing A and/or AAAA DNS records for {mail_domain}!") elif remote_data["MTA_STS"] != f"{mail_domain}.": print("Missing MTA-STS CNAME record:") print(f"mta-sts.{mail_domain}. CNAME {mail_domain}.") - elif require_iroh and remote_data["IROH"] != f"{mail_domain}.": - print("Missing iroh CNAME record:") - print(f"iroh.{mail_domain}. CNAME {mail_domain}.") elif remote_data["WWW"] != f"{mail_domain}.": print("Missing www CNAME record:") print(f"www.{mail_domain}. CNAME {mail_domain}.") diff --git a/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 b/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 index 5797b4c0..6cda5f1b 100644 --- a/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 +++ b/cmdeploy/src/cmdeploy/nginx/nginx.conf.j2 @@ -96,6 +96,26 @@ http { include /etc/nginx/fastcgi_params; fastcgi_param SCRIPT_FILENAME /usr/lib/cgi-bin/newemail.py; } + + # Proxy to iroh-relay service. + location /relay { + proxy_pass http://127.0.0.1:3340; + proxy_http_version 1.1; + + # Upgrade header is normally set to "iroh derp http" or "websocket". + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + location /relay/probe { + proxy_pass http://127.0.0.1:3340; + proxy_http_version 1.1; + } + + location /generate_204 { + proxy_pass http://127.0.0.1:3340; + proxy_http_version 1.1; + } } # Redirect www. to non-www @@ -108,16 +128,4 @@ http { return 301 $scheme://{{ config.domain_name }}$request_uri; access_log syslog:server=unix:/dev/log,facility=local7; } - - # Pass iroh. to iroh-relay service. - server { - listen 8443 ssl; - {% if not disable_ipv6 %} - listen [::]:8443 ssl; - {% endif %} - server_name iroh.{{ config.domain_name }}; - location / { - proxy_pass http://127.0.0.1:3340; - } - } } diff --git a/cmdeploy/src/cmdeploy/remote/rdns.py b/cmdeploy/src/cmdeploy/remote/rdns.py index 107d7d23..77093503 100644 --- a/cmdeploy/src/cmdeploy/remote/rdns.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -15,7 +15,7 @@ from .rshell import CalledProcessError, shell -def perform_initial_checks(mail_domain, iroh_enabled): +def perform_initial_checks(mail_domain): """Collecting initial DNS settings.""" assert mail_domain if not shell("dig", fail_ok=True): @@ -23,14 +23,13 @@ def perform_initial_checks(mail_domain, iroh_enabled): A = query_dns("A", mail_domain) AAAA = query_dns("AAAA", mail_domain) MTA_STS = query_dns("CNAME", f"mta-sts.{mail_domain}") - IROH = query_dns("CNAME", f"iroh.{mail_domain}") WWW = query_dns("CNAME", f"www.{mail_domain}") - res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, IROH=IROH, WWW=WWW) + res = dict(mail_domain=mail_domain, A=A, AAAA=AAAA, MTA_STS=MTA_STS, WWW=WWW) res["acme_account_url"] = shell("acmetool account-url", fail_ok=True) res["dkim_entry"] = get_dkim_entry(mail_domain, dkim_selector="opendkim") - if not MTA_STS or (not IROH and not iroh_enabled) or not WWW or (not A and not AAAA): + if not MTA_STS or not WWW or (not A and not AAAA): return res # parse out sts-id if exists, example: "v=STSv1; id=2090123" diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index 13424ef3..f5f2c023 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -18,13 +18,13 @@ def test_ls(self, sshexec): def test_perform_initial(self, sshexec, maildomain): res = sshexec( - remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain, iroh_enabled=True) + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) ) assert res["A"] or res["AAAA"] def test_logged(self, sshexec, maildomain, capsys): sshexec.logged( - remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain, iroh_enabled=True) + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) ) out, err = capsys.readouterr() assert err.startswith("Collecting") @@ -33,7 +33,7 @@ def test_logged(self, sshexec, maildomain, capsys): sshexec.verbose = True sshexec.logged( - remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain, iroh_enabled=True) + remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) ) out, err = capsys.readouterr() lines = err.split("\n") @@ -44,7 +44,7 @@ def test_exception(self, sshexec, capsys): try: sshexec.logged( remote.rdns.perform_initial_checks, - kwargs=dict(mail_domain=None, iroh_enabled=True), + kwargs=dict(mail_domain=None), ) except sshexec.FuncError as e: assert "rdns.py" in str(e) diff --git a/cmdeploy/src/cmdeploy/tests/test_dns.py b/cmdeploy/src/cmdeploy/tests/test_dns.py index a69a7f73..fd11095f 100644 --- a/cmdeploy/src/cmdeploy/tests/test_dns.py +++ b/cmdeploy/src/cmdeploy/tests/test_dns.py @@ -26,7 +26,6 @@ def mockdns(mockdns_base): "AAAA": {"some.domain": "fde5:cd7a:9e1c:3240:5a99:936f:cdac:53ae"}, "CNAME": { "mta-sts.some.domain": "some.domain.", - "iroh.some.domain": "some.domain.", "www.some.domain": "some.domain.", }, } @@ -36,31 +35,30 @@ def mockdns(mockdns_base): class TestPerformInitialChecks: def test_perform_initial_checks_ok1(self, mockdns): - remote_data = remote.rdns.perform_initial_checks("some.domain", iroh_enabled=True) + remote_data = remote.rdns.perform_initial_checks("some.domain") assert remote_data["A"] == mockdns["A"]["some.domain"] assert remote_data["AAAA"] == mockdns["AAAA"]["some.domain"] assert remote_data["MTA_STS"] == mockdns["CNAME"]["mta-sts.some.domain"] - assert remote_data["IROH"] == mockdns["CNAME"]["iroh.some.domain"] assert remote_data["WWW"] == mockdns["CNAME"]["www.some.domain"] @pytest.mark.parametrize("drop", ["A", "AAAA"]) def test_perform_initial_checks_with_one_of_A_AAAA(self, mockdns, drop): del mockdns[drop] - remote_data = remote.rdns.perform_initial_checks("some.domain", iroh_enabled=True) + remote_data = remote.rdns.perform_initial_checks("some.domain") assert not remote_data[drop] l = [] - res = check_initial_remote_data(remote_data, require_iroh=True, print=l.append) + res = check_initial_remote_data(remote_data, print=l.append) assert res assert not l def test_perform_initial_checks_no_mta_sts(self, mockdns): del mockdns["CNAME"]["mta-sts.some.domain"] - remote_data = remote.rdns.perform_initial_checks("some.domain", iroh_enabled=True) + remote_data = remote.rdns.perform_initial_checks("some.domain") assert not remote_data["MTA_STS"] l = [] - res = check_initial_remote_data(remote_data, require_iroh=True, print=l.append) + res = check_initial_remote_data(remote_data, print=l.append) assert not res assert len(l) == 2 From b268efbc6e2916ec54a47714bf16a6976009ab02 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Dec 2024 17:28:58 +0100 Subject: [PATCH 53/66] DNS: fix _mta-sts TXT record on initial setup --- cmdeploy/src/cmdeploy/remote/rdns.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/remote/rdns.py b/cmdeploy/src/cmdeploy/remote/rdns.py index 77093503..7da689df 100644 --- a/cmdeploy/src/cmdeploy/remote/rdns.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -10,6 +10,7 @@ - can freely call each other. """ +import datetime import re from .rshell import CalledProcessError, shell @@ -34,7 +35,10 @@ def perform_initial_checks(mail_domain): # parse out sts-id if exists, example: "v=STSv1; id=2090123" parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=") - res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else "" + if len(parts) == 2 and len(parts[1]) > 1: + res["sts_id"] = parts[1].rstrip('"') + else: + res["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M") return res From 46f6a07239afb85a5cc93181db5bf9b641927301 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Dec 2024 17:43:03 +0100 Subject: [PATCH 54/66] Revert "DNS: fix _mta-sts TXT record on initial setup" This reverts commit 6d4af3cf0c0b63cc75ab0a8220a131330b3a7aad. --- cmdeploy/src/cmdeploy/remote/rdns.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmdeploy/src/cmdeploy/remote/rdns.py b/cmdeploy/src/cmdeploy/remote/rdns.py index 7da689df..77093503 100644 --- a/cmdeploy/src/cmdeploy/remote/rdns.py +++ b/cmdeploy/src/cmdeploy/remote/rdns.py @@ -10,7 +10,6 @@ - can freely call each other. """ -import datetime import re from .rshell import CalledProcessError, shell @@ -35,10 +34,7 @@ def perform_initial_checks(mail_domain): # parse out sts-id if exists, example: "v=STSv1; id=2090123" parts = query_dns("TXT", f"_mta-sts.{mail_domain}").split("id=") - if len(parts) == 2 and len(parts[1]) > 1: - res["sts_id"] = parts[1].rstrip('"') - else: - res["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M") + res["sts_id"] = parts[1].rstrip('"') if len(parts) == 2 else "" return res From 69fe5eac2b0f2cd70a9a0dc957145bd7f1cb7159 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Dec 2024 17:43:22 +0100 Subject: [PATCH 55/66] DNS: more elegant solution to fix mta-sts record --- cmdeploy/src/cmdeploy/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/dns.py b/cmdeploy/src/cmdeploy/dns.py index d3c541bb..a0606836 100644 --- a/cmdeploy/src/cmdeploy/dns.py +++ b/cmdeploy/src/cmdeploy/dns.py @@ -29,7 +29,7 @@ def check_initial_remote_data(remote_data, *, print=print): def get_filled_zone_file(remote_data): sts_id = remote_data.get("sts_id") if not sts_id: - sts_id = datetime.datetime.now().strftime("%Y%m%d%H%M") + remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M") template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2") content = template.read_text() From 8e5174ae44dca0e512c2937ffcd2778fe704dea2 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Dec 2024 20:33:07 +0100 Subject: [PATCH 56/66] DNS: add -all to cmdeploy dns --- cmdeploy/src/cmdeploy/cmdeploy.py | 8 +++++++- cmdeploy/src/cmdeploy/dns.py | 14 ++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index b36b29f0..e53585ac 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -106,6 +106,12 @@ def dns_cmd_options(parser): default=None, help="write out a zonefile", ) + parser.add_argument( + "--all", + dest="all", + action="store_true", + help="check both required and recommended DNS records" + ) def dns_cmd(args, out): @@ -131,7 +137,7 @@ def dns_cmd(args, out): return 0 retcode = dns.check_full_zone( - sshexec, remote_data=remote_data, zonefile=zonefile, out=out + sshexec, remote_data=remote_data, zonefile=zonefile, out=out, all=args.all ) return retcode diff --git a/cmdeploy/src/cmdeploy/dns.py b/cmdeploy/src/cmdeploy/dns.py index a0606836..edf23067 100644 --- a/cmdeploy/src/cmdeploy/dns.py +++ b/cmdeploy/src/cmdeploy/dns.py @@ -40,7 +40,7 @@ def get_filled_zone_file(remote_data): return zonefile -def check_full_zone(sshexec, remote_data, out, zonefile) -> int: +def check_full_zone(sshexec, remote_data, out, zonefile, all) -> int: """Check existing DNS records, optionally write them to zone file and return (exitcode, remote_data) tuple.""" @@ -49,16 +49,18 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int: kwargs=dict(zonefile=zonefile, mail_domain=remote_data["mail_domain"]), ) + returncode = 0 if required_diff: out.red("Please set required DNS entries at your DNS provider:\n") for line in required_diff: out(line) - return 1 - elif recommended_diff: + print() + returncode += 1 + if recommended_diff and (all or not required_diff): out("WARNING: these recommended DNS entries are not set:\n") for line in recommended_diff: out(line) - return 0 - out.green("Great! All your DNS entries are verified and correct.") - return 0 + if not (recommended_diff or required_diff): + out.green("Great! All your DNS entries are verified and correct.") + return returncode From 08c88caa46f625c494cd02a591dcf7e6e5ecc5bd Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Dec 2024 22:06:09 +0100 Subject: [PATCH 57/66] CI: test all DNS records --- .github/workflows/test-and-deploy-ipv4only.yaml | 2 +- .github/workflows/test-and-deploy.yaml | 2 +- cmdeploy/src/cmdeploy/dns.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-and-deploy-ipv4only.yaml b/.github/workflows/test-and-deploy-ipv4only.yaml index 37d684fd..2f25b25a 100644 --- a/.github/workflows/test-and-deploy-ipv4only.yaml +++ b/.github/workflows/test-and-deploy-ipv4only.yaml @@ -96,5 +96,5 @@ jobs: run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow - name: cmdeploy dns - run: cmdeploy dns -v + run: cmdeploy dns -v --all diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index 53aa79a8..77fe556a 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -94,5 +94,5 @@ jobs: run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow - name: cmdeploy dns - run: cmdeploy dns -v + run: cmdeploy dns -v --all diff --git a/cmdeploy/src/cmdeploy/dns.py b/cmdeploy/src/cmdeploy/dns.py index edf23067..ead1c895 100644 --- a/cmdeploy/src/cmdeploy/dns.py +++ b/cmdeploy/src/cmdeploy/dns.py @@ -55,11 +55,13 @@ def check_full_zone(sshexec, remote_data, out, zonefile, all) -> int: for line in required_diff: out(line) print() - returncode += 1 + returncode = 1 if recommended_diff and (all or not required_diff): out("WARNING: these recommended DNS entries are not set:\n") for line in recommended_diff: out(line) + if all: + returncode = 1 if not (recommended_diff or required_diff): out.green("Great! All your DNS entries are verified and correct.") From 97c31e382053c8a6f46959adb052dd58a047dfba Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Dec 2024 23:16:45 +0100 Subject: [PATCH 58/66] fix tests --- cmdeploy/src/cmdeploy/tests/test_dns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmdeploy/src/cmdeploy/tests/test_dns.py b/cmdeploy/src/cmdeploy/tests/test_dns.py index fd11095f..a546faeb 100644 --- a/cmdeploy/src/cmdeploy/tests/test_dns.py +++ b/cmdeploy/src/cmdeploy/tests/test_dns.py @@ -110,7 +110,7 @@ def test_check_zonefile_output_required_fine(self, cm_data, mockdns_base, mockou parse_zonefile_into_dict(zonefile_mocked, mockdns_base, only_required=True) mssh = MockSSHExec() mockdns_base["mail_domain"] = "some.domain" - res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile) + res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile, all=False) assert res == 0 assert "WARNING" in mockout.captured_plain[0] assert len(mockout.captured_plain) == 9 @@ -120,7 +120,7 @@ def test_check_zonefile_output_full(self, cm_data, mockdns_base, mockout): parse_zonefile_into_dict(zonefile, mockdns_base) mssh = MockSSHExec() mockdns_base["mail_domain"] = "some.domain" - res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile) + res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile, all=True) assert res == 0 assert not mockout.captured_red assert "correct" in mockout.captured_green[0] From a2fbb5dc3749c4801b1e41aaf425583e214224be Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Dec 2024 23:18:06 +0100 Subject: [PATCH 59/66] add changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 097a8ad5..f8bc8e18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## untagged +- add `--all` to `cmdeploy dns` + ([#462](https://github.com/deltachat/chatmail/pull/462)) + +- fix `_mta-sts` TXT DNS record + ([#461](https://github.com/deltachat/chatmail/pull/461) + - deploy `iroh-relay` and also update "realtime relay services" in privacy policy. ([#434](https://github.com/deltachat/chatmail/pull/434)) ([#451](https://github.com/deltachat/chatmail/pull/451)) From 88a8dc905baf69f62462931cf117f23c58a11595 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Dec 2024 23:23:26 +0100 Subject: [PATCH 60/66] DNS: recommend cmdeploy dns --all in the README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eaf00981..851084d4 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ scripts/cmdeploy status To display and check all recommended DNS records: ``` -scripts/cmdeploy dns +scripts/cmdeploy dns --all ``` To test whether your chatmail service is working correctly: From d11038b7b33e0630b0dc115655b1aa49608aa9ec Mon Sep 17 00:00:00 2001 From: missytake Date: Thu, 19 Dec 2024 14:17:37 +0100 Subject: [PATCH 61/66] DNS: out() instead of print() --- cmdeploy/src/cmdeploy/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/dns.py b/cmdeploy/src/cmdeploy/dns.py index ead1c895..552d96ba 100644 --- a/cmdeploy/src/cmdeploy/dns.py +++ b/cmdeploy/src/cmdeploy/dns.py @@ -54,7 +54,7 @@ def check_full_zone(sshexec, remote_data, out, zonefile, all) -> int: out.red("Please set required DNS entries at your DNS provider:\n") for line in required_diff: out(line) - print() + out("") returncode = 1 if recommended_diff and (all or not required_diff): out("WARNING: these recommended DNS entries are not set:\n") From a7b808ebafa8aa362e2e8334b715b3bcc9b77da1 Mon Sep 17 00:00:00 2001 From: missytake Date: Fri, 20 Dec 2024 10:53:36 +0100 Subject: [PATCH 62/66] Release 1.5.0 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8bc8e18..3a26c7ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## untagged +## 1.5.0 2024-12-20 + - add `--all` to `cmdeploy dns` ([#462](https://github.com/deltachat/chatmail/pull/462)) From 5c78619750a4169385678c81f6a4d325b9c84900 Mon Sep 17 00:00:00 2001 From: missytake Date: Fri, 20 Dec 2024 23:28:54 +0100 Subject: [PATCH 63/66] DNS: make --all non-optional for cmdeploy dns --- cmdeploy/src/cmdeploy/cmdeploy.py | 8 +------- cmdeploy/src/cmdeploy/dns.py | 6 ++---- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index e53585ac..b36b29f0 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -106,12 +106,6 @@ def dns_cmd_options(parser): default=None, help="write out a zonefile", ) - parser.add_argument( - "--all", - dest="all", - action="store_true", - help="check both required and recommended DNS records" - ) def dns_cmd(args, out): @@ -137,7 +131,7 @@ def dns_cmd(args, out): return 0 retcode = dns.check_full_zone( - sshexec, remote_data=remote_data, zonefile=zonefile, out=out, all=args.all + sshexec, remote_data=remote_data, zonefile=zonefile, out=out ) return retcode diff --git a/cmdeploy/src/cmdeploy/dns.py b/cmdeploy/src/cmdeploy/dns.py index 552d96ba..663ae66c 100644 --- a/cmdeploy/src/cmdeploy/dns.py +++ b/cmdeploy/src/cmdeploy/dns.py @@ -40,7 +40,7 @@ def get_filled_zone_file(remote_data): return zonefile -def check_full_zone(sshexec, remote_data, out, zonefile, all) -> int: +def check_full_zone(sshexec, remote_data, out, zonefile) -> int: """Check existing DNS records, optionally write them to zone file and return (exitcode, remote_data) tuple.""" @@ -56,12 +56,10 @@ def check_full_zone(sshexec, remote_data, out, zonefile, all) -> int: out(line) out("") returncode = 1 - if recommended_diff and (all or not required_diff): + if recommended_diff: out("WARNING: these recommended DNS entries are not set:\n") for line in recommended_diff: out(line) - if all: - returncode = 1 if not (recommended_diff or required_diff): out.green("Great! All your DNS entries are verified and correct.") From 6a32192e50ed2791bf9788f56742ef14c73fc027 Mon Sep 17 00:00:00 2001 From: missytake Date: Fri, 20 Dec 2024 23:30:53 +0100 Subject: [PATCH 64/66] Revert rest of #462 This reverts commit 88a8dc905baf69f62462931cf117f23c58a11595. --- .github/workflows/test-and-deploy-ipv4only.yaml | 2 +- .github/workflows/test-and-deploy.yaml | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-and-deploy-ipv4only.yaml b/.github/workflows/test-and-deploy-ipv4only.yaml index 2f25b25a..37d684fd 100644 --- a/.github/workflows/test-and-deploy-ipv4only.yaml +++ b/.github/workflows/test-and-deploy-ipv4only.yaml @@ -96,5 +96,5 @@ jobs: run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow - name: cmdeploy dns - run: cmdeploy dns -v --all + run: cmdeploy dns -v diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index 77fe556a..53aa79a8 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -94,5 +94,5 @@ jobs: run: CHATMAIL_DOMAIN2=nine.testrun.org cmdeploy test --slow - name: cmdeploy dns - run: cmdeploy dns -v --all + run: cmdeploy dns -v diff --git a/README.md b/README.md index 851084d4..eaf00981 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ scripts/cmdeploy status To display and check all recommended DNS records: ``` -scripts/cmdeploy dns --all +scripts/cmdeploy dns ``` To test whether your chatmail service is working correctly: From d6205d9a0447945296d709074f8ff71e071aadcf Mon Sep 17 00:00:00 2001 From: missytake Date: Fri, 20 Dec 2024 23:33:59 +0100 Subject: [PATCH 65/66] add changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a26c7ee..f4bba606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ## 1.5.0 2024-12-20 +- cmdeploy dns: always show recommended DNS records + ([#463](https://github.com/deltachat/chatmail/pull/463)) + - add `--all` to `cmdeploy dns` ([#462](https://github.com/deltachat/chatmail/pull/462)) From 5b8de76c228ac9159caa91762e2aeaee44c9dba4 Mon Sep 17 00:00:00 2001 From: missytake Date: Fri, 20 Dec 2024 23:36:52 +0100 Subject: [PATCH 66/66] fix tests --- cmdeploy/src/cmdeploy/tests/test_dns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmdeploy/src/cmdeploy/tests/test_dns.py b/cmdeploy/src/cmdeploy/tests/test_dns.py index a546faeb..fd11095f 100644 --- a/cmdeploy/src/cmdeploy/tests/test_dns.py +++ b/cmdeploy/src/cmdeploy/tests/test_dns.py @@ -110,7 +110,7 @@ def test_check_zonefile_output_required_fine(self, cm_data, mockdns_base, mockou parse_zonefile_into_dict(zonefile_mocked, mockdns_base, only_required=True) mssh = MockSSHExec() mockdns_base["mail_domain"] = "some.domain" - res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile, all=False) + res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile) assert res == 0 assert "WARNING" in mockout.captured_plain[0] assert len(mockout.captured_plain) == 9 @@ -120,7 +120,7 @@ def test_check_zonefile_output_full(self, cm_data, mockdns_base, mockout): parse_zonefile_into_dict(zonefile, mockdns_base) mssh = MockSSHExec() mockdns_base["mail_domain"] = "some.domain" - res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile, all=True) + res = check_full_zone(mssh, mockdns_base, out=mockout, zonefile=zonefile) assert res == 0 assert not mockout.captured_red assert "correct" in mockout.captured_green[0]