Revert to python3-lxml for compat

addons/mail/models/mail_mail.py

@@ -16,7 +16,8 @@
 
 from odoo import _, api, fields, models
 from odoo import tools
-from odoo.addons.base.models.ir_mail_server import MailDeliveryException
+from odoo.addons.base.models.ir_mail_server import MailDeliveryException, MailDeliveryWhitelistException
+from odoo.addons.base.models.ir_cron import db_whitelisted
 
 _logger = logging.getLogger(__name__)
 _UNFOLLOW_REGEX = re.compile(r'<span id="mail_unfollow".*?<\/span>', re.DOTALL)
@@ -551,10 +552,18 @@ def send(self, auto_commit=False, raise_exception=False):
                 email sending process has failed
             :return: True
         """
+
+        # Return out of send() if db not in whitelist
+        if not db_whitelisted(self.env.cr.dbname):
+            _logger.warning('Database cannot send emails as it is not on the whitelist.')
+            return
+
         for mail_server_id, alias_domain_id, smtp_from, batch_ids in self._split_by_mail_configuration():
             smtp_session = None
             try:
                 smtp_session = self.env['ir.mail_server'].connect(mail_server_id=mail_server_id, smtp_from=smtp_from)
+            except MailDeliveryWhitelistException:
+                pass
             except Exception as exc:
                 if raise_exception:
                     # To be consistent and backward compatible with mail_mail.send() raised

addons/mail/models/mail_template.py

@@ -11,6 +11,8 @@
 from odoo.tools import is_html_empty
 from odoo.tools.safe_eval import safe_eval, time
 
+from odoo.addons.base.models.ir_cron import db_whitelisted
+
 _logger = logging.getLogger(__name__)
 
 
@@ -596,6 +598,10 @@ def send_mail(self, res_id, force_send=False, raise_exception=False, email_value
 
         Attachment = self.env['ir.attachment']  # TDE FIXME: should remove default_type from context
 
+        if force_send and raise_exception and not db_whitelisted(self.env.cr.dbname):
+            # Allows auto functions, like create users, to continue without failing
+            raise_exception = False
+
         # create a mail_mail based on values, without attachments
         values = self._generate_template(
             [res_id],

debian/control

@@ -35,7 +35,8 @@ Depends:
  python3-jinja2,
  python3-libsass,
 # After lxml 5.2, lxml-html-clean is in a separate package
- python3-lxml-html-clean | python3-lxml,
+# WilldooIT Patch: reverting to original package for compatibility
+ python3-lxml,
  python3-num2words,
  python3-ofxparse,
  python3-passlib,

odoo/addons/base/models/ir_cron.py

@@ -9,6 +9,9 @@
 from dateutil.relativedelta import relativedelta
 from psycopg2 import sql
 
+import json
+import re
+
 import odoo
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
@@ -22,6 +25,16 @@
 ODOO_NOTIFY_FUNCTION = os.getenv('ODOO_NOTIFY_FUNCTION', 'pg_notify')
 
 
+def db_whitelisted(db_name):
+    cron_whitelist = odoo.tools.config.get("db_cron_whitelist") and json.loads(odoo.tools.config["db_cron_whitelist"]) or []
+    if db_name not in cron_whitelist:
+        for cw_name in cron_whitelist:
+            if re.match(cw_name, db_name):
+                break
+        else:
+            return False
+    return True
+
 class BadVersion(Exception):
     pass
 
@@ -112,6 +125,10 @@ def method_direct_trigger(self):
 
     @classmethod
     def _process_jobs(cls, db_name):
+
+        if not db_whitelisted(db_name):
+            return False
+
         """ Execute every job ready to be run on this database. """
         try:
             db = odoo.sql_db.db_connect(db_name)

odoo/addons/base/models/ir_mail_server.py

@@ -14,17 +14,22 @@
 import ssl
 import sys
 import threading
+import os
 
 from socket import gaierror, timeout
 from OpenSSL import crypto as SSLCrypto
 from OpenSSL.crypto import Error as SSLCryptoError, FILETYPE_PEM
 from OpenSSL.SSL import Error as SSLError
 from urllib3.contrib.pyopenssl import PyOpenSSLContext
+from OpenSSL.SSL import Context as SSLContext, Error as SSLError
+from unittest.mock import MagicMock
 
 from odoo import api, fields, models, tools, _
 from odoo.exceptions import UserError
 from odoo.tools import ustr, pycompat, formataddr, email_normalize, encapsulate_email, email_domain_extract, email_domain_normalize
 
+from odoo.addons.base.models.ir_cron import db_whitelisted
+
 
 _logger = logging.getLogger(__name__)
 _test_logger = logging.getLogger('odoo.tests')
@@ -36,6 +41,10 @@ class MailDeliveryException(Exception):
     """Specific exception subclass for mail delivery errors"""
 
 
+class MailDeliveryWhitelistException(MailDeliveryException):
+    """Specific exception subclass for non whitelisted mail delivery attempts"""
+
+
 def make_wrap_property(name):
     return property(
         lambda self: getattr(self.__obj__, name),
@@ -413,6 +422,8 @@ def connect(self, host=None, port=None, user=None, password=None, encryption=Non
                  _("Please define at least one SMTP server, "
                    "or provide the SMTP parameters explicitly.")))
 
+        self._is_allowed_to_send(smtp_server, raise_exception=True)
+
         if smtp_encryption == 'ssl':
             if 'SMTP_SSL' not in smtplib.__all__:
                 raise UserError(
@@ -719,6 +730,8 @@ def send_email(self, message, mail_server_id=None, smtp_server=None, smtp_port=N
                 smtp.quit()
         except smtplib.SMTPServerDisconnected:
             raise
+        except MailDeliveryWhitelistException:
+            raise
         except Exception as e:
             params = (ustr(smtp_server), e.__class__.__name__, ustr(e))
             msg = _("Mail delivery failed via SMTP server '%s'.\n%s: %s", *params)
@@ -837,3 +850,45 @@ def _is_test_mode(self):
         outgoing mail server.
         """
         return getattr(threading.current_thread(), 'testing', False) or self.env.registry.in_test_mode()
+
+    def _is_allowed_to_send(self, smtp_server: str = None, raise_exception: bool = False) -> bool:
+        """
+        Return True if the database is allowed to send email.
+
+        Emails are not allowed unless the database is whitelisted by using the
+        `db_cron_whitelist` option in the odoo config file.
+
+        During development, we don't want to enable this option, and we do not want
+        to send emails to real users, but we do want to test emails using local
+        development mail servers such as MailHog.
+
+        To avoid accidentally allowing real local email servers such as postfix
+        to send emails, the mail server must be explicitly configured with the
+        environment variable `ODOO_DEV_SMTP_SERVER` if the database is not
+        whitelisted.
+        """
+
+        if db_whitelisted(self.env.cr.dbname):
+            return True
+
+        # Some odoo tests in base explicitly patch ir.mail_server to return False from
+        # _is_test_mode() in which case we want to allow sending mails.
+        if isinstance(self.connect, MagicMock):
+            return True
+
+        dev_smtp_server = os.environ.get("ODOO_DEV_SMTP_SERVER")
+        if dev_smtp_server in ["localhost", "127.0.0.1"] and (
+            (smtp_server and smtp_server == dev_smtp_server)
+            or
+            (len(self) <= 1 and self.smtp_host == dev_smtp_server)
+        ):
+            _logger.info("Allowing local development SMTP server: %s", smtp_server)
+            return True
+
+        msg = _("Database cannot send emails as it is not on the whitelist.")
+        _logger.warning(msg)
+
+        if raise_exception:
+            raise MailDeliveryWhitelistException(msg)
+
+        return False

odoo/http.py

@@ -340,6 +340,26 @@ def db_filter(dbs, host=None):
     :rtype: List[str]
     """
 
+    if config.get('db_filter_multi'):
+        host_m = host
+
+        if host_m is None:
+            host_m = request.httprequest.environ.get('HTTP_HOST', '')
+        host_m = host_m.partition(':')[0]
+        if host_m.startswith('www.'):
+            host_m = host_m[4:]
+        domain = host_m.partition('.')[0]
+
+        db_dict = json.loads(config["db_filter_multi"])
+        if isinstance(db_dict, dict) and host_m in db_dict:
+            ndbs = []
+            for dbfilter in db_dict[host_m]:
+                dbfilter_re = re.compile(
+                    dbfilter.replace("%h", re.escape(host_m))
+                    .replace("%d", re.escape(domain)))
+                ndbs.extend([db for db in dbs if dbfilter_re.match(db)])
+            return sorted(list(set(ndbs)))
+
     if config['dbfilter']:
         #        host
         #     -----------

odoo/modules/graph.py

@@ -21,6 +21,11 @@ def _ignored_modules(cr):
         result += [m[0] for m in cr.fetchall()]
     return result
 
+
+def check_package_delayed(info):
+    return int(info.get('load_priority', 0)) > 0
+
+
 class Graph(dict):
     """ Modules dependency graph.
 
@@ -30,8 +35,17 @@ class Graph(dict):
 
     def add_node(self, name, info):
         max_depth, father = 0, None
+
+        package_delayed = check_package_delayed(info)
+
         for d in info['depends']:
             n = self.get(d) or Node(d, self, None)  # lazy creation, do not use default value for get()
+
+            if package_delayed:
+                deepest_nodes = n.deepest_nodes()
+                if deepest_nodes:
+                    n = deepest_nodes[-1]
+
             if n.depth >= max_depth:
                 father = n
                 max_depth = n.depth
@@ -77,12 +91,30 @@ def add_modules(self, cr, module_list, force=None):
         dependencies = dict([(p, info['depends']) for p, info in packages])
         current, later = set([p for p, info in packages]), set()
 
+        delayed_packages = []
+        do_delayed_package = False
+
         while packages and current > later:
             package, info = packages[0]
             deps = info['depends']
 
+            # Have we looped through all without adding anything?
+            if delayed_packages and package == delayed_packages[0][0]:
+                delayed_packages.sort(key=lambda p: int(p[1]['load_priority']))
+                do_delayed_package = delayed_packages[0][0]
+                delayed_packages = []
+
             # if all dependencies of 'package' are already in the graph, add 'package' in the graph
             if all(dep in self for dep in deps):
+                if check_package_delayed(info) and package != do_delayed_package:
+                    delayed_packages.append((package, info))
+                    packages.append((package, info))
+                    packages.pop(0)
+                    continue
+
+                delayed_packages = []
+                do_delayed_package = False
+
                 if not package in current:
                     packages.pop(0)
                     continue
@@ -101,7 +133,7 @@ def add_modules(self, cr, module_list, force=None):
 
         for package in later:
             unmet_deps = [p for p in dependencies[package] if p not in self]
-            _logger.info('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps))
+            _logger.warning('module %s: Unmet dependencies: %s', package, ', '.join(unmet_deps))
 
         return len(self) - len_graph
 
@@ -159,6 +191,15 @@ def add_child(self, name, info):
         self.children.sort(key=lambda x: x.name)
         return node
 
+    def deepest_nodes(self):
+        next_level = [self]
+        while next_level:
+            last_level = next_level
+            next_level = []
+            for node in last_level:
+                next_level.extend(node.children)
+        return last_level
+
     def __setattr__(self, name, value):
         super(Node, self).__setattr__(name, value)
         if name in ('init', 'update', 'demo'):

odoo/modules/loading.py

@@ -182,6 +182,29 @@ def load_module_graph(env, graph, status=None, perform_checks=True,
             if package.name != 'base':
                 env.flush_all()
 
+        # Before Loading, check if any other modules have a "pre-install" test to be run
+        loader = odoo.tests.loader
+        updating = tools.config.options['init'] or tools.config.options['update']
+        test_results = None
+        if tools.config.options['test_enable'] and (needs_update or not updating):
+            module_names = (sorted(registry._init_modules))
+            for test_module_name in module_names:
+                preinstalls = loader.find_pre_install_tests(test_module_name)
+                if module_name in preinstalls:
+                    _logger.info("Starting pre install tests")
+                    tests_before = registry._assertion_report.testsRun
+                    tests_t0, tests_q0 = time.time(), odoo.sql_db.sql_counter
+                    with odoo.api.Environment.manage():
+                        result = loader.run_suite(loader.make_suite(test_module_name, 'pre_install_%s' % module_name), test_module_name)
+                        registry._assertion_report.update(result)
+                    _logger.info(
+                        "%d pre-install-tests in %.2fs, %s queries",
+                        registry._assertion_report.testsRun - tests_before,
+                        time.time() - tests_t0,
+                        odoo.sql_db.sql_counter - tests_q0)
+
+        # End of pre-install tests
+
         load_openerp_module(package.name)
 
         if new_install:
@@ -288,6 +311,18 @@ def load_module_graph(env, graph, status=None, perform_checks=True,
                 # tests may have reset the environment
                 module = env['ir.module.module'].browse(module_id)
 
+            # If this module has pre-install tests, but that module is already installed,
+            # then we have either a circular reference, or need a load_priority in the
+            # manifest
+
+            preinstalls = loader.find_pre_install_tests(module_name)
+            loaded_modules = (sorted(registry._init_modules))
+            if any(n in loaded_modules for n in preinstalls):
+                _logger.error(
+                    "Module %s: Preinstall test not run as module already installed",
+                    module_name
+                )
+
         if needs_update:
             processed_modules.append(package.name)
 

odoo/service/server.py

@@ -1330,6 +1330,15 @@ def preload_registries(dbnames):
                 module_names = (registry.updated_modules if update_module else
                                 sorted(registry._init_modules))
                 _logger.info("Starting post tests")
+
+                # Run pre_install tests which were missed because the module was never installed
+                for module_name in module_names:
+                    preinstalls = loader.find_pre_install_tests(module_name)
+                    for preinstall in preinstalls:
+                        if preinstall not in module_names:
+                            result = loader.run_suite(loader.make_suite(module_name, 'pre_install_%s' % preinstall), module_name)
+                            registry._assertion_report.update(result)
+
                 tests_before = registry._assertion_report.testsRun
                 post_install_suite = loader.make_suite(module_names, 'post_install')
                 if post_install_suite.has_http_case():

odoo/tests/loader.py

@@ -76,6 +76,19 @@ def make_suite(module_names, position='at_install'):
     return OdooSuite(sorted(tests, key=lambda t: t.test_sequence))
 
 
+def find_pre_install_tests(module_name):
+    mods = get_test_modules(module_name)
+    pre_installs = set()
+    for mod in mods:
+        for test in unwrap_suite(unittest.TestLoader().loadTestsFromModule(mod)):
+            for tag in test.test_tags if hasattr(test, 'test_tags') else []:
+                if tag.startswith('+'):
+                    tag = tag.replace('+', '')
+                if tag.startswith('pre_install_'):
+                    pre_installs.add(tag.replace('pre_install_', ''))
+    return pre_installs
+
+
 def run_suite(suite, module_name=None):
     # avoid dependency hell
     from ..modules import module