From bc65b0748324ea3a61e4d0ade848a451160590bd Mon Sep 17 00:00:00 2001 From: Franck Nijhof Date: Fri, 6 Oct 2023 20:19:08 +0200 Subject: [PATCH] Add Python 3.12, Drop Python 3.9 (#241) --- .github/workflows/builder.yml | 4 +- README.md | 10 +- python/{3.9 => 3.12}/Dockerfile | 4 +- python/{3.9 => 3.12}/arm-alignment.patch | 0 .../{3.9 => 3.12}/asynctio_unix_events.patch | 30 ++--- python/{3.9 => 3.12}/build.yaml | 4 +- ...o-optimize-to-add-remove-readers-and.patch | 113 ++++++++++++++++++ ...e-_BaseSelectorImpl._key_from_fd-wit.patch | 73 +++++++++++ ...ors-add-get-method-to-_SelectorMappi.patch | 43 +++++++ ...ors-optimize-EpollSelector.select-10.patch | 57 +++++++++ python/{3.9 => 3.12}/musl-find_library.patch | 0 python/3.9/fix-xattrs-glibc.patch | 16 --- 12 files changed, 312 insertions(+), 42 deletions(-) rename python/{3.9 => 3.12}/Dockerfile (97%) rename python/{3.9 => 3.12}/arm-alignment.patch (100%) rename python/{3.9 => 3.12}/asynctio_unix_events.patch (97%) rename python/{3.9 => 3.12}/build.yaml (88%) create mode 100644 python/3.12/gh-106527-asyncio-optimize-to-add-remove-readers-and.patch create mode 100644 python/3.12/gh-106554-replace-_BaseSelectorImpl._key_from_fd-wit.patch create mode 100644 python/3.12/gh-106664-selectors-add-get-method-to-_SelectorMappi.patch create mode 100644 python/3.12/gh-106751-selectors-optimize-EpollSelector.select-10.patch rename python/{3.9 => 3.12}/musl-find_library.patch (100%) delete mode 100644 python/3.9/fix-xattrs-glibc.patch diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index c191e64..2375d21 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -12,7 +12,7 @@ env: DEBIAN_LATEST: "bookworm" UBUNTU_LATEST: "20.4" RASPBIAN_LATEST: "bullseye" - PYTHON_LATEST: "3.11" + PYTHON_LATEST: "3.12" jobs: init: @@ -250,7 +250,7 @@ jobs: matrix: arch: ${{ fromJson(needs.init.outputs.architectures_alpine) }} version: ["3.16", "3.17", "3.18"] - python: ["3.9", "3.10", "3.11"] + python: ["3.10", "3.11", "3.12"] steps: - name: Checkout the repository uses: actions/checkout@v4.1.0 diff --git a/README.md b/README.md index a3401cc..04cf0a3 100644 --- a/README.md +++ b/README.md @@ -29,11 +29,11 @@ We support the latest 3 release with the latest 3 Alpine version. | Image | OS | Tags | latest | |-------|----|------|--------| -| armhf-base-python | Alpine | 3.9, 3.10, 3.11, 3.9-alpine3.16, 3.9-alpine3.17, 3.9-alpine3.18, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18 | 3.11-alpine3.18 | -| armv7-base-python | Alpine | 3.9, 3.10, 3.11, 3.9-alpine3.16, 3.9-alpine3.17, 3.9-alpine3.18, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18 | 3.11-alpine3.18 | -| aarch64-base-python | Alpine | 3.9, 3.10, 3.11, 3.9-alpine3.16, 3.9-alpine3.17, 3.9-alpine3.18, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18 | 3.11-alpine3.18 | -| amd64-base-python | Alpine | 3.9, 3.10, 3.11, 3.9-alpine3.16, 3.9-alpine3.17, 3.9-alpine3.18, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18 | 3.11-alpine3.18 | -| i386-base-python | Alpine | 3.9, 3.10, 3.11, 3.9-alpine3.16, 3.9-alpine3.17, 3.9-alpine3.18, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18 | 3.11-alpine3.18 | +| armhf-base-python | Alpine | 3.10, 3.11, 3.12, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18, 3.12-alpine3.16, 3.12-alpine3.17, 3.12-alpine3.18 | 3.12-alpine3.18 | +| armv7-base-python | Alpine | 3.10, 3.11, 3.12, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18, 3.12-alpine3.16, 3.12-alpine3.17, 3.12-alpine3.18 | 3.12-alpine3.18 | +| aarch64-base-python | Alpine | 3.10, 3.11, 3.12, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18, 3.12-alpine3.16, 3.12-alpine3.17, 3.12-alpine3.18 | 3.12-alpine3.18 | +| amd64-base-python | Alpine | 3.10, 3.11, 3.12, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18, 3.12-alpine3.16, 3.12-alpine3.17, 3.12-alpine3.18 | 3.12-alpine3.18 | +| i386-base-python | Alpine | 3.10, 3.11, 3.12, 3.10-alpine3.16, 3.10-alpine3.17, 3.10-alpine3.18, 3.11-alpine3.16, 3.11-alpine3.17, 3.11-alpine3.18, 3.12-alpine3.16, 3.12-alpine3.17, 3.12-alpine3.18 | 3.12-alpine3.18 | ## Others diff --git a/python/3.9/Dockerfile b/python/3.12/Dockerfile similarity index 97% rename from python/3.9/Dockerfile rename to python/3.12/Dockerfile index 6f0f64b..a209429 100644 --- a/python/3.9/Dockerfile +++ b/python/3.12/Dockerfile @@ -62,7 +62,7 @@ RUN set -ex \ xz-dev \ zlib-dev \ bluez-dev \ -# add build deps before removing fetch deps in case there's overlap + # add build deps before removing fetch deps in case there's overlap && apk del .fetch-deps \ \ && for i in /usr/src/*.patch; do \ @@ -78,8 +78,8 @@ RUN set -ex \ --with-lto \ --with-system-libmpdec \ --with-system-expat \ - --with-system-ffi \ --without-ensurepip \ + --without-static-libpython \ && make -j "$(nproc)" \ LDFLAGS="-Wl,--strip-all" \ CFLAGS="-fno-semantic-interposition -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free" \ diff --git a/python/3.9/arm-alignment.patch b/python/3.12/arm-alignment.patch similarity index 100% rename from python/3.9/arm-alignment.patch rename to python/3.12/arm-alignment.patch diff --git a/python/3.9/asynctio_unix_events.patch b/python/3.12/asynctio_unix_events.patch similarity index 97% rename from python/3.9/asynctio_unix_events.patch rename to python/3.12/asynctio_unix_events.patch index 45bf3fc..626198d 100644 --- a/python/3.9/asynctio_unix_events.patch +++ b/python/3.12/asynctio_unix_events.patch @@ -1,16 +1,16 @@ -diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py -index f34a5b4b44..b1d0f1e61e 100644 ---- a/Lib/asyncio/unix_events.py -+++ b/Lib/asyncio/unix_events.py -@@ -369,6 +369,11 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): - fut.set_result(total_sent) - return - -+ # On 32-bit architectures truncate to 1GiB to avoid OverflowError, -+ # see bpo-38319. -+ if sys.maxsize < 2 ** 32: -+ blocksize = min(blocksize, 2 ** 30) -+ - try: - sent = os.sendfile(fd, fileno, offset, blocksize) +diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py +index f34a5b4b44..b1d0f1e61e 100644 +--- a/Lib/asyncio/unix_events.py ++++ b/Lib/asyncio/unix_events.py +@@ -369,6 +369,11 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): + fut.set_result(total_sent) + return + ++ # On 32-bit architectures truncate to 1GiB to avoid OverflowError, ++ # see bpo-38319. ++ if sys.maxsize < 2 ** 32: ++ blocksize = min(blocksize, 2 ** 30) ++ + try: + sent = os.sendfile(fd, fileno, offset, blocksize) except (BlockingIOError, InterruptedError): \ No newline at end of file diff --git a/python/3.9/build.yaml b/python/3.12/build.yaml similarity index 88% rename from python/3.9/build.yaml rename to python/3.12/build.yaml index 163dfb9..ec5a35e 100644 --- a/python/3.9/build.yaml +++ b/python/3.12/build.yaml @@ -9,9 +9,9 @@ cosign: base_identity: https://github.com/home-assistant/docker-base/.* identity: https://github.com/home-assistant/docker-base/.* args: - PYTHON_VERSION: 3.9.17 + PYTHON_VERSION: 3.12.0 PIP_VERSION: 23.2.1 - GPG_KEY: E3FF2839C048B25C084DEBE9B26995E310250568 + GPG_KEY: 7169605F62C751356D054A26A821E680E5FA6305 labels: io.hass.base.name: python org.opencontainers.image.source: https://github.com/home-assistant/docker-base diff --git a/python/3.12/gh-106527-asyncio-optimize-to-add-remove-readers-and.patch b/python/3.12/gh-106527-asyncio-optimize-to-add-remove-readers-and.patch new file mode 100644 index 0000000..29f94f6 --- /dev/null +++ b/python/3.12/gh-106527-asyncio-optimize-to-add-remove-readers-and.patch @@ -0,0 +1,113 @@ +From b7dc795dfd175c0d25a479cfaf94a13c368a5a7b Mon Sep 17 00:00:00 2001 +From: "J. Nick Koston" +Date: Sat, 22 Jul 2023 16:07:40 -0500 +Subject: [PATCH] gh-106527: asyncio: optimize to add/remove readers and + writers (#106528) + +--- + Lib/asyncio/selector_events.py | 64 +++++++++---------- + Lib/test/test_asyncio/test_selector_events.py | 36 +++++------ + 2 files changed, 47 insertions(+), 53 deletions(-) + +diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py +index f895750e3c..d521b4e2e2 100644 +--- a/Lib/asyncio/selector_events.py ++++ b/Lib/asyncio/selector_events.py +@@ -274,9 +274,8 @@ def _ensure_fd_no_transport(self, fd): + def _add_reader(self, fd, callback, *args): + self._check_closed() + handle = events.Handle(callback, args, self, None) +- try: +- key = self._selector.get_key(fd) +- except KeyError: ++ key = self._selector.get_map().get(fd) ++ if key is None: + self._selector.register(fd, selectors.EVENT_READ, + (handle, None)) + else: +@@ -290,30 +289,27 @@ def _add_reader(self, fd, callback, *args): + def _remove_reader(self, fd): + if self.is_closed(): + return False +- try: +- key = self._selector.get_key(fd) +- except KeyError: ++ key = self._selector.get_map().get(fd) ++ if key is None: + return False ++ mask, (reader, writer) = key.events, key.data ++ mask &= ~selectors.EVENT_READ ++ if not mask: ++ self._selector.unregister(fd) + else: +- mask, (reader, writer) = key.events, key.data +- mask &= ~selectors.EVENT_READ +- if not mask: +- self._selector.unregister(fd) +- else: +- self._selector.modify(fd, mask, (None, writer)) ++ self._selector.modify(fd, mask, (None, writer)) + +- if reader is not None: +- reader.cancel() +- return True +- else: +- return False ++ if reader is not None: ++ reader.cancel() ++ return True ++ else: ++ return False + + def _add_writer(self, fd, callback, *args): + self._check_closed() + handle = events.Handle(callback, args, self, None) +- try: +- key = self._selector.get_key(fd) +- except KeyError: ++ key = self._selector.get_map().get(fd) ++ if key is None: + self._selector.register(fd, selectors.EVENT_WRITE, + (None, handle)) + else: +@@ -328,24 +324,22 @@ def _remove_writer(self, fd): + """Remove a writer callback.""" + if self.is_closed(): + return False +- try: +- key = self._selector.get_key(fd) +- except KeyError: ++ key = self._selector.get_map().get(fd) ++ if key is None: + return False ++ mask, (reader, writer) = key.events, key.data ++ # Remove both writer and connector. ++ mask &= ~selectors.EVENT_WRITE ++ if not mask: ++ self._selector.unregister(fd) + else: +- mask, (reader, writer) = key.events, key.data +- # Remove both writer and connector. +- mask &= ~selectors.EVENT_WRITE +- if not mask: +- self._selector.unregister(fd) +- else: +- self._selector.modify(fd, mask, (reader, None)) ++ self._selector.modify(fd, mask, (reader, None)) + +- if writer is not None: +- writer.cancel() +- return True +- else: +- return False ++ if writer is not None: ++ writer.cancel() ++ return True ++ else: ++ return False + + def add_reader(self, fd, callback, *args): + """Add a reader callback.""" +-- +2.39.2 (Apple Git-143) + diff --git a/python/3.12/gh-106554-replace-_BaseSelectorImpl._key_from_fd-wit.patch b/python/3.12/gh-106554-replace-_BaseSelectorImpl._key_from_fd-wit.patch new file mode 100644 index 0000000..e742d48 --- /dev/null +++ b/python/3.12/gh-106554-replace-_BaseSelectorImpl._key_from_fd-wit.patch @@ -0,0 +1,73 @@ +From aeef8591e41b68341af308e56a744396c66879cc Mon Sep 17 00:00:00 2001 +From: "J. Nick Koston" +Date: Fri, 14 Jul 2023 08:46:30 -1000 +Subject: [PATCH] gh-106554: replace `_BaseSelectorImpl._key_from_fd` with + `dict.get` (#106555) + +--- + Lib/selectors.py | 21 ++++----------------- + 1 file changed, 4 insertions(+), 17 deletions(-) + +diff --git a/Lib/selectors.py b/Lib/selectors.py +index dfcc125dcd..6d82935445 100644 +--- a/Lib/selectors.py ++++ b/Lib/selectors.py +@@ -276,19 +276,6 @@ def close(self): + def get_map(self): + return self._map + +- def _key_from_fd(self, fd): +- """Return the key associated to a given file descriptor. +- +- Parameters: +- fd -- file descriptor +- +- Returns: +- corresponding key, or None if not found +- """ +- try: +- return self._fd_to_key[fd] +- except KeyError: +- return None + + + class SelectSelector(_BaseSelectorImpl): +@@ -336,7 +323,7 @@ def select(self, timeout=None): + if fd in w: + events |= EVENT_WRITE + +- key = self._key_from_fd(fd) ++ key = self._fd_to_key.get(fd) + if key: + ready.append((key, events & key.events)) + return ready +@@ -426,7 +413,7 @@ def select(self, timeout=None): + if event & ~self._EVENT_WRITE: + events |= EVENT_READ + +- key = self._key_from_fd(fd) ++ key = self._fd_to_key.get(fd) + if key: + ready.append((key, events & key.events)) + return ready +@@ -479,7 +466,7 @@ def select(self, timeout=None): + if event & ~select.EPOLLOUT: + events |= EVENT_READ + +- key = self._key_from_fd(fd) ++ key = self._fd_to_key.get(fd) + if key: + ready.append((key, events & key.events)) + return ready +@@ -574,7 +561,7 @@ def select(self, timeout=None): + if flag == select.KQ_FILTER_WRITE: + events |= EVENT_WRITE + +- key = self._key_from_fd(fd) ++ key = self._fd_to_key.get(fd) + if key: + ready.append((key, events & key.events)) + return ready +-- +2.39.2 (Apple Git-143) + diff --git a/python/3.12/gh-106664-selectors-add-get-method-to-_SelectorMappi.patch b/python/3.12/gh-106664-selectors-add-get-method-to-_SelectorMappi.patch new file mode 100644 index 0000000..b1ec461 --- /dev/null +++ b/python/3.12/gh-106664-selectors-add-get-method-to-_SelectorMappi.patch @@ -0,0 +1,43 @@ +From 8d2f3c36caf9ecdee1176314b18388aef6e7f2c2 Mon Sep 17 00:00:00 2001 +From: "J. Nick Koston" +Date: Thu, 13 Jul 2023 09:18:53 -1000 +Subject: [PATCH] gh-106664: selectors: add get() method to _SelectorMapping + (#106665) + +It can be used to avoid raising and catching KeyError twice via __getitem__. + +Co-authored-by: Inada Naoki +--- + Lib/selectors.py | 14 +++++++++----- + Lib/test/test_selectors.py | 6 ++++++ + 2 files changed, 15 insertions(+), 5 deletions(-) + +diff --git a/Lib/selectors.py b/Lib/selectors.py +index af6a4f94b5..dfcc125dcd 100644 +--- a/Lib/selectors.py ++++ b/Lib/selectors.py +@@ -66,12 +66,16 @@ def __init__(self, selector): + def __len__(self): + return len(self._selector._fd_to_key) + ++ def get(self, fileobj, default=None): ++ fd = self._selector._fileobj_lookup(fileobj) ++ return self._selector._fd_to_key.get(fd, default) ++ + def __getitem__(self, fileobj): +- try: +- fd = self._selector._fileobj_lookup(fileobj) +- return self._selector._fd_to_key[fd] +- except KeyError: +- raise KeyError("{!r} is not registered".format(fileobj)) from None ++ fd = self._selector._fileobj_lookup(fileobj) ++ key = self._selector._fd_to_key.get(fd) ++ if key is None: ++ raise KeyError("{!r} is not registered".format(fileobj)) ++ return key + + def __iter__(self): + return iter(self._selector._fd_to_key) +-- +2.39.2 (Apple Git-143) + diff --git a/python/3.12/gh-106751-selectors-optimize-EpollSelector.select-10.patch b/python/3.12/gh-106751-selectors-optimize-EpollSelector.select-10.patch new file mode 100644 index 0000000..79cfcc6 --- /dev/null +++ b/python/3.12/gh-106751-selectors-optimize-EpollSelector.select-10.patch @@ -0,0 +1,57 @@ +From aecf6aca515a203a823a87c711f15cbb82097c8b Mon Sep 17 00:00:00 2001 +From: "J. Nick Koston" +Date: Tue, 18 Jul 2023 00:16:32 -1000 +Subject: [PATCH] gh-106751: selectors: optimize EpollSelector.select() + (#106754) + +Co-authored-by: Pieter Eendebak +--- + Lib/selectors.py | 17 +++++++++-------- + 1 file changed, 9 insertions(+), 8 deletions(-) + +diff --git a/Lib/selectors.py b/Lib/selectors.py +index 6d82935445..a42d156340 100644 +--- a/Lib/selectors.py ++++ b/Lib/selectors.py +@@ -430,6 +430,9 @@ class PollSelector(_PollLikeSelector): + + if hasattr(select, 'epoll'): + ++ _NOT_EPOLLIN = ~select.EPOLLIN ++ _NOT_EPOLLOUT = ~select.EPOLLOUT ++ + class EpollSelector(_PollLikeSelector): + """Epoll-based selector.""" + _selector_cls = select.epoll +@@ -452,22 +455,20 @@ def select(self, timeout=None): + # epoll_wait() expects `maxevents` to be greater than zero; + # we want to make sure that `select()` can be called when no + # FD is registered. +- max_ev = max(len(self._fd_to_key), 1) ++ max_ev = len(self._fd_to_key) or 1 + + ready = [] + try: + fd_event_list = self._selector.poll(timeout, max_ev) + except InterruptedError: + return ready +- for fd, event in fd_event_list: +- events = 0 +- if event & ~select.EPOLLIN: +- events |= EVENT_WRITE +- if event & ~select.EPOLLOUT: +- events |= EVENT_READ + +- key = self._fd_to_key.get(fd) ++ fd_to_key = self._fd_to_key ++ for fd, event in fd_event_list: ++ key = fd_to_key.get(fd) + if key: ++ events = ((event & _NOT_EPOLLIN and EVENT_WRITE) ++ | (event & _NOT_EPOLLOUT and EVENT_READ)) + ready.append((key, events & key.events)) + return ready + +-- +2.39.2 (Apple Git-143) + diff --git a/python/3.9/musl-find_library.patch b/python/3.12/musl-find_library.patch similarity index 100% rename from python/3.9/musl-find_library.patch rename to python/3.12/musl-find_library.patch diff --git a/python/3.9/fix-xattrs-glibc.patch b/python/3.9/fix-xattrs-glibc.patch deleted file mode 100644 index 6902425..0000000 --- a/python/3.9/fix-xattrs-glibc.patch +++ /dev/null @@ -1,16 +0,0 @@ -diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c -index 8f8ba25..72b92da 100644 ---- a/Modules/posixmodule.c -+++ b/Modules/posixmodule.c -@@ -103,8 +103,9 @@ corresponding Unix manual entries for more information on calls."); - #undef HAVE_SCHED_SETAFFINITY - #endif - --#if defined(HAVE_SYS_XATTR_H) && defined(__GLIBC__) && !defined(__FreeBSD_kernel__) && !defined(__GNU__) -+#if defined(HAVE_SYS_XATTR_H) && defined(__linux__) && !defined(__FreeBSD_kernel__) && !defined(__GNU__) - #define USE_XATTRS -+#include - #endif - - #ifdef USE_XATTRS -