Merge branch 'main' into object-reload-refactor

Ultimaker · Nov 1, 2023 · f31980d · f31980d
2 parents 93f2171 + 2a81d57
commit f31980d
Show file tree

Hide file tree

Showing 226 changed files with 61,358 additions and 52,851 deletions.
diff --git a/.github/workflows/installers.yml b/.github/workflows/installers.yml
@@ -31,28 +31,41 @@ on:
         type: boolean
 
   schedule:
-    # Daily at 5:15 CET
+    # Daily at 4:15 CET (main-branch) and 5:15 CET (release-branch)
     - cron: '15 3 * * *'
+    - cron: '15 4 * * *'
 
 env:
-  CURA_CONAN_VERSION: ${{ inputs.cura_conan_version || 'cura/latest@ultimaker/testing' }}
   CONAN_ARGS: ${{ inputs.conan_args || '' }}
   ENTERPRISE: ${{ inputs.enterprise || false }}
   STAGING: ${{ inputs.staging || false }}
+  LATEST_RELEASE: '5.6'
+  LATEST_RELEASE_SCHEDULE_HOUR: 4
 
 jobs:
-  default-values:
+  default_values:
     runs-on: ubuntu-latest
     outputs:
       cura_conan_version: ${{ steps.default.outputs.cura_conan_version  }}
+      release_tag: ${{ steps.default.outputs.release_tag  }}
 
     steps:
     - name: Output default values
       id: default
       shell: python
       run: |
         import os
-        cura_conan_version = "cura/latest@ultimaker/testing" if "${{ github.event.inputs.cura_conan_version }}" == "" else "${{ github.event.inputs.cura_conan_version }}" 
+        import datetime
+        
+        if "${{ github.event_name }}" != "schedule":
+            cura_conan_version = "${{ github.event.inputs.cura_conan_version }}"
+        else:
+            now = datetime.datetime.now()
+            cura_conan_version = "cura/latest@ultimaker/stable" if now.hour == int(os.environ['LATEST_RELEASE_SCHEDULE_HOUR']) else "cura/latest@ultimaker/testing"
+        
+        release_tag = f"nightly-{os.environ['LATEST_RELEASE']}" if "/stable" in cura_conan_version else "nightly"
+
+        # Set cura_conan_version environment variable
         output_env = os.environ["GITHUB_OUTPUT"]
         content = ""
         if os.path.exists(output_env):
@@ -61,12 +74,24 @@ jobs:
         with open(output_env, "w") as f:
             f.write(content)
             f.writelines(f"cura_conan_version={cura_conan_version}\n")
+            f.writelines(f"release_tag={release_tag}\n")
+
+        summary_env = os.environ["GITHUB_STEP_SUMMARY"]
+        content = ""
+        if os.path.exists(summary_env):
+            with open(summary_env, "r") as f:
+                content = f.read()
+        
+        with open(summary_env, "w") as f:
+            f.write(content)
+            f.writelines(f"# cura_conan_version = {cura_conan_version}\n")
+            f.writelines(f"# release_tag = {release_tag}\n")
 
   windows-installer:
     uses: ./.github/workflows/windows.yml
-    needs: [ default-values ]
+    needs: [ default_values ]
     with:
-      cura_conan_version: ${{ needs.default-values.outputs.cura_conan_version }}
+      cura_conan_version: ${{ needs.default_values.outputs.cura_conan_version }}
       conan_args: ${{ github.event.inputs.conan_args }}
       enterprise: ${{ github.event.inputs.enterprise == 'true' }}
       staging: ${{ github.event.inputs.staging == 'true' }}
@@ -76,9 +101,9 @@ jobs:
 
   linux-installer:
     uses: ./.github/workflows/linux.yml
-    needs: [ default-values ]
+    needs: [ default_values ]
     with:
-      cura_conan_version: ${{ needs.default-values.outputs.cura_conan_version }}
+      cura_conan_version: ${{ needs.default_values.outputs.cura_conan_version }}
       conan_args: ${{ github.event.inputs.conan_args }}
       enterprise: ${{ github.event.inputs.enterprise == 'true' }}
       staging: ${{ github.event.inputs.staging == 'true' }}
@@ -88,33 +113,33 @@ jobs:
 
   macos-installer:
     uses: ./.github/workflows/macos.yml
-    needs: [ default-values ]
+    needs: [ default_values ]
     with:
-      cura_conan_version: ${{ needs.default-values.outputs.cura_conan_version }}
+      cura_conan_version: ${{ needs.default_values.outputs.cura_conan_version }}
       conan_args: ${{ github.event.inputs.conan_args }}
       enterprise: ${{ github.event.inputs.enterprise == 'true' }}
       staging: ${{ github.event.inputs.staging == 'true' }}
       architecture: X64
-      operating_system: macos-12
+      operating_system: self-hosted-X64
     secrets: inherit
 
   macos-arm-installer:
     uses: ./.github/workflows/macos.yml
-    needs: [ default-values ]
+    needs: [ default_values ]
     with:
-      cura_conan_version: ${{ needs.default-values.outputs.cura_conan_version }}
+      cura_conan_version: ${{ needs.default_values.outputs.cura_conan_version }}
       conan_args: ${{ github.event.inputs.conan_args }}
       enterprise: ${{ github.event.inputs.enterprise == 'true' }}
       staging: ${{ github.event.inputs.staging == 'true' }}
       architecture: ARM64
-      operating_system: self-hosted
+      operating_system: self-hosted-ARM64
     secrets: inherit
 
   # Run and update nightly release when the nightly input is set to true or if the schedule is triggered
   update-nightly-release:
     if: ${{ inputs.nightly || github.event_name == 'schedule' }}
     runs-on: ubuntu-latest
-    needs: [ windows-installer, linux-installer, macos-installer, macos-arm-installer ]
+    needs: [ default_values, windows-installer, linux-installer, macos-installer, macos-arm-installer ]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -182,8 +207,8 @@ jobs:
 
       - name: Update nightly release for Linux
         run: |
-          gh release upload nightly installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-linux-X64.AppImage --clobber
-          gh release upload nightly installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-linux-X64.AppImage.asc  --clobber
+          gh release upload ${{ needs.default_values.outputs.release_tag }} installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-linux-X64.AppImage --clobber
+          gh release upload ${{ needs.default_values.outputs.release_tag }} installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-linux-X64.AppImage.asc  --clobber
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -206,8 +231,8 @@ jobs:
 
       - name: Update nightly release for Windows
         run: |
-          gh release upload nightly installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-win64-X64.msi --clobber
-          gh release upload nightly installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-win64-X64.exe --clobber
+          gh release upload ${{ needs.default_values.outputs.release_tag }} installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-win64-X64.msi --clobber
+          gh release upload ${{ needs.default_values.outputs.release_tag }} installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-win64-X64.exe --clobber
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -230,8 +255,8 @@ jobs:
 
       - name: Update nightly release for MacOS (X64)
         run: |
-          gh release upload nightly installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-macos-X64.dmg --clobber
-          gh release upload nightly installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-macos-X64.pkg --clobber
+          gh release upload ${{ needs.default_values.outputs.release_tag }} installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-macos-X64.dmg --clobber
+          gh release upload ${{ needs.default_values.outputs.release_tag }} installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-macos-X64.pkg --clobber
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -254,14 +279,31 @@ jobs:
 
       - name: Update nightly release for MacOS (ARM-64)
         run: |
-          gh release upload nightly installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-macos-ARM64.dmg --clobber
-          gh release upload nightly installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-macos-ARM64.pkg --clobber
+          gh release upload ${{ needs.default_values.outputs.release_tag }} installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-macos-ARM64.dmg --clobber
+          gh release upload ${{ needs.default_values.outputs.release_tag }} installers/${{ steps.filename.outputs.NIGHTLY_NAME }}-macos-ARM64.pkg --clobber
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: create the release notes
+        run: |
+          import os
+          import datetime
+          from jinja2 import Template
+          
+          with open(".github/workflows/release-notes.md.jinja", "r") as f:
+              release_notes = Template(f.read())
+          
+          current_nightly_beta = "${{ needs.default_values.outputs.release_tag }}".split("nightly-")[-1]
+          with open("release-notes.md", "w") as f:
+              f.write(release_notes.render(
+                timestamp=${{ steps.filename.outputs.NIGHTLY_TIME }},
+                branch="" if ${{ needs.default_values.outputs.release_tag == 'nightly' }} else current_nightly_beta,
+                branch_specific="" if os.getenv("GITHUB_REF") == "refs/heads/main" else f"?branch={current_nightly_beta}",
+              ))
+
       - name: Update nightly release description (with date)
         if: always()
         run: |
-          gh release edit nightly --title "${{ steps.filename.outputs.NIGHTLY_NAME }}" --notes "Nightly release created on: ${{ steps.filename.outputs.NIGHTLY_TIME }}"
+          gh release edit ${{ needs.default_values.outputs.release_tag }} --title "${{ steps.filename.outputs.NIGHTLY_NAME }}" --notes-file release-notes.md
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -106,16 +106,8 @@ jobs:
             $HOME/.conan/conan_download_cache
           key: conan-${{ runner.os }}-${{ runner.arch }}-installer-cache
 
-      - name: Hack needed specifically for ubuntu-22.04 from mid-Feb 2023 onwards
-        if: ${{ startsWith(inputs.operating_system, 'ubuntu-22.04') }}
-        run: sudo apt remove libodbc2 libodbcinst2 unixodbc-common -y
-
-      # NOTE: Due to what are probably github issues, we have to remove the cache and reconfigure before the rest.
-      #       This is maybe because grub caches the disk it uses last time, which is recreated each time.
       - name: Install Linux system requirements
         run: |
-          sudo rm /var/cache/debconf/config.dat
-          sudo dpkg --configure -a
           sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
           sudo apt update
           sudo apt upgrade
@@ -155,7 +147,12 @@ jobs:
         run: conan config set storage.download_cache="$HOME/.conan/conan_download_cache"
 
       - name: Create the Packages (Bash)
-        run: conan install $CURA_CONAN_VERSION ${{ inputs.conan_args }} --build=missing --update -if cura_inst -g VirtualPythonEnv -o cura:enterprise=$ENTERPRISE -o cura:staging=$STAGING --json "cura_inst/conan_install_info.json"
+        run: conan install $CURA_CONAN_VERSION ${{ inputs.conan_args }} --build=missing --update -if cura_inst -g VirtualPythonEnv -o cura:enterprise=$ENTERPRISE -o cura:staging=$STAGING -c tools.build:skip_test=True
+
+      - name: Remove internal packages before uploading
+        run: |
+          conan remove "*@internal/*" -f || true
+          conan remove "cura_private_data*" -f || true
 
       - name: Upload the Package(s)
         if: always()
@@ -199,19 +196,12 @@ jobs:
               f.write(content)
               f.writelines(f"INSTALLER_FILENAME={installer_filename}\n")
 
-      - name: Summarize the used Conan dependencies
+      - name: Summarize the used dependencies
         shell: python
         run: |
           import os
-          import json
-          from pathlib import Path
           
-          conan_install_info_path = Path("cura_inst/conan_install_info.json")
-          conan_info = {"installed": []}
-          if os.path.exists(conan_install_info_path):
-              with open(conan_install_info_path, "r") as f:
-                  conan_info = json.load(f)
-          sorted_deps = sorted([dep["recipe"]["id"].replace('#', r' rev: ') for dep in conan_info["installed"]])
+          from cura.CuraVersion import ConanInstalls, PythonInstalls
           
           summary_env = os.environ["GITHUB_STEP_SUMMARY"]
           content = ""
@@ -223,25 +213,12 @@ jobs:
               f.write(content)
               f.writelines("# ${{ steps.filename.outputs.INSTALLER_FILENAME }}\n")
               f.writelines("## Conan packages:\n")
-              for dep in sorted_deps:
-                  f.writelines(f"`{dep}`\n")
+              for dep_name, dep_info in ConanInstalls.items():
+                  f.writelines(f"`{dep_name} {dep_info['version']} {dep_info['revision']}`\n")
 
-      - name: Summarize the used Python modules
-        shell: python
-        run: |
-          import os
-          import pkg_resources
-          summary_env = os.environ["GITHUB_STEP_SUMMARY"]
-          content = ""
-          if os.path.exists(summary_env):
-              with open(summary_env, "r") as f:
-                  content = f.read()
-          
-          with open(summary_env, "w") as f:
-              f.write(content)
               f.writelines("## Python modules:\n")
-              for package in pkg_resources.working_set:
-                  f.writelines(f"`{package.key}/{package.version}`\n")
+              for dep_name, dep_info in PythonInstalls.items():
+                  f.writelines(f"`{dep_name} {dep_info['version']}`\n")
 
       - name: Create the Linux AppImage (Bash)
         run: |

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -27,18 +27,19 @@ on:
       architecture:
         description: 'Architecture'
         required: true
-        default: 'X64'
+        default: 'ARM64'
         type: choice
         options:
           - X64
           - ARM64
       operating_system:
         description: 'OS'
         required: true
-        default: 'macos-11'
+        default: 'self-hosted-ARM64'
         type: choice
         options:
-          - self-hosted
+          - self-hosted-X64
+          - self-hosted-ARM64
           - macos-11
           - macos-12
   workflow_call:
@@ -66,12 +67,12 @@ on:
       architecture:
         description: 'Architecture'
         required: true
-        default: 'X64'
+        default: 'ARM64'
         type: string
       operating_system:
         description: 'OS'
         required: true
-        default: 'macos-11'
+        default: 'self-hosted-ARM64'
         type: string
 
 env:
@@ -102,7 +103,7 @@ jobs:
       - name: Setup Python and pip
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10.x'
+          python-version: '3.11.x'
           cache: 'pip'
           cache-dependency-path: .github/workflows/requirements-conan-package.txt
 
@@ -155,10 +156,14 @@ jobs:
         run: conan config set storage.download_cache="$HOME/.conan/conan_download_cache"
 
       - name: Create the Packages (Bash)
-        run: conan install $CURA_CONAN_VERSION ${{ inputs.conan_args }} --build=missing --update -if cura_inst -g VirtualPythonEnv -o cura:enterprise=$ENTERPRISE -o cura:staging=$STAGING --json "cura_inst/conan_install_info.json"
+        run: conan install $CURA_CONAN_VERSION ${{ inputs.conan_args }} --build=missing --update -if cura_inst -g VirtualPythonEnv -o cura:enterprise=$ENTERPRISE -o cura:staging=$STAGING -c tools.build:skip_test=True
+
+      - name: Remove internal packages before uploading
+        run: |
+          conan remove "*@internal/*" -f || true
+          conan remove "cura_private_data*" -f || true
 
       - name: Upload the Package(s)
-        if: ${{ inputs.operating_system != 'self-hosted' }}
         run: |
           conan upload "*" -r cura --all -c
 
@@ -203,19 +208,12 @@ jobs:
               f.write(content)
               f.writelines(f"INSTALLER_FILENAME={installer_filename}\n")
 
-      - name: Summarize the used Conan dependencies
+      - name: Summarize the used dependencies
         shell: python
         run: |
           import os
-          import json
-          from pathlib import Path
           
-          conan_install_info_path = Path("cura_inst/conan_install_info.json")
-          conan_info = {"installed": []}
-          if os.path.exists(conan_install_info_path):
-              with open(conan_install_info_path, "r") as f:
-                  conan_info = json.load(f)
-          sorted_deps = sorted([dep["recipe"]["id"].replace('#', r' rev: ') for dep in conan_info["installed"]])
+          from cura.CuraVersion import ConanInstalls, PythonInstalls
           
           summary_env = os.environ["GITHUB_STEP_SUMMARY"]
           content = ""
@@ -227,25 +225,12 @@ jobs:
               f.write(content)
               f.writelines("# ${{ steps.filename.outputs.INSTALLER_FILENAME }}\n")
               f.writelines("## Conan packages:\n")
-              for dep in sorted_deps:
-                  f.writelines(f"`{dep}`\n")
+              for dep_name, dep_info in ConanInstalls.items():
+                  f.writelines(f"`{dep_name} {dep_info['version']} {dep_info['revision']}`\n")
 
-      - name: Summarize the used Python modules
-        shell: python
-        run: |
-          import os
-          import pkg_resources
-          summary_env = os.environ["GITHUB_STEP_SUMMARY"]
-          content = ""
-          if os.path.exists(summary_env):
-              with open(summary_env, "r") as f:
-                  content = f.read()
-          
-          with open(summary_env, "w") as f:
-              f.write(content)
               f.writelines("## Python modules:\n")
-              for package in pkg_resources.working_set:
-                  f.writelines(f"`{package.key}/{package.version}`\n")
+              for dep_name, dep_info in PythonInstalls.items():
+                  f.writelines(f"`{dep_name} {dep_info['version']}`\n")
 
       - name: Create the Macos dmg (Bash)
         run: python ../cura_inst/packaging/MacOS/build_macos.py --source_path ../cura_inst --dist_path . --cura_conan_version $CURA_CONAN_VERSION --filename "${{ steps.filename.outputs.INSTALLER_FILENAME }}" --build_dmg --build_pkg --app_name "$CURA_APP_NAME"