Merge branch 'feature/config-overhaul' into dl_dev-little-fixes

.env

@@ -1,4 +1,4 @@
-EOS_VERSION=latest
+EOS_VERSION=main
 EOS_PORT=8503
 
 PYTHON_VERSION=3.12.6

.github/workflows/docker-build.yml

@@ -1,27 +1,53 @@
 name: docker-build
 
 on:
-  release:
-    types: [published]
+  # pipeline runs per trigger condition, so release trigger not required as tag is sufficient
+  #release:
+  #  types: [published]
+  push:
+    branches:
+      - 'main'
+      - 'feature/config-overhaul'
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - 'main'
+      - 'feature/config-overhaul'
 
 env:
-  REGISTRY: ghcr.io
-
+  DOCKERHUB_REPO: akkudoktor/eos
+  GHCR_REPO: ghcr.io/akkudoktor-eos/eos
+  EOS_LICENSE: Apache-2.0
+
+# From https://docs.docker.com/build/ci/github-actions/multi-platform/
+# Changes:
+#  - adjusted rw permissions
+#  - manually set undetected license (label+annotation)
+#  - set description for index manifest
+#  - add attestation
+#  - conditionally don't push on pr
+#  - on pr just use amd64 platform
 jobs:
-  variables:
+  # Build platform matrix excludes. if-conditional with matrix on job level is not
+  # supported, see https://github.com/actions/runner/issues/1985
+  platform-excludes:
+    runs-on: ubuntu-latest
     outputs:
-      repository: ${{ steps.var.outputs.repository}}
-    runs-on: "ubuntu-latest"
+      excludes: ${{ steps.excludes.outputs.matrix }}
     steps:
-      - name: Setting global variables
-        uses: actions/github-script@v6
-        id: var
-        with:
-          script: |
-            core.setOutput('repository', '${{ github.repository }}'.toLowerCase());
+      - id: excludes
+        run: |
+          if ${{ github.event_name == 'pull_request' }}; then
+            echo 'matrix=[
+              {"platform": "linux/arm64"}
+            ]' | tr -d '[:space:]' >> $GITHUB_OUTPUT
+          else
+            echo 'matrix=[]' >> $GITHUB_OUTPUT
+          fi
+
   build:
-    needs:
-      - variables
+    needs: platform-excludes
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -34,6 +60,7 @@ jobs:
         platform:
           - linux/amd64
           - linux/arm64
+        exclude: ${{ fromJSON(needs.platform-excludes.outputs.excludes) }}
     steps:
       - name: Prepare
         run: |
@@ -44,35 +71,64 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: "${{ env.REGISTRY }}/${{ needs.variables.outputs.repository }}"
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+          images: |
+            ${{ env.DOCKERHUB_REPO }}
+            ${{ env.GHCR_REPO }}
+          labels: |
+            org.opencontainers.image.licenses=${{ env.EOS_LICENSE }}
+          annotations: |
+            org.opencontainers.image.licenses=${{ env.EOS_LICENSE }}
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+
+      # Prepare to extract description so it can be manually set for index manifest (group of platform manifests)
+      - name: Prepare description
+        id: get_description
+        run: |
+          echo "EOS_REPO_DESCRIPTION=$(jq -cr '.labels."org.opencontainers.image.description"' <<< "$DOCKER_METADATA_OUTPUT_JSON")" >> $GITHUB_ENV
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Login to GitHub
+      - name: Login to GHCR
         uses: docker/login-action@v3
         with:
-          registry: ${{ env.REGISTRY }}
+          registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Build and push by digest
         id: build
         uses: docker/build-push-action@v6
         with:
           platforms: ${{ matrix.platform }}
           labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=image,name=${{ env.REGISTRY }}/${{ needs.variables.outputs.repository }},push-by-digest=true,name-canonical=true,push=true
+          annotations: ${{ steps.meta.outputs.annotations }}
+          outputs: type=image,"name=${{ env.DOCKERHUB_REPO }},${{ env.GHCR_REPO }}",push-by-digest=true,name-canonical=true,"push=${{ github.event_name != 'pull_request' }}","annotation-index.org.opencontainers.image.description=${{ env.EOS_REPO_DESCRIPTION }}"
+          #push: ${{ github.event_name != 'pull_request' }}
 
-      - name: Generate artifact attestation
-        uses: actions/attest-build-provenance@v1
+      - name: Generate artifact attestation DockerHub
+        uses: actions/attest-build-provenance@v2
         with:
-          subject-name: "${{ env.REGISTRY }}/${{ needs.variables.outputs.repository }}"
+          subject-name: docker.io/${{ env.DOCKERHUB_REPO }}
           subject-digest: ${{ steps.build.outputs.digest }}
-          push-to-registry: true
+          push-to-registry: ${{ github.event_name != 'pull_request' }}
+
+      - name: Generate artifact attestation GitHub
+        uses: actions/attest-build-provenance@v2
+        with:
+          subject-name: ${{ env.GHCR_REPO }}
+          subject-digest: ${{ steps.build.outputs.digest }}
+          push-to-registry: ${{ github.event_name != 'pull_request' }}
 
       - name: Export digest
         run: |
@@ -90,9 +146,14 @@ jobs:
 
   merge:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
     needs:
       - build
-      - variables
+    # skip for pull requests
+    if: ${{ github.event_name != 'pull_request' }}
     steps:
       - name: Download digests
         uses: actions/download-artifact@v4
@@ -101,28 +162,50 @@ jobs:
           pattern: digests-*
           merge-multiple: true
 
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: "${{ env.REGISTRY }}/${{ needs.variables.outputs.repository }}"
-
-      - name: Login to GitHub
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          images: |
+            ${{ env.DOCKERHUB_REPO }}
+            ${{ env.GHCR_REPO }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+          labels: |
+            org.opencontainers.image.licenses=${{ env.EOS_LICENSE }}
+          annotations: |
+            org.opencontainers.image.licenses=${{ env.EOS_LICENSE }}
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
 
       - name: Create manifest list and push
         working-directory: /tmp/digests
         run: |
           docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.REGISTRY }}/${{ needs.variables.outputs.repository }}@sha256:%s ' *)
+            $(printf '${{ env.DOCKERHUB_REPO }}@sha256:%s ' *)
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.GHCR_REPO }}@sha256:%s ' *)
 
       - name: Inspect image
         run: |
-          docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ needs.variables.outputs.repository }}:${{ steps.meta.outputs.version }}
+          docker buildx imagetools inspect ${{ env.DOCKERHUB_REPO }}:${{ steps.meta.outputs.version }}
+          docker buildx imagetools inspect ${{ env.GHCR_REPO }}:${{ steps.meta.outputs.version }}

README.md

@@ -10,7 +10,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md).
 
 ## Installation
 
-The project requires Python 3.10 or newer. Currently there are no official packages or images published.
+The project requires Python 3.10 or newer. Official docker images can be found at [akkudoktor/eos](https://hub.docker.com/r/akkudoktor/eos).
 
 Following sections describe how to locally start the EOS server on `http://localhost:8503`.
 
@@ -49,7 +49,7 @@ Windows:
 ### Docker
 
 ```bash
-docker compose up --build
+docker compose up
 ```
 
 ## Configuration

single_test_optimization.py

@@ -12,12 +12,15 @@
 
 from akkudoktoreos.config.config import get_config
 from akkudoktoreos.core.ems import get_ems
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.optimization.genetic import (
     OptimizationParameters,
     optimization_problem,
 )
 from akkudoktoreos.prediction.prediction import get_prediction
 
+get_logger(__name__, logging_level="DEBUG")
+
 
 def prepare_optimization_real_parameters() -> OptimizationParameters:
     """Prepare and return optimization parameters with real world data.

src/akkudoktoreos/optimization/genetic.py

@@ -1,3 +1,4 @@
+import logging
 import random
 import time
 from pathlib import Path
@@ -15,6 +16,7 @@
 )
 from akkudoktoreos.core.ems import EnergieManagementSystemParameters, SimulationResult
 from akkudoktoreos.core.pydantic import ParametersBaseModel
+from akkudoktoreos.core.logging import get_logger
 from akkudoktoreos.devices.battery import (
     Battery,
     ElectricVehicleParameters,
@@ -26,6 +28,8 @@
 from akkudoktoreos.prediction.interpolator import SelfConsumptionPropabilityInterpolator
 from akkudoktoreos.utils.utils import NumpyEncoder
 
+logger = get_logger(__name__)
+
 
 class OptimizationParameters(ParametersBaseModel):
     ems: EnergieManagementSystemParameters
@@ -114,10 +118,14 @@ def __init__(
         self.fix_seed = fixed_seed
         self.optimize_ev = True
         self.optimize_dc_charge = False
+        self.fitness_history: dict[str, Any] = {}
 
-        # Set a fixed seed for random operations if provided
-        if fixed_seed is not None:
-            random.seed(fixed_seed)
+        # Set a fixed seed for random operations if provided or in debug mode
+        if self.fix_seed is not None:
+            random.seed(self.fix_seed)
+        elif logger.level == logging.DEBUG:
+            self.fix_seed = random.randint(1, 100000000000)
+            random.seed(self.fix_seed)
 
     def decode_charge_discharge(
         self, discharge_hours_bin: np.ndarray
@@ -494,6 +502,8 @@ def optimize(
         hof = tools.HallOfFame(1)
         stats = tools.Statistics(lambda ind: ind.fitness.values)
         stats.register("min", np.min)
+        stats.register("avg", np.mean)
+        stats.register("max", np.max)
 
         if self.verbose:
             print("Start optimize:", start_solution)
@@ -504,7 +514,7 @@ def optimize(
                 population.insert(0, creator.Individual(start_solution))
 
         # Run the evolutionary algorithm
-        algorithms.eaMuPlusLambda(
+        pop, log = algorithms.eaMuPlusLambda(
             population,
             self.toolbox,
             mu=100,
@@ -517,6 +527,14 @@ def optimize(
             verbose=self.verbose,
         )
 
+        # Store fitness history
+        self.fitness_history = {
+            "gen": log.select("gen"),  # Generation numbers (X-axis)
+            "avg": log.select("avg"),  # Average fitness for each generation (Y-axis)
+            "max": log.select("max"),  # Maximum fitness for each generation (Y-axis)
+            "min": log.select("min"),  # Minimum fitness for each generation (Y-axis)
+        }
+
         member: dict[str, list[float]] = {"bilanz": [], "verluste": [], "nebenbedingung": []}
         for ind in population:
             if hasattr(ind, "extra_data"):
@@ -632,6 +650,8 @@ def optimierung_ems(
             "start_solution": start_solution,
             "spuelstart": washingstart_int,
             "extra_data": extra_data,
+            "fitness_history": self.fitness_history,
+            "fixed_seed": self.fix_seed,
         }
         from akkudoktoreos.utils.visualize import prepare_visualize
 

src/akkudoktoreos/prediction/elecpriceakkudoktor.py

@@ -187,6 +187,13 @@ def _update_data(
             self.config.prediction_hours
             - ((highest_orig_datetime - self.start_datetime).total_seconds() // 3600)
         )
+
+        if needed_prediction_hours <= 0:
+            logger.warning(
+                f"No prediction needed. needed_prediction_hours={needed_prediction_hours}, prediction_hours={self.config.prediction_hours},highest_orig_datetime {highest_orig_datetime}, start_datetime {self.start_datetime}"
+            )  # this might keep data longer than self.start_datetime + self.config.prediction_hours in the records
+            return
+
         if amount_datasets > 800:  # we do the full ets with seasons of 1 week
             prediction = self._predict_ets(
                 history, seasonal_periods=168, prediction_hours=needed_prediction_hours