remove pre/post network folder prefixes

Snakefile

@@ -43,6 +43,7 @@ wildcard_constraints:
     ll=r"(v|c)([0-9\.]+|opt)",
     opts=r"[-+a-zA-Z0-9\.]*",
     sector_opts=r"[-+a-zA-Z0-9\.\s]*",
+    planning_horizons=r"[0-9]{4}",
 
 
 include: "rules/common.smk"

doc/foresight.rst

@@ -206,21 +206,21 @@ The myopic code solves the network for the time steps included in
    baseyear=2020, capacities installed before 2020 are added. In addition, the
    network comprises additional generator, storage, and link capacities with
    p_nom_extendable=True. The non-solved network is saved in
-   ``results/run_name/networks/prenetworks-brownfield``.
+   ``resources/run_name/networks``.
 
 The base year is the first element in ``planning_horizons``. Step 1 is
 implemented with the rule add_baseyear for the base year and with the rule
 add_brownfield for the remaining planning_horizons.
 
 2. The 2020 network is optimized. The solved network is saved in
-   ``results/run_name/networks/postnetworks``
+   ``results/run_name/networks``
 
 3. For the next planning horizon, e.g. 2030, the capacities from a previous time
    step are added if they are still in operation (i.e., if they fulfil planning
    horizon <= commissioned year + lifetime). In addition, the network comprises
    additional generator, storage, and link capacities with
    p_nom_extendable=True. The non-solved network is saved in
-   ``results/run_name/networks/prenetworks-brownfield``.
+   ``results/run_name/networks``.
 
 Steps 2 and 3 are solved recursively for all the planning_horizons included in
 ``config/config.yaml``.
@@ -231,7 +231,7 @@ Rule overview
 - rule add_existing baseyear
 
   The rule add_existing_baseyear loads the network in
-  ‘results/run_name/networks/prenetworks’ and performs the following operations:
+  ``resources/run_name/networks`` and performs the following operations:
 
   1. Add the conventional, wind and solar power generators that were installed
      before the base year.
@@ -260,17 +260,13 @@ Rule overview
   2045.
 
   Then, the resulting network is saved in
-  ``results/run_name/networks/prenetworks-brownfield``.
+  ``resources/run_name/networks``.
 
 - rule add_brownfield
 
-  The rule add_brownfield loads the network in
-  ``results/run_name/networks/prenetworks`` and performs the following
-  operation:
-
-  1. Read the capacities optimized in the previous time step and add them to the
+  The rule add_brownfield loads the network and reads the capacities optimized
+  in the previous time step and add them to the
      network if they are still in operation (i.e., if they fulfill planning
      horizon < commissioned year + lifetime)
 
-  Then, the resulting network is saved in
-  ``results/run_name/networks/prenetworks_brownfield``.
+

doc/release_notes.rst

@@ -11,6 +11,14 @@ Release Notes
 Upcoming Release
 ================
 
+* Breaking change: The location of several network files has changed to
+  distinguish more clearly between solved and unsolved networks. Unsolved
+  networks that were previously stored in ``results/{run}/prenetworks`` are now
+  placed in ``resources/{run}/networks``. Solved networks that were previously
+  stored in ``results/{run}/postnetworks`` are now stored in
+  ``results/{run}/networks``. Brownfield networks are identified by a
+  ``_brownfield`` suffix in their filename.
+
 * Bugfix: The missing lifetime attribute was added to hydrogen steel tanks, which is important for pathway planning.
 
 * Bugfix: The length factor on the transmission lines haversine length was

rules/build_sector.smk

@@ -1218,18 +1218,19 @@ rule prepare_sector_network:
             "direct_heat_source_utilisation_profiles_base_s_{clusters}_{planning_horizons}.nc"
         ),
     output:
-        RESULTS
-        + "prenetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc"
+        ),
     threads: 1
     resources:
         mem_mb=2000,
     log:
-        RESULTS
-        + "logs/prepare_sector_network_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log",
+        logs(
+            "prepare_sector_network_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log"
+        ),
     benchmark:
-        (
-            RESULTS
-            + "benchmarks/prepare_sector_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}"
+        benchmarks(
+            "prepare_sector_network/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}"
         )
     conda:
         "../envs/environment.yaml"

rules/collect.smk

@@ -33,8 +33,9 @@ rule prepare_elec_networks:
 rule prepare_sector_networks:
     input:
         expand(
-            RESULTS
-            + "prenetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            resources(
+                "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc"
+            ),
             **config["scenario"],
             run=config["run"]["name"],
         ),
@@ -53,7 +54,7 @@ rule solve_sector_networks:
     input:
         expand(
             RESULTS
-            + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
             **config["scenario"],
             run=config["run"]["name"],
         ),

rules/common.smk

@@ -142,7 +142,7 @@ def solved_previous_horizon(w):
 
     return (
         RESULTS
-        + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_"
+        + "networks/base_s_{clusters}_{opts}_{sector_opts}_"
         + planning_horizon_p
         + ".nc"
     )

rules/postprocess.smk

@@ -29,7 +29,7 @@ if config["foresight"] != "perfect":
             transmission_limit=config_provider("electricity", "transmission_limit"),
         input:
             network=RESULTS
-            + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
             regions=resources("regions_onshore_base_s_{clusters}.geojson"),
         output:
             map=RESULTS
@@ -56,7 +56,7 @@ if config["foresight"] != "perfect":
             foresight=config_provider("foresight"),
         input:
             network=RESULTS
-            + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
             regions=resources("regions_onshore_base_s_{clusters}.geojson"),
         output:
             map=RESULTS
@@ -82,7 +82,7 @@ if config["foresight"] != "perfect":
             plotting=config_provider("plotting"),
         input:
             network=RESULTS
-            + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
             regions=resources("regions_onshore_base_s_{clusters}.geojson"),
         output:
             map=RESULTS
@@ -119,7 +119,7 @@ if config["foresight"] == "perfect":
             plotting=config_provider("plotting"),
         input:
             network=RESULTS
-            + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc",
+            + "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc",
             regions=resources("regions_onshore_base_s_{clusters}.geojson"),
         output:
             unpack(output_map_year),
@@ -143,7 +143,7 @@ rule make_summary:
     input:
         networks=expand(
             RESULTS
-            + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+            + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
             **config["scenario"],
             allow_missing=True,
         ),

rules/solve_myopic.smk

@@ -12,8 +12,9 @@ rule add_existing_baseyear:
         heat_pump_sources=config_provider("sector", "heat_pump_sources"),
         energy_totals_year=config_provider("energy", "energy_totals_year"),
     input:
-        network=RESULTS
-        + "prenetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        network=resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc"
+        ),
         powerplants=resources("powerplants_s_{clusters}.csv"),
         busmap_s=resources("busmap_base_s.csv"),
         busmap=resources("busmap_base_s_{clusters}.csv"),
@@ -29,8 +30,9 @@ rule add_existing_baseyear:
         ),
         heating_efficiencies=resources("heating_efficiencies.csv"),
     output:
-        RESULTS
-        + "prenetworks-brownfield/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc"
+        ),
     wildcard_constraints:
         # TODO: The first planning_horizon needs to be aligned across scenarios
         # snakemake does not support passing functions to wildcard_constraints
@@ -40,12 +42,12 @@ rule add_existing_baseyear:
     resources:
         mem_mb=2000,
     log:
-        RESULTS
-        + "logs/add_existing_baseyear_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log",
+        logs(
+            "add_existing_baseyear_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log"
+        ),
     benchmark:
-        (
-            RESULTS
-            + "benchmarks/add_existing_baseyear/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}"
+        benchmarks(
+            "add_existing_baseyear/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}"
         )
     conda:
         "../envs/environment.yaml"
@@ -76,24 +78,26 @@ rule add_brownfield:
         unpack(input_profile_tech_brownfield),
         simplify_busmap=resources("busmap_base_s.csv"),
         cluster_busmap=resources("busmap_base_s_{clusters}.csv"),
-        network=RESULTS
-        + "prenetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        network=resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc"
+        ),
         network_p=solved_previous_horizon,  #solved network at previous time step
         costs=resources("costs_{planning_horizons}.csv"),
         cop_profiles=resources("cop_profiles_base_s_{clusters}_{planning_horizons}.nc"),
     output:
-        RESULTS
-        + "prenetworks-brownfield/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc"
+        ),
     threads: 4
     resources:
         mem_mb=10000,
     log:
-        RESULTS
-        + "logs/add_brownfield_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log",
+        logs(
+            "add_brownfield_base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log"
+        ),
     benchmark:
-        (
-            RESULTS
-            + "benchmarks/add_brownfield/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}"
+        benchmarks(
+            "add_brownfield/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}"
         )
     conda:
         "../envs/environment.yaml"
@@ -114,12 +118,13 @@ rule solve_sector_network_myopic:
         ),
         custom_extra_functionality=input_custom_extra_functionality,
     input:
-        network=RESULTS
-        + "prenetworks-brownfield/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        network=resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc"
+        ),
         costs=resources("costs_{planning_horizons}.csv"),
     output:
         network=RESULTS
-        + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
         config=RESULTS
         + "configs/config.base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.yaml",
     shadow:

rules/solve_overnight.smk

@@ -13,11 +13,12 @@ rule solve_sector_network:
         ),
         custom_extra_functionality=input_custom_extra_functionality,
     input:
-        network=RESULTS
-        + "prenetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        network=resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc"
+        ),
     output:
         network=RESULTS
-        + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
         config=RESULTS
         + "configs/config.base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.yaml",
     shadow:

rules/solve_perfect.smk

@@ -10,8 +10,9 @@ rule add_existing_baseyear:
         heat_pump_sources=config_provider("sector", "heat_pump_sources"),
         energy_totals_year=config_provider("energy", "energy_totals_year"),
     input:
-        network=RESULTS
-        + "prenetworks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        network=resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc"
+        ),
         powerplants=resources("powerplants_s_{clusters}.csv"),
         busmap_s=resources("busmap_base_s.csv"),
         busmap=resources("busmap_base_s_{clusters}.csv"),
@@ -28,8 +29,9 @@ rule add_existing_baseyear:
         existing_heating="data/existing_infrastructure/existing_heating_raw.csv",
         heating_efficiencies=resources("heating_efficiencies.csv"),
     output:
-        RESULTS
-        + "prenetworks-brownfield/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc",
+        resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc"
+        ),
     wildcard_constraints:
         planning_horizons=config["scenario"]["planning_horizons"][0],  #only applies to baseyear
     threads: 1
@@ -52,8 +54,7 @@ rule add_existing_baseyear:
 
 def input_network_year(w):
     return {
-        f"network_{year}": RESULTS
-        + "prenetworks/base_s_{clusters}_{opts}_{sector_opts}"
+        f"network_{year}": resources("networks/base_s_{clusters}_{opts}_{sector_opts}")
         + f"_{year}.nc"
         for year in config_provider("scenario", "planning_horizons")(w)[1:]
     }
@@ -66,16 +67,15 @@ rule prepare_perfect_foresight:
     input:
         unpack(input_network_year),
         brownfield_network=lambda w: (
-            RESULTS
-            + "prenetworks-brownfield/"
-            + "base_s_{clusters}_{opts}_{sector_opts}_"
-            + "{}.nc".format(
+            resources("networks/base_s_{clusters}_{opts}_{sector_opts}_")
+            + "{}_brownfield.nc".format(
                 str(config_provider("scenario", "planning_horizons", 0)(w))
             )
         ),
     output:
-        RESULTS
-        + "prenetworks-brownfield/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc",
+        resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc"
+        ),
     threads: 2
     resources:
         mem_mb=10000,
@@ -100,12 +100,13 @@ rule solve_sector_network_perfect:
         ),
         custom_extra_functionality=input_custom_extra_functionality,
     input:
-        network=RESULTS
-        + "prenetworks-brownfield/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc",
+        network=resources(
+            "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc"
+        ),
         costs=resources("costs_2030.csv"),
     output:
         network=RESULTS
-        + "postnetworks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc",
+        + "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc",
         config=RESULTS
         + "configs/config.base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.yaml",
     threads: solver_threads
@@ -134,7 +135,7 @@ rule solve_sector_network_perfect:
 def input_networks_make_summary_perfect(w):
     return {
         f"networks_s_{clusters}_{opts}_{sector_opts}": RESULTS
-        + f"postnetworks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc"
+        + f"networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc"
         for clusters in config_provider("scenario", "clusters")(w)
         for opts in config_provider("scenario", "opts")(w)
         for sector_opts in config_provider("scenario", "sector_opts")(w)

scripts/make_summary.py

@@ -707,7 +707,7 @@ def to_csv(df):
     networks_dict = {
         (cluster, opt + sector_opt, planning_horizon): "results/"
         + snakemake.params.RDIR
-        + f"/postnetworks/base_s_{cluster}_{opt}_{sector_opt}_{planning_horizon}.nc"
+        + f"/networks/base_s_{cluster}_{opt}_{sector_opt}_{planning_horizon}.nc"
         for cluster in snakemake.params.scenario["clusters"]
         for opt in snakemake.params.scenario["opts"]
         for sector_opt in snakemake.params.scenario["sector_opts"]

scripts/make_summary_perfect.py

@@ -724,7 +724,7 @@ def to_csv(df):
     networks_dict = {
         (clusters, opts + sector_opts): "results/"
         + run
-        + f"postnetworks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc"
+        + f"networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc"
         for clusters in snakemake.config["scenario"]["clusters"]
         for opts in snakemake.config["scenario"]["opts"]
         for sector_opts in snakemake.config["scenario"]["sector_opts"]