Powerplant data functions

.gitignore

@@ -142,4 +142,5 @@ workflow/.DS_Store
 workflow/scripts/.DS_Store
 
 # Environment credentials
-.env
+.env
+workflow/scripts/osemosys_global/powerplant_data_test.py

config/config.yaml

@@ -55,7 +55,16 @@ user_defined_capacity:
   #              first_year_of_expansion, 
   #              build_rate_per_year, 
   #              cost]
-  TRNINDEAINDNE: [0, 2030, "open", 2030, 10, 861]
+  PWRCOAINDWE01: [8, 2000, "open", 2025, 5, 1100]
+
+user_defined_capacity_transmission:
+  # technology: [capacity, 
+  #              first_year, 
+  #              "fixed/open", 
+  #              first_year_of_expansion, 
+  #              build_rate_per_year, 
+  #              cost]
+  TRNINDEAINDNE: [5, 1975, "open", 2030, 10, 861]
 
 nodes_to_add:
  #- "AAAXX" where AAA is a 3-letter country code, 

workflow/rules/preprocess.smk

@@ -15,22 +15,36 @@ demand_figures = [
 # output script files
 
 power_plant_files = [
+    'powerplant/CapitalCost',
+    'powerplant/FixedCost',
+    'powerplant/CapacityToActivityUnit',
+    'powerplant/OperationalLife',
+    'powerplant/TotalAnnualMaxCapacityInvestment',
+    'powerplant/TotalAnnualMinCapacityInvestment',
+    'powerplant/FUEL',
+    'powerplant/InputActivityRatio',
+    'powerplant/OutputActivityRatio',
+    'MODE_OF_OPERATION',
+    'REGION',
+    'powerplant/ResidualCapacity',
+    'powerplant/TECHNOLOGY',
+    'YEAR',
+    'AvailabilityFactor'
+    ]
+
+transmission_files = [
     'CapitalCost',
     'FixedCost',
     'CapacityToActivityUnit',
     'OperationalLife',
     'TotalAnnualMaxCapacityInvestment',
     'TotalAnnualMinCapacityInvestment',
     'TotalTechnologyModelPeriodActivityUpperLimit',
-    'FUEL',
     'InputActivityRatio',
     'OutputActivityRatio',
-    'MODE_OF_OPERATION',
-    'REGION',
     'ResidualCapacity',
     'TECHNOLOGY',
-    'YEAR',
-    'AvailabilityFactor'
+    'FUEL'
     ]
 
 timeslice_files = [
@@ -82,9 +96,10 @@ user_capacity_files = [
 ]
 
 GENERATED_CSVS = (
-    power_plant_files + timeslice_files + variable_cost_files + demand_files \
-    + emission_files + max_capacity_files
+    power_plant_files + transmission_files + timeslice_files + variable_cost_files \
+    + demand_files + emission_files + max_capacity_files
 )
+GENERATED_CSVS = [Path(x).stem for x in GENERATED_CSVS]
 EMPTY_CSVS = [x for x in OTOOLE_PARAMS if x not in GENERATED_CSVS]
 
 # rules
@@ -93,27 +108,68 @@ rule make_data_dir:
     output: directory('results/data')
     shell: 'mkdir -p {output}'
 
+
+def powerplant_cap_custom_csv() -> str:
+    if config["nodes_to_add"]:
+        return "resources/data/custom_nodes/residual_capacity.csv"
+    else:
+        return []   
+
 rule powerplant:
     message:
-        'Generating powerplant data...'
+        "Generating powerplant data..."
     input:
-        'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx',
-        'resources/data/weo_2020_powerplant_costs.csv',
-        'resources/data/operational_life.csv',
-        'resources/data/naming_convention_tech.csv',
-        'resources/data/Costs Line expansion.xlsx',
-        'resources/data/weo_region_mapping.csv',
-    params: 
-        trade = config['crossborderTrade'],
+        plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx',
+        weo_costs = 'resources/data/weo_2020_powerplant_costs.csv',
+        weo_regions = 'resources/data/weo_region_mapping.csv',
+        default_op_life = 'resources/data/operational_life.csv',
+        naming_convention_tech = 'resources/data/naming_convention_tech.csv',
+        line_data = 'resources/data/Costs Line expansion.xlsx',
+        default_av_factors = 'resources/data/availability_factors.csv',
+        custom_res_cap = powerplant_cap_custom_csv()
+    params:
         start_year = config['startYear'],
         end_year = config['endYear'],
-        invest_techs = config['no_invest_technologies']
+        region_name = 'GLOBAL',
+        custom_nodes = config['nodes_to_add'],
+        user_defined_capacity = config['user_defined_capacity'],
+        no_investment_techs = config['no_invest_technologies'],
+        output_data_dir = 'results/data',
+        input_data_dir = 'resources/data',
+        powerplant_data_dir = 'results/data/powerplant',
+
     output:
         csv_files = expand('results/data/{output_file}.csv', output_file = power_plant_files)
     log:
         log = 'results/logs/powerplant.log'
-    shell:
-        'python workflow/scripts/osemosys_global/powerplant_data.py 2> {log}'
+    script:
+        "../scripts/osemosys_global/powerplant/main.py"
+
+rule transmission:
+    message:
+        "Generating transmission data..."
+    input:
+        rules.powerplant.output.csv_files,
+        plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx',
+        default_op_life = 'resources/data/operational_life.csv',
+        line_data = 'resources/data/Costs Line expansion.xlsx',
+    params:
+        trade = config['crossborderTrade'],
+        start_year = config['startYear'],
+        end_year = config['endYear'],
+        region_name = 'GLOBAL',
+        custom_nodes = config['nodes_to_add'],
+        user_defined_capacity_transmission = config['user_defined_capacity_transmission'],
+        no_investment_techs = config['no_invest_technologies'],
+        output_data_dir = 'results/data',
+        input_data_dir = 'resources/data',
+        powerplant_data_dir = 'results/data/powerplant',
+    output:
+        csv_files = expand('results/data/{output_file}.csv', output_file = transmission_files)
+    log:
+        log = 'results/logs/transmission.log'
+    script:
+        "../scripts/osemosys_global/transmission/main.py"
 
 rule timeslice:
     message:

workflow/scripts/osemosys_global/custom.py

@@ -0,0 +1,48 @@
+"""Custom Nodes logic"""
+
+import pandas as pd
+import itertools
+
+
+def _get_custom_demand_expected(
+    nodes: list[str], start_year: int, end_year: int
+) -> pd.DataFrame:
+    """Gets formatted expected custom data"""
+
+    years = range(start_year, end_year + 1)
+
+    df = pd.DataFrame(
+        list(itertools.product(nodes, years)), columns=["CUSTOM_NODE", "YEAR"]
+    )
+    df["REGION"] = "GLOBAL"
+    df["FUEL"] = "ELC" + df["CUSTOM_NODE"] + "02"
+
+    return df
+
+
+def import_custom_demand_data(csv: str) -> pd.DataFrame:
+    """Gets all custom demand data"""
+    return pd.read_csv(csv)
+
+
+def get_custom_demand_data(
+    all_custom: pd.DataFrame, nodes: list[str], start_year: int, end_year: int
+) -> pd.DataFrame:
+    """Gets merged custom demand data"""
+
+    expected = _get_custom_demand_expected(nodes, start_year, end_year)
+
+    df = pd.merge(expected, all_custom, how="left", on=["CUSTOM_NODE", "YEAR"])
+    df = df[["REGION", "FUEL", "YEAR", "VALUE"]]
+
+    return df
+
+
+def merge_default_custom_data(
+    default: pd.DataFrame, custom: pd.DataFrame
+) -> pd.DataFrame:
+    assert default.columns.equals(custom.columns)
+    df = pd.concat([default, custom], ignore_index=True)
+    df["VALUE"] = df["VALUE"].round(2)
+    df = df.drop_duplicates(keep="first", subset=["REGION", "FUEL", "YEAR"])
+    return df

workflow/scripts/osemosys_global/powerplant/GEM_unused.py

@@ -0,0 +1,171 @@
+"""Function that calculates existing, planned and to be retired capacities based on GEM for coal and gas. CURRENTLY UNUSED"""
+
+import pandas as pd
+import os
+from scipy import spatial
+import numpy as np
+
+from constants import (
+    start_year,
+    input_data_dir
+)
+
+"""The output of the gem_cap function is currently unused. Implementation is being discussed so kept as standalone
+function for the time being."""
+def gem_cap(df_gen_base, tech_code_dict, op_life_dict):
+
+    # ### Calculate planned capacities based on the Global Energy Observatory datasets
+
+    # Add spatial mapping to link Global Energy Observatory region naming conventions with OSeMOSYS Global
+    df_gem_regions = pd.read_csv(os.path.join(input_data_dir,
+                                                  "gem_region_mapping.csv"), encoding = "ISO-8859-1")
+
+    # pull locations from existing powerplants in PLEXOS-World dataset
+    gen_locationsinput = pd.read_excel(os.path.join(input_data_dir, 
+                                         "PLEXOS_World_2015_Gold_V1.1.xlsx"), 
+                            sheet_name = "Attributes")
+
+    gen_lat = gen_locationsinput.loc[(gen_locationsinput['class'].isin(['Battery', 'Generator'])) & 
+                                                (gen_locationsinput['attribute'] == 'Latitude')].set_index('name')
+
+    gen_long = gen_locationsinput.loc[(gen_locationsinput['class'].isin(['Battery', 'Generator'])) & 
+                                                (gen_locationsinput['attribute'] == 'Longitude')].set_index('name')                                  
+
+    gen_locations = pd.DataFrame(gen_long.index).merge(
+        gen_lat['value'], left_on = 'name', right_index = True).merge(
+            gen_long['value'], left_on = 'name', right_index = True)
+
+    gen_locations = pd.merge(gen_locations, df_gen_base[['node', 'country_code', 'node_code', 'powerplant']], 
+                             left_on = 'name', right_on = 'powerplant', how = 'inner')
+
+    subcountries = gen_locations.loc[~gen_locations['node_code'].str.contains('XX')]['country_code'].str[:3].unique()
+
+    # Set column name dictionaries for different Global Energy Monitor (gem) input datasets
+    gem_coal_col = {'Country' : 'Country', 'Capacity (MW)' : 'VALUE', 
+                    'Status' : 'Status', 'Year' : 'Year_built',
+                    'RETIRED' : 'Year_retired', 'Planned Retire' : 'Year_retired_planned', 
+                    'Latitude' : 'Latitude', 'Longitude' : 'Longitude'}
+
+    gem_gas_col = {'Country' : 'Country', 'Capacity elec. (MW)' : 'VALUE', 
+                    'Status' : 'Status', 'Start year' : 'Year_built',
+                    'Retired year' : 'Year_retired', 'Planned retire' : 'Year_retired_planned', 
+                    'Latitude' : 'Latitude', 'Longitude' : 'Longitude', 
+                    'Technology' : 'Technology'}
+
+    # Set technology dictionary to match with OSeMOSYS global technologies
+    gem_gas_dict = {'CC' : 'CCG', 
+                    'GT' : 'OCG',
+                    'ICCC' : 'CCG',
+                    'ISCC' : 'CCG',
+                    'ST' : 'OCG',
+                    'AFC' : 'CCG'}
+
+    # Set which status criteria are used to filter datasets
+    new_criteria = ['operating', 'proposed', 'announced', 'pre-permit', 'permitted', 'construction']# 'shelved' & cancelled' not included
+    old_criteria = ['mothballed', 'retired', 'operating']# operating added because currently operating plants can already have an intended retirement year added
+
+    # Import gem Datasets
+    gem_coal = pd.read_excel(os.path.join(input_data_dir, 
+                                                     'Global-Coal-Plant-Tracker-Jan-2022.xlsx'),
+                                        sheet_name = 'Units', usecols = gem_coal_col.keys())
+
+
+    gem_gas = pd.read_excel(os.path.join(input_data_dir, 
+                                                     'Global-Gas-Plant-Tracker-Feb-2022.xlsx'),
+                                        sheet_name = 'Gas Units', usecols = gem_gas_col.keys())
+
+    # Add Technology columns and drop entries with no capacity values
+    gem_coal.rename(columns = gem_coal_col, inplace = True)
+    gem_coal['Technology'] = 'COA'
+    gem_coal = gem_coal[pd.to_numeric(gem_coal['VALUE'], errors='coerce').notnull()]
+
+    gem_gas.rename(columns = gem_gas_col, inplace = True)
+    gem_gas['Technology'] = gem_gas['Technology'].map(gem_gas_dict)
+    gem_gas = gem_gas[pd.to_numeric(gem_gas['VALUE'], errors='coerce').notnull()]
+
+    # For entries in the gem dataset with no specified gas technology same assumptions are applied as with OSeMOSYS Global
+    gem_gas.loc[
+       (gem_gas["Technology"].isna()) & (gem_gas["VALUE"].astype(float) > 130),
+       "Technology",
+    ] = "CCG"
+    gem_gas.loc[
+        (gem_gas["Technology"].isna()) & (gem_gas["VALUE"].astype(float) <= 130),
+        "Technology",
+    ] = "OCG"
+
+    # Combine different datasets
+    gem_all = pd.concat([gem_coal, gem_gas])
+
+    # Add spatial mapping
+    gem_all = gem_all.merge(df_gem_regions, left_on = 'Country', right_on = 'gem_region')
+
+    gem_concat = pd.DataFrame(columns = gem_all.columns)
+
+    # Matches the lat/longs of plants in the gem datasets with lat/longs of the nearest plants in PLEXOS-World. 
+    # The associated sub-country node of the nearest plant is assumed to be the node for the gem dataset entry. 
+    for a in subcountries:
+
+        gen_locations_sc = gen_locations.loc[gen_locations['country_code'].str.contains(a)].reset_index(drop = True)
+        gem_all_sc = gem_all.loc[gem_all['country_code'] == a].reset_index(drop = True)
+
+        source = spatial.KDTree(gen_locations_sc[['value_x', 'value_y']].to_numpy())
+
+        output = pd.DataFrame(source.query([gem_all_sc[['Latitude', 'Longitude']
+                                                                       ].to_numpy()])[1].transpose())
+
+        gem_all_sc = gem_all_sc.merge(output, left_index = True, right_index = True).set_index(0)
+        gem_all_sc = gem_all_sc.merge(gen_locations_sc[['node_code']], left_index = True, 
+                                      right_index = True)
+
+        gem_concat = pd.concat([gem_concat, gem_all_sc])
+
+    # Adds matched sub-country entries to original df and sets node codes.    
+    gem_all = gem_all.loc[~gem_all['country_code'].isin(subcountries)]
+    gem_all = pd.concat([gem_all, gem_concat], axis = 0).reset_index(drop = True)
+    gem_all['node_code'].fillna(gem_all['country_code'] + 'XX', inplace = True)
+
+    # Filters datframe for new plants by year entry
+    gem_all_new = gem_all.loc[(gem_all['Status'].isin(new_criteria)) & (gem_all['Year_built'].notna())
+                              ].reset_index(drop = True)
+
+    # Strips year entry to single value (last year taken if range is given e.g. 2025-)
+    gem_all_new['YEAR'] = np.where(gem_all_new['Year_built'].astype(str).str.len() == 4, gem_all_new['Year_built'], 
+                                   gem_all_new['Year_built'].str[-4:])
+
+    # Drops non-existing or non-numerical entries (e.g. 'no year found') and entries from < model_start_year
+    gem_all_new['YEAR'] = gem_all_new['YEAR'].apply(pd.to_numeric, errors = 'coerce')
+    gem_all_new = gem_all_new[(gem_all_new['YEAR'].notna()) & 
+                              (gem_all_new['YEAR'] > start_year)].reset_index()
+
+    gem_all_retired = gem_all.loc[(gem_all['Status'].isin(old_criteria)) & (gem_all['Year_retired'].notna()) | 
+                                  (gem_all['Year_retired_planned'].notna())]
+
+    # Pulls retirement OR planned retirement year
+    gem_all_retired['YEAR'] = np.where(gem_all_retired['Year_retired'].notna(), gem_all_retired['Year_retired'], 
+                                       gem_all_retired['Year_retired_planned'])
+
+    # Strips year entry to single value (last year taken if range is given e.g. 2025-2030)
+    gem_all_retired['YEAR'] = np.where(gem_all_retired['YEAR'].astype(str).str.len().isin({4,6}), gem_all_retired['YEAR'], 
+                                       gem_all_retired['YEAR'].str[-4:])
+
+    # Drops non-existing or non-numerical entries (e.g. 'no year found') and entries from < model_start_year
+    gem_all_retired['YEAR'] = gem_all_retired['YEAR'].apply(pd.to_numeric, errors = 'coerce')
+    gem_all_retired = gem_all_retired[(gem_all_retired ['YEAR'].notna()) & 
+                                      (gem_all_retired['YEAR'] > start_year)].reset_index()
+
+    # Group values by technology, node & year
+    gem_all_new_agg = gem_all_new.groupby(['node_code', 'Technology', 'YEAR'], 
+                                              as_index=False)['VALUE'].sum()
+
+    # Adds lifetime to planned capacities, calculates future retirement year and adds to retirement dataframe. 
+    tech_code_dict_inv = {v: k for k, v in tech_code_dict.items()}
+    gem_all_new_agg_oplife = gem_all_new_agg.copy()
+    gem_all_new_agg_oplife['operational_life'] =  gem_all_new_agg_oplife['Technology'
+                                                                         ].map(tech_code_dict_inv).map(op_life_dict)
+
+    gem_all_new_agg_oplife['YEAR'] = gem_all_new_agg_oplife['YEAR'] + gem_all_new_agg_oplife['operational_life']
+
+    gem_all_retired = pd.concat([gem_all_retired, gem_all_new_agg_oplife])
+
+    gem_all_retired_agg = gem_all_retired.groupby(['node_code', 'Technology', 'YEAR'], 
+                                              as_index=False)['VALUE'].sum()