From f3044f346df412955baf44094a6b700b0ba2c587 Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Fri, 6 Sep 2024 15:56:43 -0400 Subject: [PATCH 01/12] Removed unused packages/variables in powerplant_data --- .gitignore | 3 ++- .../scripts/osemosys_global/powerplant_data.py | 17 ----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index ed635069..d3eab62a 100644 --- a/.gitignore +++ b/.gitignore @@ -141,4 +141,5 @@ workflow/.DS_Store workflow/scripts/.DS_Store # Environment credentials -.env \ No newline at end of file +.env +workflow/scripts/osemosys_global/powerplant_data_test.py diff --git a/workflow/scripts/osemosys_global/powerplant_data.py b/workflow/scripts/osemosys_global/powerplant_data.py index ca096756..08228575 100644 --- a/workflow/scripts/osemosys_global/powerplant_data.py +++ b/workflow/scripts/osemosys_global/powerplant_data.py @@ -15,12 +15,10 @@ import os # from osemosys_global.configuration import ConfigFile, ConfigPaths from configuration import ConfigFile, ConfigPaths -import yaml from constants import SET_DTYPES from utils import apply_dtypes import logging logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) -import os from scipy import spatial def main(): @@ -111,8 +109,6 @@ def main(): ) - emissions = [] - # Technologies that will have 00 and 01 suffixes to represent PLEXOS # historical values and future values # duplicate_techs = ['CCG', 'OCG', 'COA'] @@ -346,12 +342,6 @@ def main(): # Convert total capacity from MW to GW df_res_cap['value'] = df_res_cap['value'].div(1000) - - df_res_cap_plot = df_res_cap[['node_code', - 'tech_code', - 'model_year', - 'value']] - # Rename 'model_year' to 'year' and 'total_capacity' to 'value' df_res_cap.rename({'model_year': 'YEAR', 'value': 'VALUE'}, @@ -538,9 +528,6 @@ def main(): gem_all_new_agg_oplife['YEAR'] = gem_all_new_agg_oplife['YEAR'] + gem_all_new_agg_oplife['operational_life'] gem_all_retired = pd.concat([gem_all_retired, gem_all_new_agg_oplife]) - - gem_all_retired_agg = gem_all_retired.groupby(['node_code', 'Technology', 'YEAR'], - as_index=False)['VALUE'].sum() # ### Add input and output activity ratios @@ -1569,10 +1556,6 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life #Replace VALUE with CAPEX df['VALUE'] = df['TECHNOLOGY'].map(capex_dict) - capex_df = df[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD for TRN df_max_cap_inv = pd.concat([df_max_cap_inv, max_cap_techs_df]).drop_duplicates() From d590bd3d0df4efef0f3b807868ed6725b5247008 Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Thu, 12 Sep 2024 13:14:58 -0400 Subject: [PATCH 02/12] First functional split of powerplant_data in functions -Split the 'main' in powerplant_data into individual functions to improve readability and the ease of debugging. --- .../osemosys_global/powerplant_data.py | 716 ++++++++++-------- 1 file changed, 411 insertions(+), 305 deletions(-) diff --git a/workflow/scripts/osemosys_global/powerplant_data.py b/workflow/scripts/osemosys_global/powerplant_data.py index 1a9ff4e7..814c9c9b 100644 --- a/workflow/scripts/osemosys_global/powerplant_data.py +++ b/workflow/scripts/osemosys_global/powerplant_data.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -# OSeMOSYS-PLEXOS global model: Powerplant data - # Import modules import warnings warnings.simplefilter(action='ignore', category=FutureWarning) @@ -11,9 +9,7 @@ pd.options.mode.chained_assignment = None # default='warn' import numpy as np import itertools -import urllib import os -# from osemosys_global.configuration import ConfigFile, ConfigPaths from configuration import ConfigFile, ConfigPaths from constants import SET_DTYPES from utils import apply_dtypes @@ -23,6 +19,9 @@ def main(): + """Main retrieves the script input data, sets user configurations + and calls the different functions as defined within the script.""" + # CONFIGURATION PARAMETERS config_paths = ConfigPaths() @@ -44,8 +43,6 @@ def main(): model_start_year = config.get('startYear') model_end_year = config.get('endYear') years = list(range(model_start_year, model_end_year + 1)) - - region_name = config.region_name tech_capacity = config.get('user_defined_capacity') custom_nodes = config.get('nodes_to_add') @@ -73,9 +70,19 @@ def main(): df_op_life = pd.read_csv(os.path.join(input_data_dir, "operational_life.csv") ) + + op_life_dict = dict(zip(list(df_op_life['tech']), + list(df_op_life['years']))) + df_tech_code = pd.read_csv(os.path.join(input_data_dir, "naming_convention_tech.csv") ) + + tech_list = list(df_tech_code['code']) + + tech_code_dict = dict(zip(list(df_tech_code['tech']), + list(df_tech_code['code']))) + df_trn_efficiencies = pd.read_excel(os.path.join(input_data_dir, "Costs Line expansion.xlsx"), sheet_name = 'Interface' @@ -93,12 +100,142 @@ def main(): "residual_capacity.csv") ) - - # Technologies that will have 00 and 01 suffixes to represent PLEXOS - # historical values and future values - # duplicate_techs = ['CCG', 'OCG', 'COA'] + else: + + df_custom_res_cap = pd.DataFrame() + + # USER INPUTS + + """Technologies that will have 00 and 01 suffixes to represent PLEXOS + historical values and future values.""" duplicate_techs = ['CCG', 'OCG'] + + + """ Add extra nodes which exist in 2050 but are not in the 2015 data""" + nodes_extra_list = ['AF-SOM', + 'AF-TCD', + 'AS-TLS', + 'EU-MLT', + 'NA-BLZ', + 'NA-HTI', + 'SA-BRA-J1', + 'SA-BRA-J2', + 'SA-BRA-J3', + 'SA-SUR'] + + mode_list = [1,2] + + thermal_fuel_list_oar = ['COA', + 'COG', + 'OCG', + 'CCG', + 'PET', + 'URN', + 'OIL', + 'OTH', + 'CCS' + ] + + thermal_fuels_list_mining = ['COA', + 'COG', + 'GAS', + 'PET', + 'URN', + 'OIL', + 'OTH' + ] + + thermal_fuel_list_iar = ['COA', + 'COG', + 'PET', + 'URN', + 'OIL', + 'OTH' + ] + + renewables_list = ['BIO', + 'GEO', + 'HYD', + 'SPV', + 'CSP', + 'WAS', + 'WAV', + 'WON', + 'WOF'] + + costs_dict = {'Biomass - waste incineration - CHP':'WAS', + 'Biomass Power plant':'BIO', + 'CCGT':'CCG', + 'CCGT - CHP':'COG', + 'Concentrating solar power':'CSP', + 'Gas turbine':'OCG', + 'Geothermal':'GEO', + 'Hydropower - large-scale':'HYD', + 'Marine':'WAV', + 'Nuclear':'URN', + 'Solar photovoltaics - Large scale':'SPV', + 'Steam Coal - SUBCRITICAL':'COA', + 'Steam Coal - SUPERCRITICAL':'COA', + 'Steam Coal - ULTRASUPERCRITICAL':'COA', + 'Wind onshore':'WON', + 'Wind offshore':'WOF', + 'Petroleum':'PET', + 'Oil':'OIL', + 'Other':'OTH', + 'IGCC + CCS':'CCS', + 'Coal* + CCS':'CCS',} # Added OIL, OTH, PET, WOF to WEO 2018 + + # CALL FUNCTIONS + + gen_table = generator_table(df, df_dict, model_start_year, custom_nodes, tech_code_dict, op_life_dict, nodes_extra_list) + + df_eff_node, df_eff_tech = average_efficiency(gen_table) + + df_res_cap, custom_techs = residual_capacity(gen_table, tech_list, df_tech_code, model_start_year, model_end_year, + custom_nodes, df_custom_res_cap, years, output_data_dir, duplicate_techs, region_name) + + gem_cap(gen_table, input_data_dir, model_start_year, tech_code_dict, op_life_dict) + + df_ratios = activity_master_start(gen_table, custom_nodes, nodes_extra_list, years, duplicate_techs, mode_list) + + df_oar, df_oar_base = activity_output_pwr(df_ratios, thermal_fuel_list_oar, region_name) + + df_iar_base = activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, df_eff_node, df_eff_tech, region_name) + + df_oar_upstream, df_oar_int = activity_upstream(df_iar_base, renewables_list, thermal_fuels_list_mining) + + df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar = activity_transmission(df_oar_base, df, model_start_year, + model_end_year, df_trn_efficiencies, + region_name) + + df_oar_final, df_iar_final = activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, + df_int_trn_oar, df_iar_base, df_iar_trn, df_int_trn_iar, + duplicate_techs, output_data_dir) + + df_costs = costs_pwr(df_weo_data, costs_dict) + + df_trans_capex, df_trans_fix = costs_transmission(years, df_oar_final, region_name, output_data_dir) + + costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, df_trans_fix, output_data_dir) + + capact(df_oar_final, output_data_dir) + + activity_transmission_limit(cross_border_trade, df_oar_final, output_data_dir) + + op_life_trn, op_life_out = op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict, region_name) + + op_life_transmission(op_life_trn, op_life_out, output_data_dir, region_name) + + cap_investment_constraints(df_iar_final, years, config, output_data_dir, region_name) + + output_sets(custom_nodes, df_oar_final, output_data_dir, custom_techs, years, mode_list, region_name) + + user_defined_capacity(region_name, years, output_data_dir, tech_capacity, op_life_dict) + + availability_factor(region_name, years, output_data_dir, df_af) +def generator_table(df, df_dict, model_start_year, custom_nodes, tech_code_dict, op_life_dict, nodes_extra_list): + # Create main generator table gen_cols_1 = ["child_class", "child_object", "property", "value"] df_gen = df[gen_cols_1] @@ -125,28 +262,28 @@ def main(): df_dict_nodes = df_dict[df_dict["collection"] == "Nodes"] df_dict_nodes = df_dict_nodes[["powerplant", "child_object"]] df_dict_2 = pd.merge(df_dict_fuel, df_dict_nodes, how="outer", on="powerplant") - - + + ## Merge original generator dataframe with nodes and fuels df_gen_2 = pd.merge(df_gen_2, df_dict_2, how="outer", on="powerplant") df_gen_2.rename( {"child_object_x": "fuel", "child_object_y": "node"}, axis=1, inplace=True ) - + ## Extract start year from Commission Date df_gen_2["Commission Date"] = (pd.TimedeltaIndex(df_gen_2["Commission Date"].astype(int), unit='d') + datetime(1900, 1, 1)) df_gen_2["start_year"] = df_gen_2["Commission Date"].dt.year df_gen_2.drop("Commission Date", axis=1, inplace=True) - + ## Calculate efficiency from heat rate. Units of heat rate in MJ/kWh df_gen_2["efficiency"] = 3.6 / df_gen_2["Heat Rate"].astype(float) df_gen_2.drop("Heat Rate", axis=1, inplace=True) - + ## Calcluate years of operation from start year until 2015 df_gen_2["years_of_operation"] = model_start_year - df_gen_2["start_year"] - + ## Fix blank spaces in 'fuels' columns. Appearing for 'Oil' powerplants in certain countries df_gen_2.loc[df_gen_2["fuel"].isna(), "fuel"] = ( df_gen_2["node"].str.split("-").str[:2].str.join("-") @@ -154,12 +291,10 @@ def main(): + df_gen_2["powerplant"].str.split("_", expand=True)[1] ) - ## Create column for technology df_gen_2["technology"] = df_gen_2["powerplant"].str.split("_").str[1] df_gen_2["technology"] = df_gen_2["technology"].str.title() - ## Divide Gas into CCGT and OCGT based on max capacity df_gen_2.loc[ (df_gen_2["technology"] == "Gas") & (df_gen_2["Max Capacity"].astype(float) > 130), @@ -170,7 +305,6 @@ def main(): "technology", ] = "Gas-OCGT" - # Create table with aggregated capacity df_gen_agg_node = df_gen_2[df_gen_2['start_year']<=model_start_year] df_gen_agg_node = df_gen_agg_node.groupby(['node', 'technology'], @@ -180,23 +314,9 @@ def main(): values='total_capacity').fillna(0).reset_index() df_gen_agg_node.drop('Sto', axis=1, inplace=True) # Drop 'Sto' technology. Only for USA. - - # Add extra nodes which exist in 2050 but are not in the 2015 data - if custom_nodes: - node_list = list(df_gen_agg_node['node'].unique()) + custom_nodes - else: - node_list = list(df_gen_agg_node['node'].unique()) + nodes_extra_df = pd.DataFrame(columns=['node']) - nodes_extra_list = ['AF-SOM', - 'AF-TCD', - 'AS-TLS', - 'EU-MLT', - 'NA-BLZ', - 'NA-HTI', - 'SA-BRA-J1', - 'SA-BRA-J2', - 'SA-BRA-J3', - 'SA-SUR',] + nodes_extra_df['node'] = nodes_extra_list df_gen_agg_node = pd.concat( @@ -209,11 +329,6 @@ def main(): df_gen_2['region_code'] = df_gen_2['node'].str[:2] df_gen_2['country_code'] = df_gen_2['node'].str[3:] - - # ### Add operational life column - op_life_dict = dict(zip(list(df_op_life['tech']), - list(df_op_life['years']))) - df_gen_2['operational_life'] = df_gen_2['technology'].map(op_life_dict) df_gen_2['retirement_year_data'] = (df_gen_2['operational_life'] + df_gen_2['start_year']) @@ -233,10 +348,6 @@ def main(): df_gen_2.loc[df_gen_2['retirement_diff'] <= 0, 'retirement_year_model'] = df_gen_2['retirement_year_data'] - # ### Add naming convention - tech_code_dict = dict(zip(list(df_tech_code['tech']), - list(df_tech_code['code']))) - tech_list = list(df_tech_code['code']) df_gen_2['tech_code'] = df_gen_2['technology'].map(tech_code_dict) df_gen_2.loc[df_gen_2['node'].str.len() <= 6, @@ -253,41 +364,49 @@ def main(): ) df_gen_2 = df_gen_2.loc[~df_gen_2['tech_code'].isna()] + + return df_gen_2 +def average_efficiency(df_gen_2): # ### Calculate average InputActivityRatio by node+technology and only by technology df_eff = df_gen_2[['node_code', 'efficiency', 'tech_code']] - + # Change IAR for CSP value taken from PLEXOS to 1.0 df_eff.loc[df_eff['tech_code']=='CSP', 'efficiency'] = 1 # Change IAR for URN value taken from PLEXOS to 2.2 (45%) df_eff.loc[df_eff['tech_code']=='URN', 'efficiency'] = 0.45 - + # Average efficiency by node and technology df_eff_node = df_eff.groupby(['tech_code', 'node_code'], as_index = False).mean() - + df_eff_node['node_average_iar'] = ((1 / df_eff_node['efficiency']). round(2)) - + df_eff_node.drop('efficiency', axis = 1, inplace = True) - + # Average efficiency by technology df_eff_tech = df_eff.drop(columns="node_code").groupby('tech_code', as_index = False).mean() - + df_eff_tech['tech_average_iar'] = ((1 / df_eff_tech['efficiency']). round(2)) - + df_eff_tech.drop('efficiency', axis = 1, inplace = True) + return df_eff_node, df_eff_tech + +def residual_capacity(df_gen_2, tech_list, df_tech_code, model_start_year, model_end_year, + custom_nodes, df_custom_res_cap, years, output_data_dir, duplicate_techs, + region_name): # ### Calculate residual capacity res_cap_cols = [ @@ -351,15 +470,19 @@ def main(): tech_list) df_res_cap = pd.concat([df_res_cap, df_res_cap_custom]) - # df_res_cap = apply_dtypes(df_res_cap, "Residual Capacity") df_res_cap.drop_duplicates(subset=['REGION','TECHNOLOGY','YEAR'], keep='last', inplace=True) df_res_cap = df_res_cap.loc[(df_res_cap['TECHNOLOGY'].str.startswith('PWR')) & (~df_res_cap['TECHNOLOGY'].str.endswith('00'))] + df_res_cap.to_csv(os.path.join(output_data_dir, "ResidualCapacity.csv"), index=None) + + return df_res_cap, custom_techs if custom_nodes else df_res_cap + +def gem_cap(df_gen_2, input_data_dir, model_start_year, tech_code_dict, op_life_dict): # ### Calculate planned capacities based on the Global Energy Observatory datasets @@ -513,6 +636,11 @@ def main(): gem_all_new_agg_oplife['YEAR'] = gem_all_new_agg_oplife['YEAR'] + gem_all_new_agg_oplife['operational_life'] gem_all_retired = pd.concat([gem_all_retired, gem_all_new_agg_oplife]) + + gem_all_retired_agg = gem_all_retired.groupby(['node_code', 'Technology', 'YEAR'], + as_index=False)['VALUE'].sum() + +def activity_master_start(df_gen_2, custom_nodes, nodes_extra_list, years, duplicate_techs, mode_list): # ### Add input and output activity ratios @@ -531,8 +659,6 @@ def main(): master_fuel_list = list(df_gen_2['tech_code'].unique()) master_fuel_list.append('CCS') - mode_list = [1,2] - df_ratios = pd.DataFrame(list(itertools.product(node_list, master_fuel_list, mode_list, @@ -543,39 +669,13 @@ def main(): df_ratios = createPwrTechs(df_ratios, duplicate_techs) - thermal_fuel_list = ['COA', - 'COG', - 'OCG', - 'CCG', - 'PET', - 'URN', - 'OIL', - 'OTH', - 'CCS' - ] - - thermal_fuel_list_iar = ['COA', - 'COG', - 'PET', - 'URN', - 'OIL', - 'OTH' - ] - - renewables_list = ['BIO', - 'GEO', - 'HYD', - 'SPV', - 'CSP', - 'WAS', - 'WAV', - 'WON', - 'WOF'] + return df_ratios +def activity_output_pwr(df_ratios, thermal_fuel_list_oar, region_name): # #### OutputActivityRatio - Power Generation Technologies df_oar = df_ratios.copy() - mask = df_oar['TECHNOLOGY'].apply(lambda x: x[3:6] in thermal_fuel_list) + mask = df_oar['TECHNOLOGY'].apply(lambda x: x[3:6] in thermal_fuel_list_oar) df_oar['FUEL'] = 0 df_oar['FUEL'][mask] = 1 df_oar = df_oar.loc[~((df_oar['MODE_OF_OPERATION'] > 1) & @@ -585,17 +685,22 @@ def main(): '01' ) df_oar['VALUE'] = 1 - + # Add 'REGION' column and fill 'GLOBAL' throughout df_oar['REGION'] = region_name - + # Select columns for final output table - df_oar_final = df_oar[['REGION', + df_oar_base = df_oar[['REGION', 'TECHNOLOGY', 'FUEL', 'MODE_OF_OPERATION', 'YEAR', 'VALUE',]] + + return df_oar, df_oar_base + +def activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, + df_eff_node, df_eff_tech, region_name): # #### InputActivityRatio - Power Generation Technologies # Copy OAR table with all columns to IAR @@ -674,39 +779,35 @@ def main(): df_iar['REGION'] = region_name # Select columns for final output table - df_iar_final = df_iar[['REGION', + df_iar_base = df_iar[['REGION', 'TECHNOLOGY', 'FUEL', 'MODE_OF_OPERATION', 'YEAR', 'VALUE',]] + + return df_iar_base - # #### OutputActivityRatios - Upstream - - thermal_fuels = ['COA', - 'COG', - 'GAS', - 'PET', - 'URN', - 'OIL', - 'OTH' - ] +def activity_upstream(df_iar_base, renewables_list, thermal_fuels_list_mining): + # #### OutputActivityRatios - Upstream + # We have to create a technology to produce every fuel that is input into any of the power technologies: - - df_oar_upstream = df_iar_final.copy() - + + df_oar_upstream = df_iar_base.copy() + # All mining and resource technologies have an OAR of 1... df_oar_upstream['VALUE'] = 1 - + # Renewables - set the technology as RNW + FUEL df_oar_upstream.loc[df_oar_upstream['FUEL'].str[0:3].isin(renewables_list), 'TECHNOLOGY'] = 'RNW'+df_oar_upstream['FUEL'] - + # If the fuel is a thermal fuel, we need to create the OAR for the mining technology... BUT NOT FOR THE INT FUELS... - df_oar_upstream.loc[df_oar_upstream['FUEL'].str[0:3].isin(thermal_fuels) & ~(df_oar_upstream['FUEL'].str[3:6] == "INT"), + df_oar_upstream.loc[df_oar_upstream['FUEL'].str[0:3].isin(thermal_fuels_list_mining) & + ~(df_oar_upstream['FUEL'].str[3:6] == "INT"), 'TECHNOLOGY'] = 'MIN'+df_oar_upstream['FUEL'] - + # Above should get all the outputs for the MIN technologies, but we need to adjust the mode 2 ones to just the fuel code (rather than MINCOAINT) df_oar_upstream.loc[df_oar_upstream['MODE_OF_OPERATION']==2, 'TECHNOLOGY'] = 'MIN'+df_oar_upstream['FUEL'].str[0:3]+df_oar_upstream['TECHNOLOGY'].str[6:9] @@ -715,59 +816,62 @@ def main(): # Now remove the duplicate fuels that the above created (because there's now a COA for each country, not each region, and GAS is repeated twice for each region as well): df_oar_upstream.drop_duplicates(keep='first',inplace=True) - + # Now we have to create the MINXXXINT technologies. They are all based on the MODE_OF_OPERATION == 2: df_oar_int = pd.DataFrame(df_oar_upstream.loc[df_oar_upstream['MODE_OF_OPERATION'] == 2, :]) - + # At this point we should have only the internationally traded fuels since they're all mode 2. So we can make the tech MINXXXINT and that's that. df_oar_int['TECHNOLOGY'] = 'MIN'+df_oar_int['FUEL']+'INT' # And rename the fuel to XXXINT df_oar_int['FUEL'] = df_oar_int['FUEL']+'INT' df_oar_int['MODE_OF_OPERATION'] = 1 # This is probably not strictly necessary as long as they're always the same in and out... - + # and de-duplicate this list: df_oar_int.drop_duplicates(keep='first',inplace=True) - # #### Input Activity Ratios - Upstream - + # All we need to do is take in the thermal fuels for the MINXXXINT technologies. This already exists as df_oar_int with the XXINT fuel so we can simply copy that: df_iar_int = df_oar_int.copy() df_iar_int['FUEL'] = df_iar_int['FUEL'].str[0:3] + + return df_oar_upstream, df_oar_int +def activity_transmission(df_oar_base, df, model_start_year, model_end_year, + df_trn_efficiencies, region_name): # #### Downstream Activity Ratios - + # Build transmission system outputs - - df_iar_trn = df_oar_final.copy() - + + df_iar_trn = df_oar_base.copy() + # Change the technology name to PWRTRNXXXXX df_iar_trn["TECHNOLOGY"] = "PWRTRN" + df_iar_trn["FUEL"].str[3:8] # Make all modes of operation 1 df_iar_trn["MODE_OF_OPERATION"] = 1 # And remove all the duplicate entries df_iar_trn.drop_duplicates(keep="first", inplace=True) - + # OAR for transmission technologies is IAR, but the fuel is 02 instead of 01: df_oar_trn = df_iar_trn.copy() df_oar_trn["FUEL"] = df_oar_trn["FUEL"].str[0:8] + "02" - + # Build international transmission system from original input data, but for Line rather than Generator: int_trn_cols = ["child_class", "child_object", "property", "value"] df_int_trn = df[int_trn_cols] df_int_trn = df_int_trn[df_int_trn["child_class"] == "Line"] - + # For IAR and OAR we can drop the value: df_int_trn = df_int_trn.drop(["child_class", "value"], axis=1) - + # Create MofO column based on property: df_int_trn["MODE_OF_OPERATION"] = 1 df_int_trn.loc[df_int_trn["property"] == "Min Flow", "MODE_OF_OPERATION"] = 2 - + # Use the child_object column to build the technology names: df_int_trn["codes"] = df_int_trn["child_object"].str.split(pat="-") - + # If there are only two locations, then the node is XX df_int_trn.loc[df_int_trn["codes"].str.len() == 2, "TECHNOLOGY"] = ( "TRN" + df_int_trn["codes"].str[0] + "XX" + df_int_trn["codes"].str[1] + "XX" @@ -802,11 +906,11 @@ def main(): + df_int_trn["codes"].str[2] + "XX" ) - + # Set the value (of either IAR or OAR) to 1 df_int_trn["VALUE"] = 1 df_int_trn["REGION"] = region_name - + df_int_trn = df_int_trn.drop(["property", "child_object", "codes"], axis=1) df_int_trn["YEAR"] = model_start_year @@ -816,11 +920,11 @@ def main(): df_int_trn_new = df_int_trn_new.explode(column="YEAR") df_int_trn = pd.concat([df_int_trn, df_int_trn_new]).reset_index(drop=True) - + # Now create the input and output activity ratios df_int_trn_oar = df_int_trn.copy() df_int_trn_iar = df_int_trn.copy() - + # IAR Mode 1 is input from first country: df_int_trn_iar.loc[df_int_trn_iar["MODE_OF_OPERATION"] == 1, "FUEL"] = ( "ELC" + df_int_trn_iar["TECHNOLOGY"].str[3:8] + "02" @@ -838,7 +942,7 @@ def main(): df_int_trn_oar.loc[df_int_trn_oar["MODE_OF_OPERATION"] == 1, "FUEL"] = ( "ELC" + df_int_trn_oar["TECHNOLOGY"].str[8:13] + "01" ) - + # Drop unneeded columns df_trn_efficiencies = df_trn_efficiencies.drop( [ @@ -857,24 +961,30 @@ def main(): ], axis=1, ) - + # Drop NaN values df_trn_efficiencies = df_trn_efficiencies.dropna(subset=["From"]) - + # Create tech column from To and From Codes: df_trn_efficiencies = format_transmission_name(df_trn_efficiencies) - + # Rename column 'VALUES' df_trn_efficiencies = df_trn_efficiencies.rename(columns={"Losses": "VALUE"}) - + # And adjust OAR values to be output amounts vs. losses: df_trn_efficiencies['VALUE'] = 1.0 - df_trn_efficiencies['VALUE'] - + # and add values into OAR matrix df_int_trn_oar = df_int_trn_oar.drop(["VALUE"], axis=1) df_int_trn_oar = pd.merge( df_int_trn_oar, df_trn_efficiencies, how="outer", on="TECHNOLOGY" ) + + return df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar + +def activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, + df_int_trn_oar, df_iar_final, df_iar_trn, df_int_trn_iar, + duplicate_techs, output_data_dir): # #### Output IAR and OAR @@ -882,14 +992,14 @@ def main(): df_oar_final = pd.concat( [ - df_oar_final, + df_oar_base, # If we want to split transmission to different script might have to not used df_oar_final as input to this function. df_oar_upstream, df_oar_int, df_oar_trn, df_int_trn_oar, ] ).dropna() - + # Select columns for final output table df_oar_final = df_oar_final[['REGION', 'TECHNOLOGY', @@ -897,7 +1007,7 @@ def main(): 'MODE_OF_OPERATION', 'YEAR', 'VALUE',]] - + df_iar_final = pd.concat( [ df_iar_final, @@ -905,7 +1015,7 @@ def main(): df_int_trn_iar, ] ).dropna() - + # Select columns for final output table df_iar_final = df_iar_final[['REGION', 'TECHNOLOGY', @@ -913,18 +1023,17 @@ def main(): 'MODE_OF_OPERATION', 'YEAR', 'VALUE']] - + # Add iar for techs not using PLEXOS values df_iar_newTechs = duplicatePlexosTechs(df_iar_final, duplicate_techs) for duplicate_tech in duplicate_techs: df_new_iar = newIar(df_iar_newTechs, duplicate_tech) df_iar_final = pd.concat([df_iar_final, df_new_iar]) - + # Add oar for techs not using PLEXOS values df_oar_newTechs = duplicatePlexosTechs(df_oar_final, duplicate_techs) df_oar_final = pd.concat([df_oar_final, df_oar_newTechs]) - # df_oar_final = apply_dtypes(df_oar_final, "OutputActivityRatio") df_oar_final.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'FUEL', @@ -932,6 +1041,7 @@ def main(): 'YEAR'], keep='last', inplace=True) + df_oar_final.to_csv(os.path.join(output_data_dir, "OutputActivityRatio.csv"), index=None) @@ -940,6 +1050,10 @@ def main(): df_iar_final.to_csv(os.path.join(output_data_dir, "InputActivityRatio.csv"), index=None) + + return df_oar_final, df_iar_final + +def costs_pwr(df_weo_data, costs_dict): # ### Costs: Capital, fixed, and variable @@ -979,39 +1093,121 @@ def main(): ] ) df_costs['YEAR'] = df_costs['YEAR'].astype(int) - costs_dict = {'Biomass - waste incineration - CHP':'WAS', - 'Biomass Power plant':'BIO', - 'CCGT':'CCG', - 'CCGT - CHP':'COG', - 'Concentrating solar power':'CSP', - 'Gas turbine':'OCG', - 'Geothermal':'GEO', - 'Hydropower - large-scale':'HYD', - 'Marine':'WAV', - 'Nuclear':'URN', - 'Solar photovoltaics - Large scale':'SPV', - 'Steam Coal - SUBCRITICAL':'COA', - 'Steam Coal - SUPERCRITICAL':'COA', - 'Steam Coal - ULTRASUPERCRITICAL':'COA', - 'Wind onshore':'WON', - 'Wind offshore':'WOF', - 'Petroleum':'PET', - 'Oil':'OIL', - 'Other':'OTH', - 'IGCC + CCS':'CCS', - 'Coal* + CCS':'CCS',} # Added OIL, OTH, PET, WOF to WEO 2018 df_costs = df_costs.loc[df_costs['technology'].isin(costs_dict.keys())] df_costs['technology_code'] = df_costs['technology'].replace(costs_dict) - weo_regions_dict = dict([(k, v) - for k, v - in zip(df_weo_regions['technology_code'], - df_weo_regions['weo_region'] - ) - ] - ) + return df_costs + +def format_transmission_name(df): + '''Formats PLEXOS transmission names into OSeMOSYS Global names. + + Args: + :param df: Pandas DataFrame with columns 'From' and 'To' describing the + transmission from and to contries. ie. + + Returns: + :param df: Same as df_in, except the 'From' and 'To' columns are replaced + with a single 'TECHNOLOGY' column holding OSeMOSYS Global + naming conventions + + Example: + df = pd.DataFrame( + [[AF-COD, AF-COG, 0.001], + [EU-AUT, EU-SVK, 0.004], + [AS-LBN, AS-SYR, 0.006]], + columns = ['From', 'To', 'Losses'] + ) + pd.DataFrame( + [[0.001,TRNCODXXCOGXX], + [0.004,TRNAUTXXSVKXX], + [0.006,TRNLBNXXSYRXX]] + columns = ['Losses','TECHNOLOGY'])''' + + # If from column has length 6 then it's the last three chars plus XX + df.loc[df["From"].str.len() == 6, "From"] = (df["From"].str[3:6] + "XX") + + # If from column has length 9 then it's the 3:6 and 7:9 three chars plus XX + df.loc[df["From"].str.len() == 9, "From"] = ( + df["From"].str[3:6] + df["From"].str[7:9]) + + # If to column has length 6 then it's the last three chars plus XX + df.loc[df["To"].str.len() == 6, "To"] = (df["To"].str[3:6] + "XX") + + # If to column has length 9 then it's the 3:6 and 7:9 three chars plus XX + df.loc[df["To"].str.len() == 9, "To"] = ( + df["To"].str[3:6] + df["To"].str[7:9]) + + # Combine From and To columns. + df["TECHNOLOGY"] = ("TRN" + df["From"] + df["To"]) + + # Drop to and from columns + df = df.drop(["From", "To"], axis=1) + + return df + +def get_transmission_costs(): + '''Gets electrical transmission capital and fixed cost per technology. + + Both the capital costs and fixed cost are written out to avoid having + to read in the excel file twice + Returns: + df_capex: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' + representing CAPITAL cost in millions of dollars per year. + df_fix: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' + representing FIXED cost in millions of dollars per year. + ''' + + # CONFIGURATION PARAMETERS + + config_paths = ConfigPaths() + input_data_dir = config_paths.input_data_dir + + # Read in raw data + df = pd.read_excel(os.path.join(input_data_dir, + "Costs Line expansion.xlsx"), + sheet_name = 'Lines') + + # Drop unneeded columns + df = df.drop( + [ + "Line", + "KM distance", + "HVAC/HVDC/Subsea", + "Losses", + "Unnamed: 8", + "Line Max Size (MW)", + "Unnamed: 10", + "Unnamed: 11", + "Unnamed: 12", + "Subsea lines", + "Unnamed: 14" + ], + axis=1, + ) + + # Use to/from codes to create a TECHNOLOGY columns + df = format_transmission_name(df) + + # Changes units + # Raw given in dollars -> model units in million dollars + df = df.rename(columns={'Annual FO&M (3.5% of CAPEX) ($2010 in $000)':'O&M'}) + df = df.rename(columns={'Build Cost ($2010 in $000)':'CAPEX'}) + df['O&M'] = df['O&M'].astype(float) / 1000 + df['CAPEX'] = df['CAPEX'].astype(float) / 1000 + df = df.round({'O&M': 3, 'CAPEX': 3, }) + + # Separate out fixed and capex and return values + df_capex = df.drop(['O&M'], axis=1) + df_capex = df_capex.rename(columns={'CAPEX':'VALUE'}) + df_fix = df.drop(['CAPEX'], axis=1) + df_fix = df_fix.rename(columns={'O&M':'VALUE'}) + + return df_capex, df_fix + +def costs_transmission(years, df_oar_final, region_name, output_data_dir): + # Get transmission costs df_trans_capex, df_trans_fix = get_transmission_costs() @@ -1028,6 +1224,19 @@ def main(): df_trans_capex['TECHNOLOGY'].isin(df_oar_final['TECHNOLOGY'])] df_trans_fix = df_trans_fix.loc[ df_trans_fix['TECHNOLOGY'].isin(df_oar_final['TECHNOLOGY'])] + + return df_trans_capex, df_trans_fix + +def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, + df_trans_fix, output_data_dir): + + weo_regions_dict = dict([(k, v) + for k, v + in zip(df_weo_regions['technology_code'], + df_weo_regions['weo_region'] + ) + ] + ) # Create formatted CSVs for each_cost in ['Capital', 'O&M']: @@ -1069,9 +1278,6 @@ def main(): columns = 'TECHNOLOGY', values = 'value').reset_index() df_costs_final = df_costs_final.replace([-9],[np.nan]) - #df_costs_final.set_index(['REGION', 'YEAR'], - # inplace = True) - df_costs_final = df_costs_final.interpolate(method = 'linear', limit_direction='forward').round(2) @@ -1101,6 +1307,7 @@ def main(): "FixedCost.csv"), index = None) +def capact(df_oar_final, output_data_dir): # Create CapacityToActivityUnit csv df_capact_final = df_oar_final[['REGION', @@ -1118,11 +1325,12 @@ def main(): ) df_capact_final['VALUE'] = 31.536 - # df_capact_final = apply_dtypes(df_capact_final, "CapacityToActivityUnit") df_capact_final.drop_duplicates(inplace=True) df_capact_final.to_csv(os.path.join(output_data_dir, "CapacityToActivityUnit.csv"), index = None) + +def activity_transmission_limit(cross_border_trade, df_oar_final, output_data_dir): # Set cross-border trade to 0 if False if not cross_border_trade: @@ -1146,6 +1354,7 @@ def main(): "TotalTechnologyModelPeriodActivityUpperLimit.csv"), index = None) +def op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict, region_name): # Create Operational Life data tech_code_dict_reverse = dict((v,k) for k,v in tech_code_dict.items()) @@ -1162,6 +1371,10 @@ def main(): region_name, op_life_tech, op_life_dict[op_life_tech_name]]) + + return op_life_trn, op_life_out + +def op_life_transmission(op_life_trn, op_life_out, output_data_dir, region_name): # transmission technologies for op_life_tech in op_life_trn: @@ -1173,6 +1386,8 @@ def main(): df_op_life_Out.to_csv(os.path.join(output_data_dir, "OperationalLife.csv"), index = None) + +def cap_investment_constraints(df_iar_final, years, config, output_data_dir, region_name): # Create totalAnnualMaxCapacityInvestment data @@ -1210,6 +1425,34 @@ def main(): df_min_cap_invest.to_csv(os.path.join(output_data_dir, 'TotalAnnualMinCapacityInvestment.csv'), index = None) + +def create_sets(x, df, output_dir, custom_node_elements): + """Creates a formatted otoole set csv + + Arguments: + x = Name of set given by otoole as a string + df = dataframe in extract set information from + output_dir = directory to write file to + + Returns: + None. Writes out csv file + + Example: + create_sets('TECHNOLOGY', df_oar_final, output_dir) + """ + set_elements = list(df[x].unique()) + list(df[x].unique()) + list(custom_node_elements) + set_elements = list(set(set_elements)) + set_elements = [x for x in set_elements if x != 'nan'] + set_elements.sort() + set_elements_df = pd.DataFrame(set_elements, columns = ['VALUE']) + return set_elements_df.to_csv(os.path.join(output_dir, + str(x) + '.csv' + ), + index = None + ) + +def output_sets(custom_nodes, df_oar_final, output_data_dir, custom_techs, years, + mode_list, region_name): # ## Create sets for TECHNOLOGIES, FUELS if custom_nodes: @@ -1236,35 +1479,6 @@ def main(): "REGION.csv"), index = None) - user_defined_capacity(region_name, years, output_data_dir, tech_capacity, op_life_dict) - availability_factor(region_name, years, output_data_dir, df_af) - - -def create_sets(x, df, output_dir, custom_node_elements): - """Creates a formatted otoole set csv - - Arguments: - x = Name of set given by otoole as a string - df = dataframe in extract set information from - output_dir = directory to write file to - - Returns: - None. Writes out csv file - - Example: - create_sets('TECHNOLOGY', df_oar_final, output_dir) - """ - set_elements = list(df[x].unique()) + list(df[x].unique()) + list(custom_node_elements) - set_elements = list(set(set_elements)) - set_elements = [x for x in set_elements if x != 'nan'] - set_elements.sort() - set_elements_df = pd.DataFrame(set_elements, columns = ['VALUE']) - return set_elements_df.to_csv(os.path.join(output_dir, - str(x) + '.csv' - ), - index = None - ) - def duplicatePlexosTechs(df_in, techs): """Creates new technologies to replace PLEXOS technolgoies. @@ -1356,113 +1570,6 @@ def newIar(df_in, tech): df_out['VALUE'] = round(1/iar, 3) return df_out -def get_transmission_costs(): - '''Gets electrical transmission capital and fixed cost per technology. - - Both the capital costs and fixed cost are written out to avoid having - to read in the excel file twice - - Returns: - df_capex: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' - representing CAPITAL cost in millions of dollars per year. - df_fix: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' - representing FIXED cost in millions of dollars per year. - ''' - - # CONFIGURATION PARAMETERS - - config_paths = ConfigPaths() - input_data_dir = config_paths.input_data_dir - - # Read in raw data - df = pd.read_excel(os.path.join(input_data_dir, - "Costs Line expansion.xlsx"), - sheet_name = 'Lines') - - # Drop unneeded columns - df = df.drop( - [ - "Line", - "KM distance", - "HVAC/HVDC/Subsea", - "Losses", - "Unnamed: 8", - "Line Max Size (MW)", - "Unnamed: 10", - "Unnamed: 11", - "Unnamed: 12", - "Subsea lines", - "Unnamed: 14" - ], - axis=1, - ) - - # Use to/from codes to create a TECHNOLOGY columns - df = format_transmission_name(df) - - # Changes units - # Raw given in dollars -> model units in million dollars - df = df.rename(columns={'Annual FO&M (3.5% of CAPEX) ($2010 in $000)':'O&M'}) - df = df.rename(columns={'Build Cost ($2010 in $000)':'CAPEX'}) - df['O&M'] = df['O&M'].astype(float) / 1000 - df['CAPEX'] = df['CAPEX'].astype(float) / 1000 - df = df.round({'O&M': 3, 'CAPEX': 3, }) - - # Separate out fixed and capex and return values - df_capex = df.drop(['O&M'], axis=1) - df_capex = df_capex.rename(columns={'CAPEX':'VALUE'}) - df_fix = df.drop(['CAPEX'], axis=1) - df_fix = df_fix.rename(columns={'O&M':'VALUE'}) - - return df_capex, df_fix - -def format_transmission_name(df): - '''Formats PLEXOS transmission names into OSeMOSYS Global names. - - Args: - :param df: Pandas DataFrame with columns 'From' and 'To' describing the - transmission from and to contries. ie. - - Returns: - :param df: Same as df_in, except the 'From' and 'To' columns are replaced - with a single 'TECHNOLOGY' column holding OSeMOSYS Global - naming conventions - - Example: - df = pd.DataFrame( - [[AF-COD, AF-COG, 0.001], - [EU-AUT, EU-SVK, 0.004], - [AS-LBN, AS-SYR, 0.006]], - columns = ['From', 'To', 'Losses'] - ) - pd.DataFrame( - [[0.001,TRNCODXXCOGXX], - [0.004,TRNAUTXXSVKXX], - [0.006,TRNLBNXXSYRXX]] - columns = ['Losses','TECHNOLOGY'])''' - - # If from column has length 6 then it's the last three chars plus XX - df.loc[df["From"].str.len() == 6, "From"] = (df["From"].str[3:6] + "XX") - - # If from column has length 9 then it's the 3:6 and 7:9 three chars plus XX - df.loc[df["From"].str.len() == 9, "From"] = ( - df["From"].str[3:6] + df["From"].str[7:9]) - - # If to column has length 6 then it's the last three chars plus XX - df.loc[df["To"].str.len() == 6, "To"] = (df["To"].str[3:6] + "XX") - - # If to column has length 9 then it's the 3:6 and 7:9 three chars plus XX - df.loc[df["To"].str.len() == 9, "To"] = ( - df["To"].str[3:6] + df["To"].str[7:9]) - - # Combine From and To columns. - df["TECHNOLOGY"] = ("TRN" + df["From"] + df["To"]) - - # Drop to and from columns - df = df.drop(["From", "To"], axis=1) - - return df - def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life_dict): """User-defined capacities are used when a specific technology must be invested, for a given year and capacity. This is applied hrough the @@ -1489,7 +1596,6 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life for tech, tech_params in tech_capacity.items(): techCapacity.append([tech, tech_params[0], tech_params[1]]) - #tech_capacity_dict[tech] = tech_params[2] tech_capacity_dict[tech] = tech_params[2] build_year_dict[tech] = tech_params[1] first_year_dict[tech] = tech_params[3] From 57641f826acc63ec3a1ae96cc61b7be27dab5670 Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Mon, 16 Sep 2024 18:00:22 -0400 Subject: [PATCH 03/12] Updates to powerplant_data -Split in functions -Transmission not yet separated -User config still part of the script --- .../osemosys_global/powerplant_data.py | 589 +++++++++--------- 1 file changed, 309 insertions(+), 280 deletions(-) diff --git a/workflow/scripts/osemosys_global/powerplant_data.py b/workflow/scripts/osemosys_global/powerplant_data.py index 814c9c9b..57895a93 100644 --- a/workflow/scripts/osemosys_global/powerplant_data.py +++ b/workflow/scripts/osemosys_global/powerplant_data.py @@ -1,12 +1,9 @@ -#!/usr/bin/env python -# coding: utf-8 - # Import modules import warnings warnings.simplefilter(action='ignore', category=FutureWarning) import pandas as pd from datetime import datetime -pd.options.mode.chained_assignment = None # default='warn' +pd.options.mode.chained_assignment = None import numpy as np import itertools import os @@ -21,8 +18,10 @@ def main(): """Main retrieves the script input data, sets user configurations and calls the different functions as defined within the script.""" - + # CONFIGURATION PARAMETERS + + global config, region_name, input_data_dir, output_data_dir config_paths = ConfigPaths() config = ConfigFile('config') @@ -38,29 +37,32 @@ def main(): os.makedirs(custom_nodes_dir) except FileExistsError: pass + + global model_start_year, model_end_year, years, custom_nodes + tech_capacity = config.get('user_defined_capacity') cross_border_trade = config.get('crossborderTrade') model_start_year = config.get('startYear') model_end_year = config.get('endYear') years = list(range(model_start_year, model_end_year + 1)) - tech_capacity = config.get('user_defined_capacity') custom_nodes = config.get('nodes_to_add') - + no_investment_techs = config.get('no_invest_technologies') + # Create output directory if not os.path.exists(output_data_dir): os.makedirs(output_data_dir) - # Inputs the PLEXOS-World 2015 dataset as basis for the powerplant data. + # INPUT FILES - df = pd.read_excel(os.path.join(input_data_dir, + df_pw_prop = pd.read_excel(os.path.join(input_data_dir, "PLEXOS_World_2015_Gold_V1.1.xlsx"), sheet_name = "Properties") - df_dict = pd.read_excel(os.path.join(input_data_dir, + df_pw_memb = pd.read_excel(os.path.join(input_data_dir, "PLEXOS_World_2015_Gold_V1.1.xlsx"), sheet_name = "Memberships") - df_dict = df_dict[df_dict["parent_class"] == "Generator"].rename( + df_dict = df_pw_memb[df_pw_memb["parent_class"] == "Generator"].rename( {"parent_object": "powerplant"}, axis=1 ) @@ -87,13 +89,23 @@ def main(): "Costs Line expansion.xlsx"), sheet_name = 'Interface' ) + + df_trn_lines = pd.read_excel(os.path.join(input_data_dir, + "Costs Line expansion.xlsx"), + sheet_name = 'Lines') + df_weo_regions = pd.read_csv(os.path.join(input_data_dir, "weo_region_mapping.csv") ) + + df_tech_set = os.path.join(output_data_dir, 'TECHNOLOGY.csv') + df_af = pd.read_csv(os.path.join(input_data_dir, "availability_factors.csv") ) + fuel_set = os.path.join(output_data_dir, 'FUEL.csv') + if custom_nodes: df_custom_res_cap = pd.read_csv(os.path.join(input_data_dir, "custom_nodes", @@ -106,12 +118,22 @@ def main(): # USER INPUTS - """Technologies that will have 00 and 01 suffixes to represent PLEXOS - historical values and future values.""" - duplicate_techs = ['CCG', 'OCG'] + """Change IAR for CSP value taken from PLEXOS to 1.0. + Relevant for the 'average_efficiency' function.""" + avg_csp_eff = 1 + """Change IAR for URN value taken from PLEXOS to 2.2 (45%). + Relevant for the 'average_efficiency' function.""" + avg_urn_eff = 0.45 - """ Add extra nodes which exist in 2050 but are not in the 2015 data""" + """Technologies that will have 00 and 01 suffixes to represent PLEXOS + historical values and future values. Relevant for the 'residual_capacity' + and activity functions.""" + duplicate_techs = ['CCG', 'OCG'] + + """Add extra nodes which exist in 2050 but are not in the 2015 PLEXOS-World + data to enable their addition to the workflow. Relevant for the + 'generator_table' and activity functions""" nodes_extra_list = ['AF-SOM', 'AF-TCD', 'AS-TLS', @@ -122,9 +144,13 @@ def main(): 'SA-BRA-J2', 'SA-BRA-J3', 'SA-SUR'] - + + """Sets the mode of operations for technologies. + Relevant for the activity_master_start function.""" mode_list = [1,2] + """List of technologies (thermal) for which output activity ratios need + to be developed. Relevant for the 'activity_output_pwr' function.""" thermal_fuel_list_oar = ['COA', 'COG', 'OCG', @@ -136,6 +162,18 @@ def main(): 'CCS' ] + """List of technologies (thermal) for which input activity ratios need + to be developed. Relevant for the 'activity_input_pwr' function.""" + thermal_fuel_list_iar = ['COA', + 'COG', + 'PET', + 'URN', + 'OIL', + 'OTH' + ] + + """List of upstream fuels for which output activity ratios need + to be developed. Relevant for the 'activity_upstream' function.""" thermal_fuels_list_mining = ['COA', 'COG', 'GAS', @@ -144,15 +182,10 @@ def main(): 'OIL', 'OTH' ] - - thermal_fuel_list_iar = ['COA', - 'COG', - 'PET', - 'URN', - 'OIL', - 'OTH' - ] + """List of technologies (renewable) for which activity ratios need + to be developed. Relevant for the 'activity_input_pwr' and + 'activity_upstream' functions.""" renewables_list = ['BIO', 'GEO', 'HYD', @@ -163,6 +196,8 @@ def main(): 'WON', 'WOF'] + """Technology input for the mapping of WEO cost data to OG technologies. + Relevant for the 'costs_pwr' function.""" costs_dict = {'Biomass - waste incineration - CHP':'WAS', 'Biomass Power plant':'BIO', 'CCGT':'CCG', @@ -183,62 +218,85 @@ def main(): 'Oil':'OIL', 'Other':'OTH', 'IGCC + CCS':'CCS', - 'Coal* + CCS':'CCS',} # Added OIL, OTH, PET, WOF to WEO 2018 + 'Coal* + CCS':'CCS',} + + """setings for custom iar values deviating from derived values from the + PLEXOS-World dataset. Relevant for the 'newIar' function.""" + global new_iar_ccg, new_iar_ocg, new_iar_coa, new_iar_default + + new_iar_ccg = 0.5 + new_iar_ocg = 0.35 + new_iar_coa = 0.33 + new_iar_default = 1 + + """Set iar and oar values for custom transmission entries. I.e. oar of 0.9 + assumes 10% losses. Relevant for the user_defined_capacity function.""" + df_iar_custom_val = 1 + df_oar_custom_val = 0.9 # CALL FUNCTIONS - gen_table = generator_table(df, df_dict, model_start_year, custom_nodes, tech_code_dict, op_life_dict, nodes_extra_list) - - df_eff_node, df_eff_tech = average_efficiency(gen_table) + gen_table = generator_table(df_pw_prop, df_dict, tech_code_dict, + op_life_dict, nodes_extra_list) - df_res_cap, custom_techs = residual_capacity(gen_table, tech_list, df_tech_code, model_start_year, model_end_year, - custom_nodes, df_custom_res_cap, years, output_data_dir, duplicate_techs, region_name) + df_eff_node, df_eff_tech = average_efficiency(gen_table, avg_csp_eff, + avg_urn_eff) - gem_cap(gen_table, input_data_dir, model_start_year, tech_code_dict, op_life_dict) + df_res_cap, custom_techs = residual_capacity(gen_table, tech_list, df_tech_code, + df_custom_res_cap, duplicate_techs) + + gem_cap(gen_table, tech_code_dict, op_life_dict) - df_ratios = activity_master_start(gen_table, custom_nodes, nodes_extra_list, years, duplicate_techs, mode_list) + df_ratios = activity_master_start(gen_table, nodes_extra_list, + duplicate_techs, mode_list) - df_oar, df_oar_base = activity_output_pwr(df_ratios, thermal_fuel_list_oar, region_name) + df_oar, df_oar_base = activity_output_pwr(df_ratios, thermal_fuel_list_oar) - df_iar_base = activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, df_eff_node, df_eff_tech, region_name) + df_iar_base = activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, + df_eff_node, df_eff_tech) - df_oar_upstream, df_oar_int = activity_upstream(df_iar_base, renewables_list, thermal_fuels_list_mining) + df_oar_upstream, df_oar_int = activity_upstream(df_iar_base, renewables_list, + thermal_fuels_list_mining) - df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar = activity_transmission(df_oar_base, df, model_start_year, - model_end_year, df_trn_efficiencies, - region_name) + df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar = activity_transmission(df_oar_base, + df_pw_prop, + df_trn_efficiencies) - df_oar_final, df_iar_final = activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, - df_int_trn_oar, df_iar_base, df_iar_trn, df_int_trn_iar, - duplicate_techs, output_data_dir) + df_oar_final, df_iar_final = activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, + df_oar_trn, df_int_trn_oar, df_iar_base, + df_iar_trn, df_int_trn_iar, duplicate_techs) df_costs = costs_pwr(df_weo_data, costs_dict) - df_trans_capex, df_trans_fix = costs_transmission(years, df_oar_final, region_name, output_data_dir) + df_trans_capex, df_trans_fix = get_transmission_costs(df_trn_lines, df_oar_final) + + cap_cost_base, fix_cost_base = costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, df_trans_fix) - costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, df_trans_fix, output_data_dir) + cap_act_base = capact(df_oar_final) - capact(df_oar_final, output_data_dir) + activity_transmission_limit(cross_border_trade, df_oar_final) - activity_transmission_limit(cross_border_trade, df_oar_final, output_data_dir) + op_life_trn, op_life_out = op_life(tech_code_dict, df_iar_final, + df_oar_final, op_life_dict) - op_life_trn, op_life_out = op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict, region_name) + op_life_base = op_life_transmission(op_life_trn, op_life_out, op_life_dict) - op_life_transmission(op_life_trn, op_life_out, output_data_dir, region_name) + df_max_cap_invest, df_min_cap_invest = cap_investment_constraints(df_iar_final, no_investment_techs) - cap_investment_constraints(df_iar_final, years, config, output_data_dir, region_name) + output_sets(df_oar_final, custom_techs, mode_list) - output_sets(custom_nodes, df_oar_final, output_data_dir, custom_techs, years, mode_list, region_name) + tech_set_base = user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, + df_min_cap_invest, df_max_cap_invest, df_res_cap, + df_iar_final, df_oar_final, fuel_set, op_life_base, cap_act_base, + cap_cost_base, df_iar_custom_val, df_oar_custom_val) - user_defined_capacity(region_name, years, output_data_dir, tech_capacity, op_life_dict) + availability_factor(df_af, tech_set_base) - availability_factor(region_name, years, output_data_dir, df_af) - -def generator_table(df, df_dict, model_start_year, custom_nodes, tech_code_dict, op_life_dict, nodes_extra_list): +def generator_table(df_pw_prop, df_dict, tech_code_dict, op_life_dict, nodes_extra_list): # Create main generator table gen_cols_1 = ["child_class", "child_object", "property", "value"] - df_gen = df[gen_cols_1] + df_gen = df_pw_prop[gen_cols_1] df_gen = df_gen[df_gen["child_class"] == "Generator"] df_gen.rename(columns={"child_object": "powerplant"}, inplace=True) df_gen.drop("child_class", axis=1, inplace=True) @@ -253,8 +311,8 @@ def generator_table(df, df_dict, model_start_year, custom_nodes, tech_code_dict, df_gen["Units"].astype(int) ) - gen_cols_2 = ["Commission Date", "Heat Rate", "Max Capacity", "total_capacity"] - df_gen_2 = df_gen[gen_cols_2] + gen_cols_base = ["Commission Date", "Heat Rate", "Max Capacity", "total_capacity"] + df_gen_base = df_gen[gen_cols_base] ## Compile dataframe with powerplants, nodes, and fuels df_dict_fuel = df_dict[df_dict["collection"] == "Fuels"] @@ -265,48 +323,48 @@ def generator_table(df, df_dict, model_start_year, custom_nodes, tech_code_dict, ## Merge original generator dataframe with nodes and fuels - df_gen_2 = pd.merge(df_gen_2, df_dict_2, how="outer", on="powerplant") - df_gen_2.rename( + df_gen_base = pd.merge(df_gen_base, df_dict_2, how="outer", on="powerplant") + df_gen_base.rename( {"child_object_x": "fuel", "child_object_y": "node"}, axis=1, inplace=True ) ## Extract start year from Commission Date - df_gen_2["Commission Date"] = (pd.TimedeltaIndex(df_gen_2["Commission Date"].astype(int), + df_gen_base["Commission Date"] = (pd.TimedeltaIndex(df_gen_base["Commission Date"].astype(int), unit='d') + datetime(1900, 1, 1)) - df_gen_2["start_year"] = df_gen_2["Commission Date"].dt.year - df_gen_2.drop("Commission Date", axis=1, inplace=True) + df_gen_base["start_year"] = df_gen_base["Commission Date"].dt.year + df_gen_base.drop("Commission Date", axis=1, inplace=True) ## Calculate efficiency from heat rate. Units of heat rate in MJ/kWh - df_gen_2["efficiency"] = 3.6 / df_gen_2["Heat Rate"].astype(float) - df_gen_2.drop("Heat Rate", axis=1, inplace=True) + df_gen_base["efficiency"] = 3.6 / df_gen_base["Heat Rate"].astype(float) + df_gen_base.drop("Heat Rate", axis=1, inplace=True) - ## Calcluate years of operation from start year until 2015 - df_gen_2["years_of_operation"] = model_start_year - df_gen_2["start_year"] + ## Calcluate years of operation from start year + df_gen_base["years_of_operation"] = model_start_year - df_gen_base["start_year"] ## Fix blank spaces in 'fuels' columns. Appearing for 'Oil' powerplants in certain countries - df_gen_2.loc[df_gen_2["fuel"].isna(), "fuel"] = ( - df_gen_2["node"].str.split("-").str[:2].str.join("-") + df_gen_base.loc[df_gen_base["fuel"].isna(), "fuel"] = ( + df_gen_base["node"].str.split("-").str[:2].str.join("-") + " " - + df_gen_2["powerplant"].str.split("_", expand=True)[1] + + df_gen_base["powerplant"].str.split("_", expand=True)[1] ) ## Create column for technology - df_gen_2["technology"] = df_gen_2["powerplant"].str.split("_").str[1] - df_gen_2["technology"] = df_gen_2["technology"].str.title() + df_gen_base["technology"] = df_gen_base["powerplant"].str.split("_").str[1] + df_gen_base["technology"] = df_gen_base["technology"].str.title() ## Divide Gas into CCGT and OCGT based on max capacity - df_gen_2.loc[ - (df_gen_2["technology"] == "Gas") & (df_gen_2["Max Capacity"].astype(float) > 130), + df_gen_base.loc[ + (df_gen_base["technology"] == "Gas") & (df_gen_base["Max Capacity"].astype(float) > 130), "technology", ] = "Gas-CCGT" - df_gen_2.loc[ - (df_gen_2["technology"] == "Gas") & (df_gen_2["Max Capacity"].astype(float) <= 130), + df_gen_base.loc[ + (df_gen_base["technology"] == "Gas") & (df_gen_base["Max Capacity"].astype(float) <= 130), "technology", ] = "Gas-OCGT" # Create table with aggregated capacity - df_gen_agg_node = df_gen_2[df_gen_2['start_year']<=model_start_year] + df_gen_agg_node = df_gen_base[df_gen_base['start_year']<=model_start_year] df_gen_agg_node = df_gen_agg_node.groupby(['node', 'technology'], as_index=False)['total_capacity'].sum() df_gen_agg_node = df_gen_agg_node.pivot(index='node', @@ -326,59 +384,59 @@ def generator_table(df, df_dict, model_start_year, custom_nodes, tech_code_dict, ).fillna(0).sort_values(by='node').set_index('node').round(2) # Add region and country code columns - df_gen_2['region_code'] = df_gen_2['node'].str[:2] - df_gen_2['country_code'] = df_gen_2['node'].str[3:] + df_gen_base['region_code'] = df_gen_base['node'].str[:2] + df_gen_base['country_code'] = df_gen_base['node'].str[3:] - df_gen_2['operational_life'] = df_gen_2['technology'].map(op_life_dict) - df_gen_2['retirement_year_data'] = (df_gen_2['operational_life'] - + df_gen_2['start_year']) - df_gen_2['retirement_diff'] = ((df_gen_2['years_of_operation'] - - df_gen_2['operational_life'])/ - df_gen_2['operational_life']) + df_gen_base['operational_life'] = df_gen_base['technology'].map(op_life_dict) + df_gen_base['retirement_year_data'] = (df_gen_base['operational_life'] + + df_gen_base['start_year']) + df_gen_base['retirement_diff'] = ((df_gen_base['years_of_operation'] + - df_gen_base['operational_life'])/ + df_gen_base['operational_life']) ''' Set retirement year based on years of operation. If (years of operation - operational life) is more than 50% of operational life, set retirement year ''' - df_gen_2.loc[df_gen_2['retirement_diff'] >= 0.5, + df_gen_base.loc[df_gen_base['retirement_diff'] >= 0.5, 'retirement_year_model'] = 2028 - df_gen_2.loc[(df_gen_2['retirement_diff'] < 0.5) & - (df_gen_2['retirement_diff'] > 0), + df_gen_base.loc[(df_gen_base['retirement_diff'] < 0.5) & + (df_gen_base['retirement_diff'] > 0), 'retirement_year_model'] = 2033 - df_gen_2.loc[df_gen_2['retirement_diff'] <= 0, - 'retirement_year_model'] = df_gen_2['retirement_year_data'] + df_gen_base.loc[df_gen_base['retirement_diff'] <= 0, + 'retirement_year_model'] = df_gen_base['retirement_year_data'] - df_gen_2['tech_code'] = df_gen_2['technology'].map(tech_code_dict) + df_gen_base['tech_code'] = df_gen_base['technology'].map(tech_code_dict) - df_gen_2.loc[df_gen_2['node'].str.len() <= 6, - 'node_code'] = (df_gen_2['node']. + df_gen_base.loc[df_gen_base['node'].str.len() <= 6, + 'node_code'] = (df_gen_base['node']. str.split('-'). str[1:]. str.join("") + 'XX') - df_gen_2.loc[df_gen_2['node'].str.len() > 6, - 'node_code'] = (df_gen_2['node']. + df_gen_base.loc[df_gen_base['node'].str.len() > 6, + 'node_code'] = (df_gen_base['node']. str.split('-'). str[1:]. str.join("") ) - df_gen_2 = df_gen_2.loc[~df_gen_2['tech_code'].isna()] + df_gen_base = df_gen_base.loc[~df_gen_base['tech_code'].isna()] - return df_gen_2 + return df_gen_base -def average_efficiency(df_gen_2): +def average_efficiency(df_gen_base, avg_csp_eff, avg_urn_eff): # ### Calculate average InputActivityRatio by node+technology and only by technology - df_eff = df_gen_2[['node_code', + df_eff = df_gen_base[['node_code', 'efficiency', 'tech_code']] - # Change IAR for CSP value taken from PLEXOS to 1.0 - df_eff.loc[df_eff['tech_code']=='CSP', 'efficiency'] = 1 + # Change IAR for CSP value taken from PLEXOS + df_eff.loc[df_eff['tech_code']=='CSP', 'efficiency'] = avg_csp_eff - # Change IAR for URN value taken from PLEXOS to 2.2 (45%) - df_eff.loc[df_eff['tech_code']=='URN', 'efficiency'] = 0.45 + # Change IAR for URN value taken from PLEXOS + df_eff.loc[df_eff['tech_code']=='URN', 'efficiency'] = avg_urn_eff # Average efficiency by node and technology df_eff_node = df_eff.groupby(['tech_code', @@ -404,9 +462,7 @@ def average_efficiency(df_gen_2): return df_eff_node, df_eff_tech -def residual_capacity(df_gen_2, tech_list, df_tech_code, model_start_year, model_end_year, - custom_nodes, df_custom_res_cap, years, output_data_dir, duplicate_techs, - region_name): +def residual_capacity(df_gen_base, tech_list, df_tech_code, df_custom_res_cap, duplicate_techs): # ### Calculate residual capacity res_cap_cols = [ @@ -417,7 +473,7 @@ def residual_capacity(df_gen_2, tech_list, df_tech_code, model_start_year, model "retirement_year_model", ] - df_res_cap = df_gen_2[res_cap_cols] + df_res_cap = df_gen_base[res_cap_cols] for each_year in range(model_start_year, model_end_year+1): df_res_cap[str(each_year)] = 0 @@ -482,7 +538,9 @@ def residual_capacity(df_gen_2, tech_list, df_tech_code, model_start_year, model return df_res_cap, custom_techs if custom_nodes else df_res_cap -def gem_cap(df_gen_2, input_data_dir, model_start_year, tech_code_dict, op_life_dict): +"""The output of the gem_cap function is currently unused. Implementation is being discussed so kept as standalone +function for the time being.""" +def gem_cap(df_gen_base, tech_code_dict, op_life_dict): # ### Calculate planned capacities based on the Global Energy Observatory datasets @@ -505,7 +563,7 @@ def gem_cap(df_gen_2, input_data_dir, model_start_year, tech_code_dict, op_life_ gen_lat['value'], left_on = 'name', right_index = True).merge( gen_long['value'], left_on = 'name', right_index = True) - gen_locations = pd.merge(gen_locations, df_gen_2[['node', 'country_code', 'node_code', 'powerplant']], + gen_locations = pd.merge(gen_locations, df_gen_base[['node', 'country_code', 'node_code', 'powerplant']], left_on = 'name', right_on = 'powerplant', how = 'inner') subcountries = gen_locations.loc[~gen_locations['node_code'].str.contains('XX')]['country_code'].str[:3].unique() @@ -640,15 +698,15 @@ def gem_cap(df_gen_2, input_data_dir, model_start_year, tech_code_dict, op_life_ gem_all_retired_agg = gem_all_retired.groupby(['node_code', 'Technology', 'YEAR'], as_index=False)['VALUE'].sum() -def activity_master_start(df_gen_2, custom_nodes, nodes_extra_list, years, duplicate_techs, mode_list): +def activity_master_start(df_gen_base, nodes_extra_list, duplicate_techs, mode_list): # ### Add input and output activity ratios # Create master table for activity ratios if custom_nodes: - node_list = list(df_gen_2['node_code'].unique()) + custom_nodes + node_list = list(df_gen_base['node_code'].unique()) + custom_nodes else: - node_list = list(df_gen_2['node_code'].unique()) + node_list = list(df_gen_base['node_code'].unique()) # Add extra nodes which are not present in 2015 but will be by 2050 for each_node in nodes_extra_list: if len(each_node) <= 6: @@ -656,7 +714,7 @@ def activity_master_start(df_gen_2, custom_nodes, nodes_extra_list, years, dupli else: node_list.append("".join(each_node.split('-')[1:])) - master_fuel_list = list(df_gen_2['tech_code'].unique()) + master_fuel_list = list(df_gen_base['tech_code'].unique()) master_fuel_list.append('CCS') df_ratios = pd.DataFrame(list(itertools.product(node_list, @@ -671,7 +729,7 @@ def activity_master_start(df_gen_2, custom_nodes, nodes_extra_list, years, dupli return df_ratios -def activity_output_pwr(df_ratios, thermal_fuel_list_oar, region_name): +def activity_output_pwr(df_ratios, thermal_fuel_list_oar): # #### OutputActivityRatio - Power Generation Technologies df_oar = df_ratios.copy() @@ -700,7 +758,7 @@ def activity_output_pwr(df_ratios, thermal_fuel_list_oar, region_name): return df_oar, df_oar_base def activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, - df_eff_node, df_eff_tech, region_name): + df_eff_node, df_eff_tech): # #### InputActivityRatio - Power Generation Technologies # Copy OAR table with all columns to IAR @@ -837,8 +895,7 @@ def activity_upstream(df_iar_base, renewables_list, thermal_fuels_list_mining): return df_oar_upstream, df_oar_int -def activity_transmission(df_oar_base, df, model_start_year, model_end_year, - df_trn_efficiencies, region_name): +def activity_transmission(df_oar_base, df_pw_prop, df_trn_efficiencies): # #### Downstream Activity Ratios @@ -859,7 +916,7 @@ def activity_transmission(df_oar_base, df, model_start_year, model_end_year, # Build international transmission system from original input data, but for Line rather than Generator: int_trn_cols = ["child_class", "child_object", "property", "value"] - df_int_trn = df[int_trn_cols] + df_int_trn = df_pw_prop[int_trn_cols] df_int_trn = df_int_trn[df_int_trn["child_class"] == "Line"] # For IAR and OAR we can drop the value: @@ -984,7 +1041,7 @@ def activity_transmission(df_oar_base, df, model_start_year, model_end_year, def activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, df_int_trn_oar, df_iar_final, df_iar_trn, df_int_trn_iar, - duplicate_techs, output_data_dir): + duplicate_techs): # #### Output IAR and OAR @@ -1027,7 +1084,8 @@ def activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, # Add iar for techs not using PLEXOS values df_iar_newTechs = duplicatePlexosTechs(df_iar_final, duplicate_techs) for duplicate_tech in duplicate_techs: - df_new_iar = newIar(df_iar_newTechs, duplicate_tech) + df_new_iar = newIar(df_iar_newTechs, duplicate_tech,new_iar_ccg, + new_iar_ocg, new_iar_coa, new_iar_default) df_iar_final = pd.concat([df_iar_final, df_new_iar]) # Add oar for techs not using PLEXOS values @@ -1096,7 +1154,7 @@ def costs_pwr(df_weo_data, costs_dict): df_costs = df_costs.loc[df_costs['technology'].isin(costs_dict.keys())] df_costs['technology_code'] = df_costs['technology'].replace(costs_dict) - + return df_costs def format_transmission_name(df): @@ -1146,31 +1204,21 @@ def format_transmission_name(df): return df -def get_transmission_costs(): +def get_transmission_costs(df_trn_lines, df_oar_final): '''Gets electrical transmission capital and fixed cost per technology. Both the capital costs and fixed cost are written out to avoid having to read in the excel file twice Returns: - df_capex: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' + df_trans_capex: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' representing CAPITAL cost in millions of dollars per year. - df_fix: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' + df_trans_fix: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' representing FIXED cost in millions of dollars per year. ''' - # CONFIGURATION PARAMETERS - - config_paths = ConfigPaths() - input_data_dir = config_paths.input_data_dir - - # Read in raw data - df = pd.read_excel(os.path.join(input_data_dir, - "Costs Line expansion.xlsx"), - sheet_name = 'Lines') - # Drop unneeded columns - df = df.drop( + df = df_trn_lines.drop( [ "Line", "KM distance", @@ -1199,17 +1247,10 @@ def get_transmission_costs(): df = df.round({'O&M': 3, 'CAPEX': 3, }) # Separate out fixed and capex and return values - df_capex = df.drop(['O&M'], axis=1) - df_capex = df_capex.rename(columns={'CAPEX':'VALUE'}) - df_fix = df.drop(['CAPEX'], axis=1) - df_fix = df_fix.rename(columns={'O&M':'VALUE'}) - - return df_capex, df_fix - -def costs_transmission(years, df_oar_final, region_name, output_data_dir): - - # Get transmission costs - df_trans_capex, df_trans_fix = get_transmission_costs() + df_trans_capex = df.drop(['O&M'], axis=1) + df_trans_capex = df_trans_capex.rename(columns={'CAPEX':'VALUE'}) + df_trans_fix = df.drop(['CAPEX'], axis=1) + df_trans_fix = df_trans_fix.rename(columns={'O&M':'VALUE'}) df_trans_capex['REGION'] = region_name df_trans_capex['YEAR'] = [years] * len(df_trans_capex) @@ -1228,7 +1269,7 @@ def costs_transmission(years, df_oar_final, region_name, output_data_dir): return df_trans_capex, df_trans_fix def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, - df_trans_fix, output_data_dir): + df_trans_fix): weo_regions_dict = dict([(k, v) for k, v @@ -1241,6 +1282,7 @@ def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, # Create formatted CSVs for each_cost in ['Capital', 'O&M']: df_costs_temp = df_costs.loc[df_costs['parameter'].str.contains(each_cost)] + df_costs_temp.drop(['technology', 'parameter'], axis = 1, inplace = True) @@ -1295,19 +1337,22 @@ def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, df_costs_final = df_costs_final[~df_costs_final['VALUE'].isnull()] if each_cost in ['Capital']: - df_costs_final = df_costs_final.merge(df_trans_capex, how='outer') - df_costs_final = apply_dtypes(df_costs_final, "CapitalCost") - df_costs_final.to_csv(os.path.join(output_data_dir, + df_costs_final_capital = df_costs_final.merge(df_trans_capex, how='outer') + df_costs_final_capital = apply_dtypes(df_costs_final_capital, "CapitalCost") + df_costs_final_capital.to_csv(os.path.join(output_data_dir, "CapitalCost.csv"), index = None) + if each_cost in ['O&M']: - df_costs_final = df_costs_final.merge(df_trans_fix, how='outer') - df_costs_final = apply_dtypes(df_costs_final, "FixedCost") - df_costs_final.to_csv(os.path.join(output_data_dir, + df_costs_final_fixed = df_costs_final.merge(df_trans_fix, how='outer') + df_costs_final_fixed = apply_dtypes(df_costs_final_fixed, "FixedCost") + df_costs_final_fixed.to_csv(os.path.join(output_data_dir, "FixedCost.csv"), index = None) + + return df_costs_final_capital, df_costs_final_fixed -def capact(df_oar_final, output_data_dir): +def capact(df_oar_final): # Create CapacityToActivityUnit csv df_capact_final = df_oar_final[['REGION', @@ -1330,7 +1375,9 @@ def capact(df_oar_final, output_data_dir): "CapacityToActivityUnit.csv"), index = None) -def activity_transmission_limit(cross_border_trade, df_oar_final, output_data_dir): + return df_capact_final + +def activity_transmission_limit(cross_border_trade, df_oar_final): # Set cross-border trade to 0 if False if not cross_border_trade: @@ -1354,7 +1401,7 @@ def activity_transmission_limit(cross_border_trade, df_oar_final, output_data_di "TotalTechnologyModelPeriodActivityUpperLimit.csv"), index = None) -def op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict, region_name): +def op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict): # Create Operational Life data tech_code_dict_reverse = dict((v,k) for k,v in tech_code_dict.items()) @@ -1374,20 +1421,22 @@ def op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict, region_nam return op_life_trn, op_life_out -def op_life_transmission(op_life_trn, op_life_out, output_data_dir, region_name): +def op_life_transmission(op_life_trn, op_life_out, op_life_dict): # transmission technologies for op_life_tech in op_life_trn: - op_life_out.append([region_name, op_life_tech, 60]) + op_life_out.append([region_name, op_life_tech, op_life_dict.get('TRN')]) - df_op_life_Out = pd.DataFrame(op_life_out, columns = ['REGION', 'TECHNOLOGY', 'VALUE']) + op_life_base = pd.DataFrame(op_life_out, columns = ['REGION', 'TECHNOLOGY', 'VALUE']) - df_op_life_Out = apply_dtypes(df_op_life_Out, "OperationalLife") - df_op_life_Out.to_csv(os.path.join(output_data_dir, + op_life_base = apply_dtypes(op_life_base, "OperationalLife") + op_life_base.to_csv(os.path.join(output_data_dir, "OperationalLife.csv"), index = None) + + return op_life_base -def cap_investment_constraints(df_iar_final, years, config, output_data_dir, region_name): +def cap_investment_constraints(df_iar_final, no_investment_techs): # Create totalAnnualMaxCapacityInvestment data @@ -1400,7 +1449,7 @@ def cap_investment_constraints(df_iar_final, years, config, output_data_dir, reg max_cap_invest_data.append([region_name, tech, year, 0]) # Do not allow investment for all xxxABCxxxxxxx technologies - no_investment_techs = config.get('no_invest_technologies') + if not no_investment_techs: no_investment_techs = [] # Change from None type to empty list max_cap_invest_techs = list(set(df_iar_final.loc[ @@ -1426,6 +1475,8 @@ def cap_investment_constraints(df_iar_final, years, config, output_data_dir, reg 'TotalAnnualMinCapacityInvestment.csv'), index = None) + return df_max_cap_invest, df_min_cap_invest + def create_sets(x, df, output_dir, custom_node_elements): """Creates a formatted otoole set csv @@ -1451,8 +1502,7 @@ def create_sets(x, df, output_dir, custom_node_elements): index = None ) -def output_sets(custom_nodes, df_oar_final, output_data_dir, custom_techs, years, - mode_list, region_name): +def output_sets(df_oar_final, custom_techs, mode_list): # ## Create sets for TECHNOLOGIES, FUELS if custom_nodes: @@ -1499,7 +1549,6 @@ def duplicatePlexosTechs(df_in, techs): df_out['TECHNOLOGY'] = [PWRCCGAFGXX02, PWROCGAFGXX02] """ df_out = df_in.copy() - # df_out = df_out.loc[df_out['TECHNOLOGY'].str[3:6].isin(techs)] df_out = df_out.loc[(df_out['TECHNOLOGY'].str[3:6].isin(techs)) & ~(df_out['TECHNOLOGY'].str.startswith('MIN'))] df_out['TECHNOLOGY'] = df_out['TECHNOLOGY'].str.slice_replace(start=11, @@ -1541,7 +1590,8 @@ def createPwrTechs(df_in, techs): df_out = df_out.drop('tech_suffix', axis = 1) return df_out -def newIar(df_in, tech): +def newIar(df_in, tech, new_iar_ccg, + new_iar_ocg, new_iar_coa, new_iar_default): """Replaces the input activity ratio value with a hardcoded value Arguments: @@ -1559,26 +1609,76 @@ def newIar(df_in, tech): df_out = df_in.loc[df_in['TECHNOLOGY'].str[3:6] == tech] if tech == 'CCG': - iar = 0.5 + iar = new_iar_ccg elif tech == 'OCG': - iar = 0.35 + iar = new_iar_ocg elif tech == 'COA': - iar = 0.33 + iar = new_iar_coa else: logging.warning(f'Default IAR used for new {tech} power plants') - iar = 1 + iar = new_iar_default df_out['VALUE'] = round(1/iar, 3) return df_out -def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life_dict): +def custom_nodes_csv(df_custom, tech_list): + '''Add custom nodes to the model for each relevant input parameter data csv. + + Args: + df : Pandas DataFrame with columns 'From' and 'To' describing the + transmission from and to contries. ie. + + Returns: + df_out : + + ''' + df_param = pd.DataFrame(list(itertools.product(custom_nodes, + tech_list, + years) + ), + columns = ['CUSTOM_NODE', + 'FUEL_TYPE', + 'YEAR'] + ) + df_param['REGION'] = region_name + df_custom = df_custom.groupby(['CUSTOM_NODE', + 'FUEL_TYPE', + 'START_YEAR', + 'END_YEAR'], + as_index=False)['CAPACITY'].sum() + df_param = pd.merge(df_param, + df_custom, + how='left', + on=['CUSTOM_NODE', + 'FUEL_TYPE']) + df_param['TECHNOLOGY'] = ('PWR' + + df_param['FUEL_TYPE'] + + df_param['CUSTOM_NODE'] + + '01') + technologies = df_param['TECHNOLOGY'].unique() + df_param.dropna(inplace=True) + df_param.drop_duplicates(inplace=True) + df_param = df_param.loc[df_param['YEAR'] >= df_param['START_YEAR']] + df_param = df_param.loc[df_param['YEAR'] <= df_param['END_YEAR']] + df_param['VALUE'] = df_param['CAPACITY'].div(1000) + df_param['REGION'] = region_name + df_param = df_param[['REGION','TECHNOLOGY','YEAR','VALUE']] + df_param = df_param.groupby(['REGION', + 'TECHNOLOGY', + 'YEAR'], + as_index=False)['VALUE'].sum() + + return df_param, technologies + +def user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, + df_min_cap_invest, df_max_cap_invest, df_res_cap, + df_iar_final, df_oar_final, fuel_set, op_life_base, cap_act_base, + cap_cost_base, df_iar_custom_val, df_oar_custom_val): """User-defined capacities are used when a specific technology must be - invested, for a given year and capacity. This is applied hrough the + invested, for a given year and capacity. This is applied through the parameter 'TotalAnnualMinCapacityInvestment'. Args: region: From config file (e.g. 'GLOBAL') - years: From config file (e.g. [2020, 2021, ... 2050]) - output_data_dir: Output directory set in config file tech_capacity: User-defined capacity in config file (e.g. TRNAGOXXCODXX: [5, 2030]) @@ -1603,22 +1703,17 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life capex_dict[tech] = tech_params[5] # tech_capacity_df = pd.DataFrame(techCapacity, columns=['TECHNOLOGY', 'VALUE', 'YEAR']) - tech_capacity_df['REGION'] = region + tech_capacity_df['REGION'] = region_name tech_capacity_df = tech_capacity_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - tech_set = pd.read_csv(os.path.join(output_data_dir, 'TECHNOLOGY.csv')) + df_tech_set = pd.read_csv(df_tech_set) for each_tech in list(tech_capacity_df['TECHNOLOGY'].unique()): - if each_tech not in list(tech_set['VALUE']): - tech_set = pd.concat([tech_set, pd.DataFrame({'VALUE':[each_tech]})]) + if each_tech not in list(df_tech_set['VALUE']): + df_tech_set = pd.concat([df_tech_set, pd.DataFrame({'VALUE':[each_tech]})]) - df_min_cap_inv = pd.read_csv(os.path.join(output_data_dir, - 'TotalAnnualMinCapacityInvestment.csv')) - df_min_cap_inv = pd.concat([df_min_cap_inv, tech_capacity_df]) + df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_df]) df_min_cap_inv.drop_duplicates(inplace=True) - - df_max_cap_inv = pd.read_csv(os.path.join(output_data_dir, - 'TotalAnnualMaxCapacityInvestment.csv')) df = pd.DataFrame(list(itertools.product(list(tech_capacity_df['TECHNOLOGY'].unique()), years) @@ -1626,7 +1721,7 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life columns = ['TECHNOLOGY', 'YEAR'] ) - df['REGION'] = region + df['REGION'] = region_name df = pd.merge(df, df_min_cap_inv, how='left', on=['REGION', 'TECHNOLOGY', 'YEAR']) @@ -1649,7 +1744,7 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life df['VALUE'] = df['TECHNOLOGY'].map(capex_dict) # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD for TRN - df_max_cap_inv = pd.concat([df_max_cap_inv, max_cap_techs_df]).drop_duplicates() + df_max_cap_inv = pd.concat([df_max_cap_invest, max_cap_techs_df]).drop_duplicates() # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD for TRN df_max_cap_inv.drop_duplicates(subset=['REGION', @@ -1685,13 +1780,12 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life >= df_res_cap_ud_final['START_YEAR']] df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] <= df_res_cap_ud_final['END_YEAR']] - df_res_cap_ud_final['REGION'] = region + df_res_cap_ud_final['REGION'] = region_name df_res_cap_ud_final = df_res_cap_ud_final[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - df_res_cap = pd.read_csv(os.path.join(output_data_dir, - 'ResidualCapacity.csv')) + df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final]) df_res_cap.to_csv(os.path.join(output_data_dir, 'ResidualCapacity.csv'), @@ -1708,16 +1802,12 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life df_min_cap_inv.to_csv(os.path.join(output_data_dir, "TotalAnnualMinCapacityInvestment.csv"), index=None) - tech_set.drop_duplicates(inplace=True) - tech_set.to_csv(os.path.join(output_data_dir, + df_tech_set.drop_duplicates(inplace=True) + df_tech_set.to_csv(os.path.join(output_data_dir, "TECHNOLOGY.csv"), index=None) # Add IAR and OAR for custom technologies - df_iar = pd.read_csv(os.path.join(output_data_dir, - 'InputActivityRatio.csv')) - df_oar = pd.read_csv(os.path.join(output_data_dir, - 'OutputActivityRatio.csv')) tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) df_iar_custom = pd.DataFrame(list(itertools.product(tech_list, [1, 2], @@ -1754,10 +1844,10 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life 'FUEL'] = ('ELC' + df_oar_custom['TECHNOLOGY'].str[3:8] + '01') - df_iar_custom['VALUE'] = 1 - df_oar_custom['VALUE'] = 0.9 - df_iar_custom['REGION'] = region - df_oar_custom['REGION'] = region + df_iar_custom['VALUE'] = df_iar_custom_val + df_oar_custom['VALUE'] = df_oar_custom_val + df_iar_custom['REGION'] = region_name + df_oar_custom['REGION'] = region_name df_iar_custom = df_iar_custom[['REGION', 'TECHNOLOGY', @@ -1772,8 +1862,8 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life 'YEAR', 'VALUE',]] - df_iar = pd.concat([df_iar, df_iar_custom]) - df_oar = pd.concat([df_oar, df_oar_custom]) + df_iar = pd.concat([df_iar_final, df_iar_custom]) + df_oar = pd.concat([df_oar_final, df_oar_custom]) df_iar.drop_duplicates(subset=['REGION', 'TECHNOLOGY', @@ -1796,30 +1886,30 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life 'OutputActivityRatio.csv'), index=None) # Add new fuels to FUEL set, if not already present - fuel_set = pd.read_csv(os.path.join(output_data_dir, 'FUEL.csv')) fuel_list = [] fuel_list = list(df_iar_custom['FUEL'].unique()) + list(df_oar_custom['FUEL'].unique()) fuel_list = list(set(fuel_list)) + + fuel_set = pd.read_csv(fuel_set) + for each_fuel in fuel_list: if each_fuel not in list(fuel_set['VALUE']): - fuel_set = fuel_set.append(pd.DataFrame({'VALUE':[each_fuel]})) + fuel_set = pd.concat([fuel_set, pd.DataFrame({'VALUE':[each_fuel]})]) fuel_set.to_csv(os.path.join(output_data_dir, "FUEL.csv"), index=None) - op_life = pd.read_csv(os.path.join(output_data_dir, - 'OperationalLife.csv')) op_life_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) op_life_custom.loc[op_life_custom['TECHNOLOGY'].str.contains('TRN'), - 'VALUE'] = 60 - op_life_custom['REGION'] = region + 'VALUE'] = op_life_dict.get('TRN') + op_life_custom['REGION'] = region_name op_life_custom = op_life_custom[['REGION', 'TECHNOLOGY', 'VALUE']] - op_life = pd.concat([op_life, op_life_custom]) + op_life = pd.concat([op_life_base, op_life_custom]) op_life.drop_duplicates(subset=['REGION', 'TECHNOLOGY'], keep='last', @@ -1827,27 +1917,22 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life op_life.to_csv(os.path.join(output_data_dir, 'OperationalLife.csv'), index=None) - # Add CapacityToActivityUnit for custom technologies - cap_act = pd.read_csv(os.path.join(output_data_dir, - 'CapacityToActivityUnit.csv')) + cap_act_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('TRN'), 'VALUE'] = 31.536 cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('PWR'), 'VALUE'] = 31.536 - cap_act_custom['REGION'] = region + cap_act_custom['REGION'] = region_name cap_act_custom = cap_act_custom[['REGION', 'TECHNOLOGY', 'VALUE']] - cap_act = pd.concat([cap_act, cap_act_custom]) + cap_act = pd.concat([cap_act_base, cap_act_custom]) cap_act.drop_duplicates(inplace=True) cap_act.to_csv(os.path.join(output_data_dir, 'CapacityToActivityUnit.csv'), index=None) - # Add CapitalCost for custom technologies - cap_cost = pd.read_csv(os.path.join(output_data_dir, - 'CapitalCost.csv')) tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) cap_cost_trn = pd.DataFrame(list(itertools.product(tech_list, years)), @@ -1859,84 +1944,29 @@ def user_defined_capacity(region, years, output_data_dir, tech_capacity, op_life cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.startswith(each_trn), 'VALUE'] = capex_dict[each_trn] - # cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.contains('TRN'), - # 'VALUE'] = 700 cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.contains('PWRTRN'), 'VALUE'] = 300 - cap_cost_trn['REGION'] = region + cap_cost_trn['REGION'] = region_name cap_cost_trn = cap_cost_trn[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - cap_cost = pd.concat([cap_cost, cap_cost_trn]) + cap_cost = pd.concat([cap_cost_base, cap_cost_trn]) cap_cost.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], keep="last", inplace=True) cap_cost.to_csv(os.path.join(output_data_dir, 'CapitalCost.csv'), index=None) - -def custom_nodes_csv(custom_nodes, df_custom, region, years, tech_list): - '''Add custom nodes to the model for each relevant input parameter data csv. - - Args: - df : Pandas DataFrame with columns 'From' and 'To' describing the - transmission from and to contries. ie. - - Returns: - df_out : - - ''' - df_param = pd.DataFrame(list(itertools.product(custom_nodes, - tech_list, - years) - ), - columns = ['CUSTOM_NODE', - 'FUEL_TYPE', - 'YEAR'] - ) - df_param['REGION'] = region - df_custom = df_custom.groupby(['CUSTOM_NODE', - 'FUEL_TYPE', - 'START_YEAR', - 'END_YEAR'], - as_index=False)['CAPACITY'].sum() - df_param = pd.merge(df_param, - df_custom, - how='left', - on=['CUSTOM_NODE', - 'FUEL_TYPE']) - df_param['TECHNOLOGY'] = ('PWR' + - df_param['FUEL_TYPE'] + - df_param['CUSTOM_NODE'] + - '01') - technologies = df_param['TECHNOLOGY'].unique() - df_param.dropna(inplace=True) - df_param.drop_duplicates(inplace=True) - df_param = df_param.loc[df_param['YEAR'] >= df_param['START_YEAR']] - df_param = df_param.loc[df_param['YEAR'] <= df_param['END_YEAR']] - df_param['VALUE'] = df_param['CAPACITY'].div(1000) - df_param['REGION'] = region - df_param = df_param[['REGION','TECHNOLOGY','YEAR','VALUE']] - df_param = df_param.groupby(['REGION', - 'TECHNOLOGY', - 'YEAR'], - as_index=False)['VALUE'].sum() - - return df_param, technologies - - -def availability_factor(region, - years, - output_data_dir, - availability): + + return df_tech_set + +def availability_factor(availability, tech_set_base): af_dict = dict(zip(list(availability['technology']), list(availability['value']))) - df_tech = pd.read_csv(os.path.join(output_data_dir, - 'TECHNOLOGY.csv')) - tech_list = [x for x in df_tech['VALUE'] + tech_list = [x for x in tech_set_base['VALUE'] if x.startswith('PWR')] df_af_final = pd.DataFrame(list(itertools.product(tech_list, years) @@ -1946,8 +1976,7 @@ def availability_factor(region, df_af_final['TECH'] = df_af_final['TECHNOLOGY'].str[3:6] df_af_final['VALUE'] = df_af_final['TECH'].map(af_dict) df_af_final.dropna(inplace=True) - df_af_final['REGION'] = region - + df_af_final['REGION'] = region_name df_af_final = df_af_final[['REGION', 'TECHNOLOGY', 'YEAR', @@ -1958,4 +1987,4 @@ def availability_factor(region, if __name__ == "__main__": main() - logging.info('Powerplant Data Created') + logging.info('Powerplant Data Created') \ No newline at end of file From cace89b63da0f7c8095bcc18b9381f1339eb8bab Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Fri, 20 Sep 2024 16:08:58 -0400 Subject: [PATCH 04/12] CP - Functional powerplant scripts standalone (untested through Snakemake) --- workflow/rules/preprocess.smk | 33 +- workflow/scripts/osemosys_global/custom.py | 48 + .../osemosys_global/powerplant/GEM_unused.py | 171 ++ .../osemosys_global/powerplant/activity.py | 306 +++ .../powerplant/activity_transmission.py | 181 ++ .../powerplant/availability.py | 32 + .../osemosys_global/powerplant/constants.py | 191 ++ .../osemosys_global/powerplant/costs.py | 130 ++ .../powerplant/costs_transmission.py | 72 + .../osemosys_global/powerplant/data.py | 333 +++ .../powerplant/data_transmission.py | 48 + .../powerplant/investment_constraints.py | 45 + .../osemosys_global/powerplant/main.py | 274 +++ .../powerplant/operational_life.py | 41 + .../osemosys_global/powerplant/read.py | 89 + .../powerplant/residual_capacity.py | 85 + .../osemosys_global/powerplant/sets.py | 42 + .../powerplant/user_defined_capacity.py | 271 +++ .../osemosys_global/powerplant/utils.py | 21 + .../osemosys_global/powerplant_data.py | 1990 ----------------- 20 files changed, 2401 insertions(+), 2002 deletions(-) create mode 100644 workflow/scripts/osemosys_global/custom.py create mode 100644 workflow/scripts/osemosys_global/powerplant/GEM_unused.py create mode 100644 workflow/scripts/osemosys_global/powerplant/activity.py create mode 100644 workflow/scripts/osemosys_global/powerplant/activity_transmission.py create mode 100644 workflow/scripts/osemosys_global/powerplant/availability.py create mode 100644 workflow/scripts/osemosys_global/powerplant/constants.py create mode 100644 workflow/scripts/osemosys_global/powerplant/costs.py create mode 100644 workflow/scripts/osemosys_global/powerplant/costs_transmission.py create mode 100644 workflow/scripts/osemosys_global/powerplant/data.py create mode 100644 workflow/scripts/osemosys_global/powerplant/data_transmission.py create mode 100644 workflow/scripts/osemosys_global/powerplant/investment_constraints.py create mode 100644 workflow/scripts/osemosys_global/powerplant/main.py create mode 100644 workflow/scripts/osemosys_global/powerplant/operational_life.py create mode 100644 workflow/scripts/osemosys_global/powerplant/read.py create mode 100644 workflow/scripts/osemosys_global/powerplant/residual_capacity.py create mode 100644 workflow/scripts/osemosys_global/powerplant/sets.py create mode 100644 workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py create mode 100644 workflow/scripts/osemosys_global/powerplant/utils.py delete mode 100644 workflow/scripts/osemosys_global/powerplant_data.py diff --git a/workflow/rules/preprocess.smk b/workflow/rules/preprocess.smk index 54d0d03a..a064bf78 100644 --- a/workflow/rules/preprocess.smk +++ b/workflow/rules/preprocess.smk @@ -103,17 +103,26 @@ rule make_data_dir: output: directory('results/data') shell: 'mkdir -p {output}' + +def powerplant_cap_custom_csv() -> str: + if config["nodes_to_add"]: + return "resources/data/custom_nodes/residual_capacity.csv" + else: + return [] + rule powerplant: message: - 'Generating powerplant data...' + "Generating powerplant data..." input: - 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx', - 'resources/data/weo_2020_powerplant_costs.csv', - 'resources/data/operational_life.csv', - 'resources/data/naming_convention_tech.csv', - 'resources/data/Costs Line expansion.xlsx', - 'resources/data/weo_region_mapping.csv', - params: + plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx', + weo_costs = 'resources/data/weo_2020_powerplant_costs.csv', + weo_regions = 'resources/data/weo_region_mapping.csv', + default_op_life = 'resources/data/operational_life.csv', + naming_convention_tech = 'resources/data/naming_convention_tech.csv', + line_data = 'resources/data/Costs Line expansion.xlsx', + default_av_factors = 'resources/data/availability_factors.csv', + custom_res_cap = powerplant_cap_custom_csv() + params: trade = config['crossborderTrade'], start_year = config['startYear'], end_year = config['endYear'], @@ -122,8 +131,8 @@ rule powerplant: csv_files = expand('results/data/{output_file}', output_file = power_plant_files) log: log = 'results/logs/powerplant.log' - shell: - 'python workflow/scripts/osemosys_global/powerplant_data.py 2> {log}' + script: + "../scripts/osemosys_global/powerplant/main.py" rule timeslice: message: @@ -181,7 +190,7 @@ rule demand_projections: iamc_urb = "resources/data/iamc_db_URB_Countries.xlsx", iamc_missing = "resources/data/iamc_db_POP_GDPppp_URB_Countries_Missing.xlsx", td_losses = "resources/data/T&D Losses.xlsx", - ember = "resources/data/ember_yearly_electricity_data.csv", + ember = "resources/data/ember_yearly_electricity_data.csv", custom_nodes = demand_custom_csv() params: start_year = config['startYear'], @@ -203,7 +212,7 @@ rule demand_projection_figures: iamc_pop = "resources/data/iamc_db_POP_Countries.xlsx", iamc_urb = "resources/data/iamc_db_URB_Countries.xlsx", iamc_missing = "resources/data/iamc_db_POP_GDPppp_URB_Countries_Missing.xlsx", - ember = "resources/data/ember_yearly_electricity_data.csv" + ember = "resources/data/ember_yearly_electricity_data.csv" output: regression = 'results/figs/regression.png', projection = 'results/figs/projection.png' diff --git a/workflow/scripts/osemosys_global/custom.py b/workflow/scripts/osemosys_global/custom.py new file mode 100644 index 00000000..03f5bae8 --- /dev/null +++ b/workflow/scripts/osemosys_global/custom.py @@ -0,0 +1,48 @@ +"""Custom Nodes logic""" + +import pandas as pd +import itertools + + +def _get_custom_demand_expected( + nodes: list[str], start_year: int, end_year: int +) -> pd.DataFrame: + """Gets formatted expected custom data""" + + years = range(start_year, end_year + 1) + + df = pd.DataFrame( + list(itertools.product(nodes, years)), columns=["CUSTOM_NODE", "YEAR"] + ) + df["REGION"] = "GLOBAL" + df["FUEL"] = "ELC" + df["CUSTOM_NODE"] + "02" + + return df + + +def import_custom_demand_data(csv: str) -> pd.DataFrame: + """Gets all custom demand data""" + return pd.read_csv(csv) + + +def get_custom_demand_data( + all_custom: pd.DataFrame, nodes: list[str], start_year: int, end_year: int +) -> pd.DataFrame: + """Gets merged custom demand data""" + + expected = _get_custom_demand_expected(nodes, start_year, end_year) + + df = pd.merge(expected, all_custom, how="left", on=["CUSTOM_NODE", "YEAR"]) + df = df[["REGION", "FUEL", "YEAR", "VALUE"]] + + return df + + +def merge_default_custom_data( + default: pd.DataFrame, custom: pd.DataFrame +) -> pd.DataFrame: + assert default.columns.equals(custom.columns) + df = pd.concat([default, custom], ignore_index=True) + df["VALUE"] = df["VALUE"].round(2) + df = df.drop_duplicates(keep="first", subset=["REGION", "FUEL", "YEAR"]) + return df diff --git a/workflow/scripts/osemosys_global/powerplant/GEM_unused.py b/workflow/scripts/osemosys_global/powerplant/GEM_unused.py new file mode 100644 index 00000000..d80d190e --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/GEM_unused.py @@ -0,0 +1,171 @@ +"""Function to calculate residual capacity for powerplant technologies.""" + +import pandas as pd +import os +from scipy import spatial +import numpy as np + +from constants import ( + start_year, + input_data_dir +) + +"""The output of the gem_cap function is currently unused. Implementation is being discussed so kept as standalone +function for the time being.""" +def gem_cap(df_gen_base, tech_code_dict, op_life_dict): + + # ### Calculate planned capacities based on the Global Energy Observatory datasets + + # Add spatial mapping to link Global Energy Observatory region naming conventions with OSeMOSYS Global + df_gem_regions = pd.read_csv(os.path.join(input_data_dir, + "gem_region_mapping.csv"), encoding = "ISO-8859-1") + + # pull locations from existing powerplants in PLEXOS-World dataset + gen_locationsinput = pd.read_excel(os.path.join(input_data_dir, + "PLEXOS_World_2015_Gold_V1.1.xlsx"), + sheet_name = "Attributes") + + gen_lat = gen_locationsinput.loc[(gen_locationsinput['class'].isin(['Battery', 'Generator'])) & + (gen_locationsinput['attribute'] == 'Latitude')].set_index('name') + + gen_long = gen_locationsinput.loc[(gen_locationsinput['class'].isin(['Battery', 'Generator'])) & + (gen_locationsinput['attribute'] == 'Longitude')].set_index('name') + + gen_locations = pd.DataFrame(gen_long.index).merge( + gen_lat['value'], left_on = 'name', right_index = True).merge( + gen_long['value'], left_on = 'name', right_index = True) + + gen_locations = pd.merge(gen_locations, df_gen_base[['node', 'country_code', 'node_code', 'powerplant']], + left_on = 'name', right_on = 'powerplant', how = 'inner') + + subcountries = gen_locations.loc[~gen_locations['node_code'].str.contains('XX')]['country_code'].str[:3].unique() + + # Set column name dictionaries for different Global Energy Monitor (gem) input datasets + gem_coal_col = {'Country' : 'Country', 'Capacity (MW)' : 'VALUE', + 'Status' : 'Status', 'Year' : 'Year_built', + 'RETIRED' : 'Year_retired', 'Planned Retire' : 'Year_retired_planned', + 'Latitude' : 'Latitude', 'Longitude' : 'Longitude'} + + gem_gas_col = {'Country' : 'Country', 'Capacity elec. (MW)' : 'VALUE', + 'Status' : 'Status', 'Start year' : 'Year_built', + 'Retired year' : 'Year_retired', 'Planned retire' : 'Year_retired_planned', + 'Latitude' : 'Latitude', 'Longitude' : 'Longitude', + 'Technology' : 'Technology'} + + # Set technology dictionary to match with OSeMOSYS global technologies + gem_gas_dict = {'CC' : 'CCG', + 'GT' : 'OCG', + 'ICCC' : 'CCG', + 'ISCC' : 'CCG', + 'ST' : 'OCG', + 'AFC' : 'CCG'} + + # Set which status criteria are used to filter datasets + new_criteria = ['operating', 'proposed', 'announced', 'pre-permit', 'permitted', 'construction']# 'shelved' & cancelled' not included + old_criteria = ['mothballed', 'retired', 'operating']# operating added because currently operating plants can already have an intended retirement year added + + # Import gem Datasets + gem_coal = pd.read_excel(os.path.join(input_data_dir, + 'Global-Coal-Plant-Tracker-Jan-2022.xlsx'), + sheet_name = 'Units', usecols = gem_coal_col.keys()) + + + gem_gas = pd.read_excel(os.path.join(input_data_dir, + 'Global-Gas-Plant-Tracker-Feb-2022.xlsx'), + sheet_name = 'Gas Units', usecols = gem_gas_col.keys()) + + # Add Technology columns and drop entries with no capacity values + gem_coal.rename(columns = gem_coal_col, inplace = True) + gem_coal['Technology'] = 'COA' + gem_coal = gem_coal[pd.to_numeric(gem_coal['VALUE'], errors='coerce').notnull()] + + gem_gas.rename(columns = gem_gas_col, inplace = True) + gem_gas['Technology'] = gem_gas['Technology'].map(gem_gas_dict) + gem_gas = gem_gas[pd.to_numeric(gem_gas['VALUE'], errors='coerce').notnull()] + + # For entries in the gem dataset with no specified gas technology same assumptions are applied as with OSeMOSYS Global + gem_gas.loc[ + (gem_gas["Technology"].isna()) & (gem_gas["VALUE"].astype(float) > 130), + "Technology", + ] = "CCG" + gem_gas.loc[ + (gem_gas["Technology"].isna()) & (gem_gas["VALUE"].astype(float) <= 130), + "Technology", + ] = "OCG" + + # Combine different datasets + gem_all = pd.concat([gem_coal, gem_gas]) + + # Add spatial mapping + gem_all = gem_all.merge(df_gem_regions, left_on = 'Country', right_on = 'gem_region') + + gem_concat = pd.DataFrame(columns = gem_all.columns) + + # Matches the lat/longs of plants in the gem datasets with lat/longs of the nearest plants in PLEXOS-World. + # The associated sub-country node of the nearest plant is assumed to be the node for the gem dataset entry. + for a in subcountries: + + gen_locations_sc = gen_locations.loc[gen_locations['country_code'].str.contains(a)].reset_index(drop = True) + gem_all_sc = gem_all.loc[gem_all['country_code'] == a].reset_index(drop = True) + + source = spatial.KDTree(gen_locations_sc[['value_x', 'value_y']].to_numpy()) + + output = pd.DataFrame(source.query([gem_all_sc[['Latitude', 'Longitude'] + ].to_numpy()])[1].transpose()) + + gem_all_sc = gem_all_sc.merge(output, left_index = True, right_index = True).set_index(0) + gem_all_sc = gem_all_sc.merge(gen_locations_sc[['node_code']], left_index = True, + right_index = True) + + gem_concat = pd.concat([gem_concat, gem_all_sc]) + + # Adds matched sub-country entries to original df and sets node codes. + gem_all = gem_all.loc[~gem_all['country_code'].isin(subcountries)] + gem_all = pd.concat([gem_all, gem_concat], axis = 0).reset_index(drop = True) + gem_all['node_code'].fillna(gem_all['country_code'] + 'XX', inplace = True) + + # Filters datframe for new plants by year entry + gem_all_new = gem_all.loc[(gem_all['Status'].isin(new_criteria)) & (gem_all['Year_built'].notna()) + ].reset_index(drop = True) + + # Strips year entry to single value (last year taken if range is given e.g. 2025-) + gem_all_new['YEAR'] = np.where(gem_all_new['Year_built'].astype(str).str.len() == 4, gem_all_new['Year_built'], + gem_all_new['Year_built'].str[-4:]) + + # Drops non-existing or non-numerical entries (e.g. 'no year found') and entries from < model_start_year + gem_all_new['YEAR'] = gem_all_new['YEAR'].apply(pd.to_numeric, errors = 'coerce') + gem_all_new = gem_all_new[(gem_all_new['YEAR'].notna()) & + (gem_all_new['YEAR'] > start_year)].reset_index() + + gem_all_retired = gem_all.loc[(gem_all['Status'].isin(old_criteria)) & (gem_all['Year_retired'].notna()) | + (gem_all['Year_retired_planned'].notna())] + + # Pulls retirement OR planned retirement year + gem_all_retired['YEAR'] = np.where(gem_all_retired['Year_retired'].notna(), gem_all_retired['Year_retired'], + gem_all_retired['Year_retired_planned']) + + # Strips year entry to single value (last year taken if range is given e.g. 2025-2030) + gem_all_retired['YEAR'] = np.where(gem_all_retired['YEAR'].astype(str).str.len().isin({4,6}), gem_all_retired['YEAR'], + gem_all_retired['YEAR'].str[-4:]) + + # Drops non-existing or non-numerical entries (e.g. 'no year found') and entries from < model_start_year + gem_all_retired['YEAR'] = gem_all_retired['YEAR'].apply(pd.to_numeric, errors = 'coerce') + gem_all_retired = gem_all_retired[(gem_all_retired ['YEAR'].notna()) & + (gem_all_retired['YEAR'] > start_year)].reset_index() + + # Group values by technology, node & year + gem_all_new_agg = gem_all_new.groupby(['node_code', 'Technology', 'YEAR'], + as_index=False)['VALUE'].sum() + + # Adds lifetime to planned capacities, calculates future retirement year and adds to retirement dataframe. + tech_code_dict_inv = {v: k for k, v in tech_code_dict.items()} + gem_all_new_agg_oplife = gem_all_new_agg.copy() + gem_all_new_agg_oplife['operational_life'] = gem_all_new_agg_oplife['Technology' + ].map(tech_code_dict_inv).map(op_life_dict) + + gem_all_new_agg_oplife['YEAR'] = gem_all_new_agg_oplife['YEAR'] + gem_all_new_agg_oplife['operational_life'] + + gem_all_retired = pd.concat([gem_all_retired, gem_all_new_agg_oplife]) + + gem_all_retired_agg = gem_all_retired.groupby(['node_code', 'Technology', 'YEAR'], + as_index=False)['VALUE'].sum() diff --git a/workflow/scripts/osemosys_global/powerplant/activity.py b/workflow/scripts/osemosys_global/powerplant/activity.py new file mode 100644 index 00000000..05be7c75 --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/activity.py @@ -0,0 +1,306 @@ +"""Function to calculate residual capacity for powerplant technologies.""" + +import pandas as pd +import itertools + +from constants import ( + region_name, + custom_nodes, + years, + new_iar_ccg, + new_iar_ocg, + new_iar_coa, + new_iar_default, +) + +from data import( + createPwrTechs, + duplicatePlexosTechs, + newIar + ) + +from utils import apply_dtypes + +def activity_master_start(df_gen_base, nodes_extra_list, duplicate_techs, mode_list): + + # ### Add input and output activity ratios + + # Create master table for activity ratios + if custom_nodes: + node_list = list(df_gen_base['node_code'].unique()) + custom_nodes + else: + node_list = list(df_gen_base['node_code'].unique()) + # Add extra nodes which are not present in 2015 but will be by 2050 + for each_node in nodes_extra_list: + if len(each_node) <= 6: + node_list.append("".join(each_node.split('-')[1:]) + 'XX') + else: + node_list.append("".join(each_node.split('-')[1:])) + + master_fuel_list = list(df_gen_base['tech_code'].unique()) + master_fuel_list.append('CCS') + + df_ratios = pd.DataFrame(list(itertools.product(node_list, + master_fuel_list, + mode_list, + years) + ), + columns = ['node_code', 'tech_code', 'MODE_OF_OPERATION', 'YEAR'] + ) + + df_ratios = createPwrTechs(df_ratios, duplicate_techs) + + return df_ratios + +def activity_output_pwr(df_ratios, thermal_fuel_list_oar): + + # #### OutputActivityRatio - Power Generation Technologies + df_oar = df_ratios.copy() + mask = df_oar['TECHNOLOGY'].apply(lambda x: x[3:6] in thermal_fuel_list_oar) + df_oar['FUEL'] = 0 + df_oar['FUEL'][mask] = 1 + df_oar = df_oar.loc[~((df_oar['MODE_OF_OPERATION'] > 1) & + (df_oar['FUEL'] == 0))] + df_oar['FUEL'] = ('ELC' + + df_oar['TECHNOLOGY'].str[6:11] + + '01' + ) + df_oar['VALUE'] = 1 + + # Add 'REGION' column and fill 'GLOBAL' throughout + df_oar['REGION'] = region_name + + # Select columns for final output table + df_oar_base = df_oar[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + + return df_oar, df_oar_base + +def activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, + df_eff_node, df_eff_tech): + + # #### InputActivityRatio - Power Generation Technologies + # Copy OAR table with all columns to IAR + df_iar = df_oar.copy() + + df_iar['FUEL'] = 0 + + # Deal with GAS techs first... OCG and CCG + # OCG Mode 1: Domestic GAS + df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & + (df_iar['TECHNOLOGY'].str[3:6].isin(['OCG'])), + 'FUEL'] = 'GAS'+df_iar['TECHNOLOGY'].str[6:9] + # OCG Mode 2: International GAS + df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & + (df_iar['TECHNOLOGY'].str[3:6].isin(['OCG'])), + 'FUEL'] = 'GASINT' + + # CCG Mode 1: Domestic GAS + df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & + (df_iar['TECHNOLOGY'].str[3:6].isin(['CCG'])), + 'FUEL'] = 'GAS'+df_iar['TECHNOLOGY'].str[6:9] + + # CCG Mode 2: International GAS + df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & + (df_iar['TECHNOLOGY'].str[3:6].isin(['CCG'])), + 'FUEL'] = 'GASINT' + + # CCS Mode 1: Domestic COA + df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & + (df_iar['TECHNOLOGY'].str[3:6].isin(['CCS'])), + 'FUEL'] = 'COA'+df_iar['TECHNOLOGY'].str[6:9] + + # CCS Mode 2: International COA + df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & + (df_iar['TECHNOLOGY'].str[3:6].isin(['CCS'])), + 'FUEL'] = 'COAINT' + + # For non-GAS thermal fuels, domestic fuel input by country in mode 1 and + # 'international' fuel input in mode 2 + df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & + (df_iar['TECHNOLOGY'].str[3:6].isin(thermal_fuel_list_iar)), + 'FUEL'] = df_iar['TECHNOLOGY'].str[3:9] + + df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & + (df_iar['TECHNOLOGY'].str[3:6].isin(thermal_fuel_list_iar)), + 'FUEL'] = df_iar['TECHNOLOGY'].str[3:6] + 'INT' + + # For renewable fuels, input by node in mode 1 + df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & + (df_iar['TECHNOLOGY'].str[3:6].isin(renewables_list)), + 'FUEL'] = df_iar['TECHNOLOGY'].str[3:11] + + # Remove mode 2 when not used + df_iar = df_iar.loc[df_iar['FUEL'] != 0] + + # Join efficiency columns: one with node and technology average, and the + # other with technology average + df_iar = df_iar.join(df_eff_node.set_index(['tech_code', 'node_code']), + on=['tech_code', 'node_code']) + + df_iar = df_iar.join(df_eff_tech.set_index('tech_code'), + on='tech_code') + + # When available, choose node and technology average. Else, + # choose technology average + df_iar['VALUE'] = df_iar['node_average_iar'] + + df_iar.loc[df_iar['TECHNOLOGY'].str.startswith('PWRCCS'), + 'tech_average_iar'] = 3 + + df_iar.loc[df_iar['VALUE'].isna(), + 'VALUE'] = df_iar['tech_average_iar'] + df_iar.drop_duplicates(inplace=True) + + # Add 'REGION' column and fill 'GLOBAL' throughout + df_iar['REGION'] = region_name + + # Select columns for final output table + df_iar_base = df_iar[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + + return df_iar_base + +def activity_upstream(df_iar_base, renewables_list, thermal_fuels_list_mining): + + # #### OutputActivityRatios - Upstream + + # We have to create a technology to produce every fuel that is input into any of the power technologies: + + df_oar_upstream = df_iar_base.copy() + + # All mining and resource technologies have an OAR of 1... + df_oar_upstream['VALUE'] = 1 + + # Renewables - set the technology as RNW + FUEL + df_oar_upstream.loc[df_oar_upstream['FUEL'].str[0:3].isin(renewables_list), + 'TECHNOLOGY'] = 'RNW'+df_oar_upstream['FUEL'] + + # If the fuel is a thermal fuel, we need to create the OAR for the mining technology... BUT NOT FOR THE INT FUELS... + df_oar_upstream.loc[df_oar_upstream['FUEL'].str[0:3].isin(thermal_fuels_list_mining) & + ~(df_oar_upstream['FUEL'].str[3:6] == "INT"), + 'TECHNOLOGY'] = 'MIN'+df_oar_upstream['FUEL'] + + # Above should get all the outputs for the MIN technologies, but we need to adjust the mode 2 ones to just the fuel code (rather than MINCOAINT) + df_oar_upstream.loc[df_oar_upstream['MODE_OF_OPERATION']==2, + 'TECHNOLOGY'] = 'MIN'+df_oar_upstream['FUEL'].str[0:3]+df_oar_upstream['TECHNOLOGY'].str[6:9] + df_oar_upstream.loc[df_oar_upstream['MODE_OF_OPERATION']==2, + 'FUEL'] = df_oar_upstream['FUEL'].str[0:3] + + # Now remove the duplicate fuels that the above created (because there's now a COA for each country, not each region, and GAS is repeated twice for each region as well): + df_oar_upstream.drop_duplicates(keep='first',inplace=True) + + # Now we have to create the MINXXXINT technologies. They are all based on the MODE_OF_OPERATION == 2: + df_oar_int = pd.DataFrame(df_oar_upstream.loc[df_oar_upstream['MODE_OF_OPERATION'] == 2, :]) + + # At this point we should have only the internationally traded fuels since they're all mode 2. So we can make the tech MINXXXINT and that's that. + df_oar_int['TECHNOLOGY'] = 'MIN'+df_oar_int['FUEL']+'INT' + # And rename the fuel to XXXINT + df_oar_int['FUEL'] = df_oar_int['FUEL']+'INT' + df_oar_int['MODE_OF_OPERATION'] = 1 # This is probably not strictly necessary as long as they're always the same in and out... + + # and de-duplicate this list: + df_oar_int.drop_duplicates(keep='first',inplace=True) + + # #### Input Activity Ratios - Upstream + + # All we need to do is take in the thermal fuels for the MINXXXINT technologies. This already exists as df_oar_int with the XXINT fuel so we can simply copy that: + df_iar_int = df_oar_int.copy() + df_iar_int['FUEL'] = df_iar_int['FUEL'].str[0:3] + + return df_oar_upstream, df_oar_int + +def activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, + df_int_trn_oar, df_iar_final, df_iar_trn, df_int_trn_iar, + duplicate_techs): + + # #### Output IAR and OAR + + # Combine the pieces from above and output to csv: + + df_oar_final = pd.concat( + [ + df_oar_base, # If we want to split transmission to different script might have to not used df_oar_final as input to this function. + df_oar_upstream, + df_oar_int, + df_oar_trn, + df_int_trn_oar, + ] + ).dropna() + + # Select columns for final output table + df_oar_final = df_oar_final[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + + df_iar_final = pd.concat( + [ + df_iar_final, + df_iar_trn, + df_int_trn_iar, + ] + ).dropna() + + # Select columns for final output table + df_iar_final = df_iar_final[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE']] + + # Add iar for techs not using PLEXOS values + df_iar_newTechs = duplicatePlexosTechs(df_iar_final, duplicate_techs) + for duplicate_tech in duplicate_techs: + df_new_iar = newIar(df_iar_newTechs, duplicate_tech,new_iar_ccg, + new_iar_ocg, new_iar_coa, new_iar_default) + df_iar_final = pd.concat([df_iar_final, df_new_iar]) + + # Add oar for techs not using PLEXOS values + df_oar_newTechs = duplicatePlexosTechs(df_oar_final, duplicate_techs) + df_oar_final = pd.concat([df_oar_final, df_oar_newTechs]) + + df_oar_final.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', + inplace=True) + + df_iar_final = apply_dtypes(df_iar_final, "InputActivityRatio") + + return df_oar_final, df_iar_final + +def capact(df_oar_final): + + # Create CapacityToActivityUnit csv + df_capact_final = df_oar_final[['REGION', + 'TECHNOLOGY' + ]] + df_capact_final = df_capact_final.drop_duplicates() + df_capact_final = (df_capact_final + .loc[(df_capact_final['TECHNOLOGY'] + .str.startswith('PWR') + ) | + (df_capact_final['TECHNOLOGY'] + .str.contains('TRN') + ) + ] + ) + + df_capact_final['VALUE'] = 31.536 + df_capact_final.drop_duplicates(inplace=True) + + return df_capact_final \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/activity_transmission.py b/workflow/scripts/osemosys_global/powerplant/activity_transmission.py new file mode 100644 index 00000000..8dbcfad7 --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/activity_transmission.py @@ -0,0 +1,181 @@ +"""Function to calaculate activity related to transmission.""" + +import pandas as pd + +from constants import ( + region_name, + start_year, + end_year +) + +from data_transmission import format_transmission_name + +from utils import apply_dtypes + +def activity_transmission(df_oar_base, df_pw_prop, df_trn_efficiencies): + + # #### Downstream Activity Ratios + + # Build transmission system outputs + + df_iar_trn = df_oar_base.copy() + + # Change the technology name to PWRTRNXXXXX + df_iar_trn["TECHNOLOGY"] = "PWRTRN" + df_iar_trn["FUEL"].str[3:8] + # Make all modes of operation 1 + df_iar_trn["MODE_OF_OPERATION"] = 1 + # And remove all the duplicate entries + df_iar_trn.drop_duplicates(keep="first", inplace=True) + + # OAR for transmission technologies is IAR, but the fuel is 02 instead of 01: + df_oar_trn = df_iar_trn.copy() + df_oar_trn["FUEL"] = df_oar_trn["FUEL"].str[0:8] + "02" + + # Build international transmission system from original input data, but for Line rather than Generator: + int_trn_cols = ["child_class", "child_object", "property", "value"] + df_int_trn = df_pw_prop[int_trn_cols] + df_int_trn = df_int_trn[df_int_trn["child_class"] == "Line"] + + # For IAR and OAR we can drop the value: + df_int_trn = df_int_trn.drop(["child_class", "value"], axis=1) + + # Create MofO column based on property: + df_int_trn["MODE_OF_OPERATION"] = 1 + df_int_trn.loc[df_int_trn["property"] == "Min Flow", "MODE_OF_OPERATION"] = 2 + + # Use the child_object column to build the technology names: + df_int_trn["codes"] = df_int_trn["child_object"].str.split(pat="-") + + # If there are only two locations, then the node is XX + df_int_trn.loc[df_int_trn["codes"].str.len() == 2, "TECHNOLOGY"] = ( + "TRN" + df_int_trn["codes"].str[0] + "XX" + df_int_trn["codes"].str[1] + "XX" + ) + # If there are four locations, the node is already included + df_int_trn.loc[df_int_trn["codes"].str.len() == 4, "TECHNOLOGY"] = ( + "TRN" + + df_int_trn["codes"].str[0] + + df_int_trn["codes"].str[1] + + df_int_trn["codes"].str[2] + + df_int_trn["codes"].str[3] + ) + # If there are three items, and the last item is two characters, then the second item is an XX: + df_int_trn.loc[ + (df_int_trn["codes"].str.len() == 3) & (df_int_trn["codes"].str[2].str.len() == 2), + "TECHNOLOGY", + ] = ( + "TRN" + + df_int_trn["codes"].str[0] + + "XX" + + df_int_trn["codes"].str[1] + + df_int_trn["codes"].str[2] + ) + # If there are three items, and the last item is three characters, then the last item is an XX: + df_int_trn.loc[ + (df_int_trn["codes"].str.len() == 3) & (df_int_trn["codes"].str[2].str.len() == 3), + "TECHNOLOGY", + ] = ( + "TRN" + + df_int_trn["codes"].str[0] + + df_int_trn["codes"].str[1] + + df_int_trn["codes"].str[2] + + "XX" + ) + + # Set the value (of either IAR or OAR) to 1 + df_int_trn["VALUE"] = 1 + df_int_trn["REGION"] = region_name + + df_int_trn = df_int_trn.drop(["property", "child_object", "codes"], axis=1) + df_int_trn["YEAR"] = start_year + + # add in future years + df_int_trn_new = df_int_trn.copy() + df_int_trn_new["YEAR"] = [range(start_year + 1, end_year + 1)] * len(df_int_trn_new) + df_int_trn_new = df_int_trn_new.explode(column="YEAR") + + df_int_trn = pd.concat([df_int_trn, df_int_trn_new]).reset_index(drop=True) + + # Now create the input and output activity ratios + df_int_trn_oar = df_int_trn.copy() + df_int_trn_iar = df_int_trn.copy() + + # IAR Mode 1 is input from first country: + df_int_trn_iar.loc[df_int_trn_iar["MODE_OF_OPERATION"] == 1, "FUEL"] = ( + "ELC" + df_int_trn_iar["TECHNOLOGY"].str[3:8] + "02" + ) + # IAR Mode 2 is input from second country: + df_int_trn_iar.loc[df_int_trn_iar["MODE_OF_OPERATION"] == 2, "FUEL"] = ( + "ELC" + df_int_trn_iar["TECHNOLOGY"].str[8:13] + "02" + ) + + # OAR Mode 2 is output to first country: + df_int_trn_oar.loc[df_int_trn_oar["MODE_OF_OPERATION"] == 2, "FUEL"] = ( + "ELC" + df_int_trn_oar["TECHNOLOGY"].str[3:8] + "01" + ) + # OAR Mode 1 is out to the second country: + df_int_trn_oar.loc[df_int_trn_oar["MODE_OF_OPERATION"] == 1, "FUEL"] = ( + "ELC" + df_int_trn_oar["TECHNOLOGY"].str[8:13] + "01" + ) + + # Drop unneeded columns + df_trn_efficiencies = df_trn_efficiencies.drop( + [ + "Interface", + "KM distance", + "HVAC/HVDC/Subsea", + "Build Cost ($2010 in $000)", + "Build cost + FOM (3.5% of Capex per year)", + "Unnamed: 8", + "Interface is per MW", + "Unnamed: 10", + "Unnamed: 11", + "Unnamed: 12", + "Subsea lines", + "Unnamed: 14" + ], + axis=1, + ) + + # Drop NaN values + df_trn_efficiencies = df_trn_efficiencies.dropna(subset=["From"]) + + # Create tech column from To and From Codes: + df_trn_efficiencies = format_transmission_name(df_trn_efficiencies) + + # Rename column 'VALUES' + df_trn_efficiencies = df_trn_efficiencies.rename(columns={"Losses": "VALUE"}) + + # And adjust OAR values to be output amounts vs. losses: + df_trn_efficiencies['VALUE'] = 1.0 - df_trn_efficiencies['VALUE'] + + # and add values into OAR matrix + df_int_trn_oar = df_int_trn_oar.drop(["VALUE"], axis=1) + df_int_trn_oar = pd.merge( + df_int_trn_oar, df_trn_efficiencies, how="outer", on="TECHNOLOGY" + ) + + return df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar + +def activity_transmission_limit(cross_border_trade, df_oar_final): + + # Set cross-border trade to 0 if False + if not cross_border_trade: + df_crossborder_final = df_oar_final[['REGION', + 'TECHNOLOGY' + ]] + df_crossborder_final = df_crossborder_final.drop_duplicates() + df_crossborder_final = (df_crossborder_final + .loc[df_crossborder_final['TECHNOLOGY'] + .str.startswith('TRN') + ] + ) + df_crossborder_final['VALUE'] = 0 + else: + df_crossborder_final = pd.DataFrame(columns=['REGION', + 'TECHNOLOGY', + 'VALUE']) + + df_crossborder_final = apply_dtypes(df_crossborder_final, + "TotalTechnologyModelPeriodActivityUpperLimit") + + return df_crossborder_final \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/availability.py b/workflow/scripts/osemosys_global/powerplant/availability.py new file mode 100644 index 00000000..3caba9b1 --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/availability.py @@ -0,0 +1,32 @@ +"""Function to calculate residual capacity for powerplant technologies.""" + +import pandas as pd +import itertools + +from constants import( + years, + region_name + ) + +def availability_factor(availability, tech_set_base): + + af_dict = dict(zip(list(availability['technology']), + list(availability['value']))) + + tech_list = [x for x in tech_set_base['VALUE'] + if x.startswith('PWR')] + df_af_final = pd.DataFrame(list(itertools.product(tech_list, + years) + ), + columns = ['TECHNOLOGY', 'YEAR'] + ) + df_af_final['TECH'] = df_af_final['TECHNOLOGY'].str[3:6] + df_af_final['VALUE'] = df_af_final['TECH'].map(af_dict) + df_af_final.dropna(inplace=True) + df_af_final['REGION'] = region_name + df_af_final = df_af_final[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + + return df_af_final \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/constants.py b/workflow/scripts/osemosys_global/powerplant/constants.py new file mode 100644 index 00000000..0c96618c --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/constants.py @@ -0,0 +1,191 @@ +"""Constants for the powerplant module""" + +"""Change IAR for CSP value taken from PLEXOS to 1.0. +Relevant for the 'average_efficiency' function.""" +avg_csp_eff = 1 + +"""Change IAR for URN value taken from PLEXOS to 2.2 (45%). +Relevant for the 'average_efficiency' function.""" +avg_urn_eff = 0.45 + +"""Technologies that will have 00 and 01 suffixes to represent PLEXOS + historical values and future values. Relevant for the 'residual_capacity' + and activity functions.""" +duplicate_techs = ['CCG', 'OCG'] + +"""Add extra nodes which exist in 2050 but are not in the 2015 PLEXOS-World +data to enable their addition to the workflow. Relevant for the +'generator_table' and activity functions""" +nodes_extra_list = ['AF-SOM', + 'AF-TCD', + 'AS-TLS', + 'EU-MLT', + 'NA-BLZ', + 'NA-HTI', + 'SA-BRA-J1', + 'SA-BRA-J2', + 'SA-BRA-J3', + 'SA-SUR'] + +"""Sets the mode of operations for technologies. +Relevant for the activity_master_start function.""" +mode_list = [1,2] + +"""List of technologies (thermal) for which output activity ratios need +to be developed. Relevant for the 'activity_output_pwr' function.""" +thermal_fuel_list_oar = ['COA', + 'COG', + 'OCG', + 'CCG', + 'PET', + 'URN', + 'OIL', + 'OTH', + 'CCS' + ] + +"""List of technologies (thermal) for which input activity ratios need +to be developed. Relevant for the 'activity_input_pwr' function.""" +thermal_fuel_list_iar = ['COA', + 'COG', + 'PET', + 'URN', + 'OIL', + 'OTH' + ] + +"""List of upstream fuels for which output activity ratios need +to be developed. Relevant for the 'activity_upstream' function.""" +thermal_fuel_list_mining = ['COA', + 'COG', + 'GAS', + 'PET', + 'URN', + 'OIL', + 'OTH' + ] + +"""List of technologies (renewable) for which activity ratios need +to be developed. Relevant for the 'activity_input_pwr' and +'activity_upstream' functions.""" +renewables_list = ['BIO', + 'GEO', + 'HYD', + 'SPV', + 'CSP', + 'WAS', + 'WAV', + 'WON', + 'WOF'] + +"""Technology input for the mapping of WEO cost data to OG technologies. +Relevant for the 'costs_pwr' function.""" +costs_dict = {'Biomass - waste incineration - CHP':'WAS', + 'Biomass Power plant':'BIO', + 'CCGT':'CCG', + 'CCGT - CHP':'COG', + 'Concentrating solar power':'CSP', + 'Gas turbine':'OCG', + 'Geothermal':'GEO', + 'Hydropower - large-scale':'HYD', + 'Marine':'WAV', + 'Nuclear':'URN', + 'Solar photovoltaics - Large scale':'SPV', + 'Steam Coal - SUBCRITICAL':'COA', + 'Steam Coal - SUPERCRITICAL':'COA', + 'Steam Coal - ULTRASUPERCRITICAL':'COA', + 'Wind onshore':'WON', + 'Wind offshore':'WOF', + 'Petroleum':'PET', + 'Oil':'OIL', + 'Other':'OTH', + 'IGCC + CCS':'CCS', + 'Coal* + CCS':'CCS',} + +"""setings for custom iar values deviating from derived values from the + PLEXOS-World dataset. Relevant for the 'newIar' function.""" +new_iar_ccg = 0.5 +new_iar_ocg = 0.35 +new_iar_coa = 0.33 +new_iar_default = 1 + +"""Set iar and oar values for custom transmission entries. I.e. oar of 0.9 +assumes 10% losses. Relevant for the user_defined_capacity function.""" +df_iar_custom_val = 1 +df_oar_custom_val = 0.9 + +"""Set column name dictionaries for different Global Energy Monitor (gem) input datasets""" +gem_coal_col = {'Country' : 'Country', 'Capacity (MW)' : 'VALUE', + 'Status' : 'Status', 'Year' : 'Year_built', + 'RETIRED' : 'Year_retired', 'Planned Retire' : 'Year_retired_planned', + 'Latitude' : 'Latitude', 'Longitude' : 'Longitude'} + +gem_gas_col = {'Country' : 'Country', 'Capacity elec. (MW)' : 'VALUE', + 'Status' : 'Status', 'Start year' : 'Year_built', + 'Retired year' : 'Year_retired', 'Planned retire' : 'Year_retired_planned', + 'Latitude' : 'Latitude', 'Longitude' : 'Longitude', + 'Technology' : 'Technology'} + +"""Set technology dictionary to match with OSeMOSYS global technologies""" +gem_gas_dict = {'CC' : 'CCG', + 'GT' : 'OCG', + 'ICCC' : 'CCG', + 'ISCC' : 'CCG', + 'ST' : 'OCG', + 'AFC' : 'CCG'} + +if "snakemake" in globals(): + file_plexos = snakemake.input.plexos + file_default_op_life = snakemake.input.default_op_life + file_naming_convention_tech = snakemake.input.naming_convention_tech + file_weo_costs = snakemake.input.weo_costs + file_weo_regions = snakemake.input.weo_regions + file_line_data = snakemake.input.line_data + file_default_av_factors = snakemake.input.default_av_factors + file_custom_res_cap = snakemake.input.custom_res_cap + start_year = snakemake.params.start_year + end_year = snakemake.params.end_year + region_name = snakemake.params.region_name + custom_nodes = snakemake.params.custom_nodes + custom_nodes_data = snakemake.input.custom_nodes + tech_capacity = snakemake.params.user_defined_capacity + cross_border_trade = snakemake.params.crossborderTrade + no_investment_techs = snakemake.params.no_invest_technologies + output_data_dir = snakemake.params.output_data_dir + input_data_dir = snakemake.params.output_data_dir + +else: + file_plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx' + file_default_op_life = 'resources/data/operational_life.csv' + file_naming_convention_tech = 'resources/data/naming_convention_tech.csv' + file_weo_costs = 'resources/data/weo_2020_powerplant_costs.csv' + file_weo_regions = 'resources/data/weo_region_mapping.csv' + file_line_data = 'resources/data/Costs Line expansion.xlsx' + file_default_av_factors = 'resources/data/availability_factors.csv' + file_custom_res_cap = 'resources/data/custom_nodes/residual_capacity.csv' + start_year = 2020 + end_year = 2050 + region_name = 'GLOBAL' + custom_nodes = ["INDWE", "INDEA", "INDNE", "INDNO", "INDSO"] + custom_nodes_data = 'resources/data/custom_nodes/specified_annual_demand.csv' + tech_capacity = {'TRNINDEAINDNE': [0, 2030, "open", 2030, 10, 861]} + cross_border_trade = True + no_investment_techs = ["CSP", "WAV", "URN", "OTH", "WAS", + "COG", "GEO", "BIO", "PET"] + output_data_dir = 'results/data' + input_data_dir = 'resources/data' + +years = list(range(start_year, end_year + 1)) + +SET_DTYPES = { + "DAILYTIMEBRACKET": int, + "EMISSION":str, + "FUEL":str, + "MODE_OF_OPERATION":int, + "REGION":str, + "SEASON":str, + "STORAGE":str, + "TECHNOLOGY":str, + "TIMESLICE":str, + "YEAR":int, +} \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/costs.py b/workflow/scripts/osemosys_global/powerplant/costs.py new file mode 100644 index 00000000..cf9426dd --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/costs.py @@ -0,0 +1,130 @@ +"""Function to calculate residual capacity for powerplant technologies.""" + +import pandas as pd +import numpy as np +from utils import apply_dtypes + +def costs_pwr(df_weo_data, costs_dict): + + # ### Costs: Capital, fixed, and variable + + df_costs = pd.melt(df_weo_data, + id_vars = ['technology', 'weo_region', 'parameter'], + value_vars = ['2019', '2030', '2040'], + var_name = 'YEAR') + df_costs['parameter'] = df_costs['parameter'].str.split('\r\n').str[0] + df_costs['value'] = df_costs['value'].replace({'n.a.':0}) + df_costs['value'] = df_costs['value'].astype(float) + df_costs = df_costs.pivot_table(index = ['technology', 'parameter', 'YEAR'], + columns = 'weo_region', + values = 'value').reset_index() + df_costs['AS_average'] = (df_costs['China'] + + df_costs['India'] + + df_costs['Japan'] + + df_costs['Middle East']).div(4) + df_costs['SEA_average'] = (df_costs['Indonesia'] + + df_costs['Vietnam']).div(2) + df_costs['NA_average'] = (df_costs['United States']) + df_costs['SA_average'] = (df_costs['Brazil']) + df_costs['Global_average'] = (df_costs['Africa'] + + df_costs['Brazil'] + + df_costs['Europe'] + + df_costs['China'] + + df_costs['India'] + + df_costs['Japan'] + + df_costs['Middle East'] + + df_costs['Russia'] + + df_costs['United States']).div(9) + df_costs = pd.melt(df_costs, + id_vars = ['technology', 'parameter', 'YEAR'], + value_vars = [x + for x + in df_costs.columns + if x not in ['technology', 'parameter', 'YEAR'] + ] + ) + df_costs['YEAR'] = df_costs['YEAR'].astype(int) + + df_costs = df_costs.loc[df_costs['technology'].isin(costs_dict.keys())] + df_costs['technology_code'] = df_costs['technology'].replace(costs_dict) + + return df_costs + +def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, + df_trans_fix): + + weo_regions_dict = dict([(k, v) + for k, v + in zip(df_weo_regions['technology_code'], + df_weo_regions['weo_region'] + ) + ] + ) + + # Create formatted CSVs + for each_cost in ['Capital', 'O&M']: + df_costs_temp = df_costs.loc[df_costs['parameter'].str.contains(each_cost)] + + df_costs_temp.drop(['technology', 'parameter'], + axis = 1, + inplace = True) + df_costs_final = df_oar_final[['REGION', + 'TECHNOLOGY', + 'YEAR' + ]] + df_costs_final['YEAR'] = df_costs_final['YEAR'].astype(int) + df_costs_final = df_costs_final.drop_duplicates() + df_costs_final = (df_costs_final + .loc[(df_costs_final['TECHNOLOGY'] + .str.startswith('PWR') + ) & + (~df_costs_final['TECHNOLOGY'] + .str.contains('TRN') + ) + ] + ) + df_costs_final['technology_code'] = df_costs_final['TECHNOLOGY'].str[3:6] + df_costs_final['weo_region'] = df_costs_final['TECHNOLOGY'].str[6:9] + df_costs_final['weo_region'] = (df_costs_final['weo_region'] + .replace(weo_regions_dict)) + + df_costs_final = pd.merge(df_costs_final, + df_costs_temp, + on = ['technology_code', 'weo_region', 'YEAR'], + how = 'left' + ) + df_costs_final.drop(['technology_code', 'weo_region'], + axis = 1, + inplace = True) + df_costs_final = df_costs_final.fillna(-9) + df_costs_final = pd.pivot_table(df_costs_final, + index = ['REGION', 'YEAR'], + columns = 'TECHNOLOGY', + values = 'value').reset_index() + df_costs_final = df_costs_final.replace([-9],[np.nan]) + + df_costs_final = df_costs_final.interpolate(method = 'linear', + limit_direction='forward').round(2) + df_costs_final = df_costs_final.interpolate(method = 'linear', + limit_direction='backward').round(2) + df_costs_final = pd.melt(df_costs_final, + id_vars = ['REGION', 'YEAR'], + value_vars = [x for x in df_costs_final.columns + if x not in ['REGION', 'YEAR'] + ], + var_name = 'TECHNOLOGY', + value_name = 'VALUE' + ) + df_costs_final = df_costs_final[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] + df_costs_final = df_costs_final[~df_costs_final['VALUE'].isnull()] + + if each_cost in ['Capital']: + df_costs_final_capital = df_costs_final.merge(df_trans_capex, how='outer') + df_costs_final_capital = apply_dtypes(df_costs_final_capital, "CapitalCost") + + + if each_cost in ['O&M']: + df_costs_final_fixed = df_costs_final.merge(df_trans_fix, how='outer') + df_costs_final_fixed = apply_dtypes(df_costs_final_fixed, "FixedCost") + + return df_costs_final_capital, df_costs_final_fixed \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/costs_transmission.py b/workflow/scripts/osemosys_global/powerplant/costs_transmission.py new file mode 100644 index 00000000..506c19f3 --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/costs_transmission.py @@ -0,0 +1,72 @@ +"""Function to calculate residual capacity for powerplant technologies.""" + +from data_transmission import format_transmission_name + +from constants import( + region_name, + years + ) + +def get_transmission_costs(df_trn_lines, df_oar_final): + '''Gets electrical transmission capital and fixed cost per technology. + + Both the capital costs and fixed cost are written out to avoid having + to read in the excel file twice + + Returns: + df_trans_capex: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' + representing CAPITAL cost in millions of dollars per year. + df_trans_fix: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' + representing FIXED cost in millions of dollars per year. + ''' + + # Drop unneeded columns + df = df_trn_lines.drop( + [ + "Line", + "KM distance", + "HVAC/HVDC/Subsea", + "Losses", + "Unnamed: 8", + "Line Max Size (MW)", + "Unnamed: 10", + "Unnamed: 11", + "Unnamed: 12", + "Subsea lines", + "Unnamed: 14" + ], + axis=1, + ) + + # Use to/from codes to create a TECHNOLOGY columns + df = format_transmission_name(df) + + # Changes units + # Raw given in dollars -> model units in million dollars + df = df.rename(columns={'Annual FO&M (3.5% of CAPEX) ($2010 in $000)':'O&M'}) + df = df.rename(columns={'Build Cost ($2010 in $000)':'CAPEX'}) + df['O&M'] = df['O&M'].astype(float) / 1000 + df['CAPEX'] = df['CAPEX'].astype(float) / 1000 + df = df.round({'O&M': 3, 'CAPEX': 3, }) + + # Separate out fixed and capex and return values + df_trans_capex = df.drop(['O&M'], axis=1) + df_trans_capex = df_trans_capex.rename(columns={'CAPEX':'VALUE'}) + df_trans_fix = df.drop(['CAPEX'], axis=1) + df_trans_fix = df_trans_fix.rename(columns={'O&M':'VALUE'}) + + df_trans_capex['REGION'] = region_name + df_trans_capex['YEAR'] = [years] * len(df_trans_capex) + df_trans_capex = df_trans_capex.explode('YEAR') + + df_trans_fix['REGION'] = region_name + df_trans_fix['YEAR'] = [years] * len(df_trans_fix) + df_trans_fix = df_trans_fix.explode('YEAR') + + # Filter out techs that don't have activity ratios + df_trans_capex = df_trans_capex.loc[ + df_trans_capex['TECHNOLOGY'].isin(df_oar_final['TECHNOLOGY'])] + df_trans_fix = df_trans_fix.loc[ + df_trans_fix['TECHNOLOGY'].isin(df_oar_final['TECHNOLOGY'])] + + return df_trans_capex, df_trans_fix \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/data.py b/workflow/scripts/osemosys_global/powerplant/data.py new file mode 100644 index 00000000..d7eb94c3 --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/data.py @@ -0,0 +1,333 @@ +"""Functions to extract relevent data""" + +import pandas as pd +import numpy as np +from datetime import datetime +import itertools +import logging + +from constants import ( + nodes_extra_list, + start_year, + region_name, + custom_nodes, + years +) + +def set_generator_table(plexos_prop: pd.DataFrame, plexos_memb: pd.DataFrame, + op_life_dict: dict[str, int], tech_code_dict: dict[str, str]) -> pd.DataFrame: + """Sets the main generator table derived from the PLEXOS-World model. + """ + + # Create main generator table + gen_cols_1 = ["child_class", "child_object", "property", "value"] + df_gen = plexos_prop.copy()[gen_cols_1] + + df_gen = df_gen[df_gen["child_class"] == "Generator"] + df_gen.rename(columns={"child_object": "powerplant"}, inplace=True) + df_gen.drop("child_class", axis=1, inplace=True) + df_gen = pd.pivot_table(df_gen, + index="powerplant", + columns="property", + values="value", + aggfunc=np.sum, + fill_value=0, + ) + df_gen["total_capacity"] = (df_gen["Max Capacity"].astype(float)) * ( + df_gen["Units"].astype(int) + ) + + gen_cols_base = ["Commission Date", "Heat Rate", "Max Capacity", "total_capacity"] + df_gen_base = df_gen[gen_cols_base] + + df_dict = plexos_memb[plexos_memb["parent_class"] == "Generator"].rename( + {"parent_object": "powerplant"}, axis=1 + ) + + ## Compile dataframe with powerplants, nodes, and fuels + df_dict_fuel = df_dict[df_dict["collection"] == "Fuels"] + df_dict_fuel = df_dict_fuel[["powerplant", "child_object"]] + df_dict_nodes = df_dict[df_dict["collection"] == "Nodes"] + df_dict_nodes = df_dict_nodes[["powerplant", "child_object"]] + df_dict_2 = pd.merge(df_dict_fuel, df_dict_nodes, how="outer", on="powerplant") + + + ## Merge original generator dataframe with nodes and fuels + df_gen_base = pd.merge(df_gen_base, df_dict_2, how="outer", on="powerplant") + df_gen_base.rename( + {"child_object_x": "fuel", "child_object_y": "node"}, axis=1, inplace=True + ) + + ## Extract start year from Commission Date + df_gen_base["Commission Date"] = (pd.TimedeltaIndex(df_gen_base["Commission Date"].astype(int), + unit='d') + + datetime(1900, 1, 1)) + df_gen_base["start_year"] = df_gen_base["Commission Date"].dt.year + df_gen_base.drop("Commission Date", axis=1, inplace=True) + + ## Calculate efficiency from heat rate. Units of heat rate in MJ/kWh + df_gen_base["efficiency"] = 3.6 / df_gen_base["Heat Rate"].astype(float) + df_gen_base.drop("Heat Rate", axis=1, inplace=True) + + ## Calcluate years of operation from start year + df_gen_base["years_of_operation"] = start_year - df_gen_base["start_year"] + + ## Fix blank spaces in 'fuels' columns. Appearing for 'Oil' powerplants in certain countries + df_gen_base.loc[df_gen_base["fuel"].isna(), "fuel"] = ( + df_gen_base["node"].str.split("-").str[:2].str.join("-") + + " " + + df_gen_base["powerplant"].str.split("_", expand=True)[1] + ) + + ## Create column for technology + df_gen_base["technology"] = df_gen_base["powerplant"].str.split("_").str[1] + df_gen_base["technology"] = df_gen_base["technology"].str.title() + + ## Divide Gas into CCGT and OCGT based on max capacity + df_gen_base.loc[ + (df_gen_base["technology"] == "Gas") & (df_gen_base["Max Capacity"].astype(float) > 130), + "technology", + ] = "Gas-CCGT" + df_gen_base.loc[ + (df_gen_base["technology"] == "Gas") & (df_gen_base["Max Capacity"].astype(float) <= 130), + "technology", + ] = "Gas-OCGT" + + # Create table with aggregated capacity + df_gen_agg_node = df_gen_base[df_gen_base['start_year']<=start_year] + df_gen_agg_node = df_gen_agg_node.groupby(['node', 'technology'], + as_index=False)['total_capacity'].sum() + df_gen_agg_node = df_gen_agg_node.pivot(index='node', + columns='technology', + values='total_capacity').fillna(0).reset_index() + + df_gen_agg_node.drop('Sto', axis=1, inplace=True) # Drop 'Sto' technology. Only for USA. + + nodes_extra_df = pd.DataFrame(columns=['node']) + + nodes_extra_df['node'] = nodes_extra_list + + df_gen_agg_node = pd.concat( + [df_gen_agg_node,nodes_extra_df], + ignore_index=True, + sort=False, + ).fillna(0).sort_values(by='node').set_index('node').round(2) + + # Add region and country code columns + df_gen_base['region_code'] = df_gen_base['node'].str[:2] + df_gen_base['country_code'] = df_gen_base['node'].str[3:] + + df_gen_base['operational_life'] = df_gen_base['technology'].map(op_life_dict) + df_gen_base['retirement_year_data'] = (df_gen_base['operational_life'] + + df_gen_base['start_year']) + df_gen_base['retirement_diff'] = ((df_gen_base['years_of_operation'] + - df_gen_base['operational_life'])/ + df_gen_base['operational_life']) + + ''' Set retirement year based on years of operation. + If (years of operation - operational life) is more than 50% of + operational life, set retirement year + ''' + df_gen_base.loc[df_gen_base['retirement_diff'] >= 0.5, + 'retirement_year_model'] = 2028 + df_gen_base.loc[(df_gen_base['retirement_diff'] < 0.5) & + (df_gen_base['retirement_diff'] > 0), + 'retirement_year_model'] = 2033 + df_gen_base.loc[df_gen_base['retirement_diff'] <= 0, + 'retirement_year_model'] = df_gen_base['retirement_year_data'] + + df_gen_base['tech_code'] = df_gen_base['technology'].map(tech_code_dict) + + df_gen_base.loc[df_gen_base['node'].str.len() <= 6, + 'node_code'] = (df_gen_base['node']. + str.split('-'). + str[1:]. + str.join("") + + 'XX') + df_gen_base.loc[df_gen_base['node'].str.len() > 6, + 'node_code'] = (df_gen_base['node']. + str.split('-'). + str[1:]. + str.join("") + ) + + df_gen_base = df_gen_base.loc[~df_gen_base['tech_code'].isna()] + + return df_gen_base + +def average_efficiency(df_gen_base, avg_csp_eff, avg_urn_eff): + + # ### Calculate average InputActivityRatio by node+technology and only by technology + df_eff = df_gen_base[['node_code', + 'efficiency', + 'tech_code']] + + # Change IAR for CSP value taken from PLEXOS + df_eff.loc[df_eff['tech_code']=='CSP', 'efficiency'] = avg_csp_eff + + # Change IAR for URN value taken from PLEXOS + df_eff.loc[df_eff['tech_code']=='URN', 'efficiency'] = avg_urn_eff + + # Average efficiency by node and technology + df_eff_node = df_eff.groupby(['tech_code', + 'node_code'], + as_index = False).mean() + + df_eff_node['node_average_iar'] = ((1 / df_eff_node['efficiency']). + round(2)) + + df_eff_node.drop('efficiency', + axis = 1, + inplace = True) + + # Average efficiency by technology + df_eff_tech = df_eff.drop(columns="node_code").groupby('tech_code', as_index = False).mean() + + df_eff_tech['tech_average_iar'] = ((1 / df_eff_tech['efficiency']). + round(2)) + + df_eff_tech.drop('efficiency', + axis = 1, + inplace = True) + + return df_eff_node, df_eff_tech + +def createPwrTechs(df_in, techs): + """Formats power generation technology name + + Adds a 'TECHNOLOGY' column to a dataframe with formatted power + generation technology (PWR) names. Names are formatted so the suffix + for plants in the 'techs' argument list will have 00, while everything + else will have an 01 suffix + + Arguments: + df_in = dataframe with a 'tech_codes' and 'node_codes' column + tList = List of technology triads to have 00 suffix [CCG, HYD, ...] + + Returns: + df_out = same dataframe as df_in, except with a 'TECHNOLOGY' column added + to the end + + Example: + df_in['tech_code'] = ('CCG', SPV, 'OCG', 'HYD') + df_in['node_code'] = ('AGOXX', AGOXX, 'INDNP', 'INDNP') + df_out = createPwrTechs(df_in, ['CCG', 'OCG']) + df_out['TECHNOLOGY'] = [PWRCCGAFGXX00, PWRSPVAFGXX01, PWROCGINDNP00, PWRHYDINDNP01] + """ + df_out = df_in.copy() + for t in techs: + df_out.loc[df_out['tech_code'] == t, 'tech_suffix'] = '00' + df_out['tech_suffix'] = df_out['tech_suffix'].fillna('01') + df_out['TECHNOLOGY'] = ('PWR' + + df_out['tech_code'] + + df_out['node_code'] + + df_out['tech_suffix'] + ) + df_out = df_out.drop('tech_suffix', axis = 1) + return df_out + +def custom_nodes_csv(df_custom, tech_list): + '''Add custom nodes to the model for each relevant input parameter data csv. + + Args: + df : Pandas DataFrame with columns 'From' and 'To' describing the + transmission from and to contries. ie. + + Returns: + df_out : + + ''' + df_param = pd.DataFrame(list(itertools.product(custom_nodes, + tech_list, + years) + ), + columns = ['CUSTOM_NODE', + 'FUEL_TYPE', + 'YEAR'] + ) + df_param['REGION'] = region_name + df_custom = df_custom.groupby(['CUSTOM_NODE', + 'FUEL_TYPE', + 'START_YEAR', + 'END_YEAR'], + as_index=False)['CAPACITY'].sum() + df_param = pd.merge(df_param, + df_custom, + how='left', + on=['CUSTOM_NODE', + 'FUEL_TYPE']) + df_param['TECHNOLOGY'] = ('PWR' + + df_param['FUEL_TYPE'] + + df_param['CUSTOM_NODE'] + + '01') + technologies = df_param['TECHNOLOGY'].unique() + df_param.dropna(inplace=True) + df_param.drop_duplicates(inplace=True) + df_param = df_param.loc[df_param['YEAR'] >= df_param['START_YEAR']] + df_param = df_param.loc[df_param['YEAR'] <= df_param['END_YEAR']] + df_param['VALUE'] = df_param['CAPACITY'].div(1000) + df_param['REGION'] = region_name + df_param = df_param[['REGION','TECHNOLOGY','YEAR','VALUE']] + df_param = df_param.groupby(['REGION', + 'TECHNOLOGY', + 'YEAR'], + as_index=False)['VALUE'].sum() + + return df_param, technologies + +def duplicatePlexosTechs(df_in, techs): + """Creates new technologies to replace PLEXOS technolgoies. + + New technologies will end in '01', while historical ones end in '00' + + Arguments: + df_in = dataframe in otoole and og formatting with a TECHNOLOGY column + techs = List of technology triads to duplicate [CCG, HYD, ...] + + Returns: + df_out = dataframe with same columns as df_in. All tech names that include + techs values will be returned with updated naming. Remaining technologies + are deleted + + Example: + df_out = duplicatePlexosTechs(df_in, ['CCG', 'OCG']) + df_in['TECHNOLOGY'] = [PWRCCGAFGXX01, PWROCGAFGXX01, PWRHYDAFGXX01] + df_out['TECHNOLOGY'] = [PWRCCGAFGXX02, PWROCGAFGXX02] + """ + df_out = df_in.copy() + df_out = df_out.loc[(df_out['TECHNOLOGY'].str[3:6].isin(techs)) & + ~(df_out['TECHNOLOGY'].str.startswith('MIN'))] + df_out['TECHNOLOGY'] = df_out['TECHNOLOGY'].str.slice_replace(start=11, + stop=13, + repl='01') + return df_out + +def newIar(df_in, tech, new_iar_ccg, + new_iar_ocg, new_iar_coa, new_iar_default): + """Replaces the input activity ratio value with a hardcoded value + + Arguments: + df = dataframe with a 'TECHNOLOGY' and 'VALUE' column + tech = technology to replace iar for (CCG, HYD, SPV...) + + Returns: + df_out = same dataframe as df_in with a new values in 'VALUE' + + Example: + df_out = newIar(df_in, 'CCG') + df_out['TECHNOLOGY'] = [PWRCCGINDNP01, PWRCCGINDNW01] + df_out['VALUE'] = [2, 2] + """ + + df_out = df_in.loc[df_in['TECHNOLOGY'].str[3:6] == tech] + if tech == 'CCG': + iar = new_iar_ccg + elif tech == 'OCG': + iar = new_iar_ocg + elif tech == 'COA': + iar = new_iar_coa + else: + logging.warning(f'Default IAR used for new {tech} power plants') + iar = new_iar_default + df_out['VALUE'] = round(1/iar, 3) + return df_out \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/data_transmission.py b/workflow/scripts/osemosys_global/powerplant/data_transmission.py new file mode 100644 index 00000000..e801904a --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/data_transmission.py @@ -0,0 +1,48 @@ +"""Functions to extract and fromat relevent data for tranmissions""" + +def format_transmission_name(df): + '''Formats PLEXOS transmission names into OSeMOSYS Global names. + + Args: + :param df: Pandas DataFrame with columns 'From' and 'To' describing the + transmission from and to contries. ie. + + Returns: + :param df: Same as df_in, except the 'From' and 'To' columns are replaced + with a single 'TECHNOLOGY' column holding OSeMOSYS Global + naming conventions + + Example: + df = pd.DataFrame( + [[AF-COD, AF-COG, 0.001], + [EU-AUT, EU-SVK, 0.004], + [AS-LBN, AS-SYR, 0.006]], + columns = ['From', 'To', 'Losses'] + ) + pd.DataFrame( + [[0.001,TRNCODXXCOGXX], + [0.004,TRNAUTXXSVKXX], + [0.006,TRNLBNXXSYRXX]] + columns = ['Losses','TECHNOLOGY'])''' + + # If from column has length 6 then it's the last three chars plus XX + df.loc[df["From"].str.len() == 6, "From"] = (df["From"].str[3:6] + "XX") + + # If from column has length 9 then it's the 3:6 and 7:9 three chars plus XX + df.loc[df["From"].str.len() == 9, "From"] = ( + df["From"].str[3:6] + df["From"].str[7:9]) + + # If to column has length 6 then it's the last three chars plus XX + df.loc[df["To"].str.len() == 6, "To"] = (df["To"].str[3:6] + "XX") + + # If to column has length 9 then it's the 3:6 and 7:9 three chars plus XX + df.loc[df["To"].str.len() == 9, "To"] = ( + df["To"].str[3:6] + df["To"].str[7:9]) + + # Combine From and To columns. + df["TECHNOLOGY"] = ("TRN" + df["From"] + df["To"]) + + # Drop to and from columns + df = df.drop(["From", "To"], axis=1) + + return df \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/investment_constraints.py b/workflow/scripts/osemosys_global/powerplant/investment_constraints.py new file mode 100644 index 00000000..790bd2ba --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/investment_constraints.py @@ -0,0 +1,45 @@ +"""Function to calculate residual capacity for powerplant technologies.""" + +import pandas as pd + +from constants import ( + region_name, + years +) + +from utils import apply_dtypes + +def cap_investment_constraints(df_iar_final, no_investment_techs): + + # Create totalAnnualMaxCapacityInvestment data + + # Do not allow capacity investment for all PWRxxxxxxxx00 technolgoies + max_cap_invest_techs = list(set( + df_iar_final.loc[df_iar_final['TECHNOLOGY'].str.endswith('00')]['TECHNOLOGY'].tolist())) + max_cap_invest_data = [] + for tech in max_cap_invest_techs: + for year in years: + max_cap_invest_data.append([region_name, tech, year, 0]) + + # Do not allow investment for all xxxABCxxxxxxx technologies + + if not no_investment_techs: + no_investment_techs = [] # Change from None type to empty list + max_cap_invest_techs = list(set(df_iar_final.loc[ + df_iar_final['TECHNOLOGY'].str[3:6].isin(no_investment_techs)][ + 'TECHNOLOGY'].tolist())) + for tech in max_cap_invest_techs: + for year in years: + max_cap_invest_data.append([region_name, tech, year, 0]) + + # Save totalAnnualMaxCapacityInvestment + df_max_cap_invest = pd.DataFrame(max_cap_invest_data, + columns = ['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE'] + ) + df_max_cap_invest = apply_dtypes(df_max_cap_invest, "TotalAnnualMaxCapacityInvestment") + + df_min_cap_invest = pd.DataFrame(columns = ['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE'] + ) + df_min_cap_invest = apply_dtypes(df_min_cap_invest, "TotalAnnualMinCapacityInvestment") + + return df_max_cap_invest, df_min_cap_invest \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/main.py b/workflow/scripts/osemosys_global/powerplant/main.py new file mode 100644 index 00000000..295ecfce --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/main.py @@ -0,0 +1,274 @@ +"""Creates demand projections""" + +import pandas as pd +import os + +from read import( + import_plexos_2015, + import_weo_regions, + import_weo_costs, + import_op_life, + import_naming_convention_tech, + import_line_data, + import_afs, + import_custom_res_cap, +) + +from constants import( + file_plexos, + file_default_op_life, + file_naming_convention_tech, + file_weo_costs, + file_weo_regions, + file_line_data, + file_custom_res_cap, + file_default_av_factors, + avg_csp_eff, + avg_urn_eff, + duplicate_techs, + nodes_extra_list, + mode_list, + thermal_fuel_list_oar, + thermal_fuel_list_iar, + renewables_list, + thermal_fuel_list_mining, + costs_dict, + cross_border_trade, + no_investment_techs, + tech_capacity, + df_iar_custom_val, + df_oar_custom_val, + custom_nodes, + output_data_dir, + ) + +from data import( + set_generator_table, + average_efficiency, + ) + +from residual_capacity import res_capacity + +from activity import( + activity_master_start, + activity_output_pwr, + activity_input_pwr, + activity_upstream, + activity_master_end, + capact + ) + +from activity_transmission import( + activity_transmission, + activity_transmission_limit + ) + +from costs import( + costs_pwr, + costs_end + ) + +from costs_transmission import get_transmission_costs + +from operational_life import( + set_op_life, + set_op_life_transmission + ) + +from investment_constraints import cap_investment_constraints + +from sets import( + create_sets, + output_sets + ) + +from user_defined_capacity import set_user_defined_capacity + +from availability import availability_factor + +def main( + plexos_prop: pd.DataFrame, + plexos_memb: pd.DataFrame, + weo_costs: pd.DataFrame, + weo_regions: pd.DataFrame, + default_op_life: pd.DataFrame, + naming_convention_tech: pd.DataFrame, + line_data: pd.DataFrame, + interface_data: pd.DataFrame, + custom_res_cap: pd.DataFrame, + default_av_factors: pd.DataFrame, +): + + # CALL FUNCTIONS + + # return generator_table + gen_table = set_generator_table(plexos_prop, plexos_memb, + op_life_dict, tech_code_dict) + + # Calculate average technology efficiencies. + df_eff_node, df_eff_tech = average_efficiency(gen_table, avg_csp_eff, + avg_urn_eff) + + # Calculate residual capacity. + df_res_cap, custom_techs = res_capacity(gen_table, tech_list, tech_code, + custom_res_cap, duplicate_techs) + + df_ratios = activity_master_start(gen_table, nodes_extra_list, + duplicate_techs, mode_list) + + df_oar, df_oar_base = activity_output_pwr(df_ratios, thermal_fuel_list_oar) + + df_iar_base = activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, + df_eff_node, df_eff_tech) + + df_oar_upstream, df_oar_int = activity_upstream(df_iar_base, renewables_list, + thermal_fuel_list_mining) + + df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar = activity_transmission(df_oar_base, + plexos_prop, + interface_data) + + df_oar_final, df_iar_final = activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, + df_oar_trn, df_int_trn_oar, df_iar_base, + df_iar_trn, df_int_trn_iar, duplicate_techs) + + df_costs = costs_pwr(weo_costs, costs_dict) + + df_trans_capex, df_trans_fix = get_transmission_costs(trn_line, df_oar_final) + + df_cap_cost_final, df_fix_cost_final = costs_end(weo_regions, df_costs, df_oar_final, + df_trans_capex, df_trans_fix) + + df_capact_final = capact(df_oar_final) + + df_crossborder_final = activity_transmission_limit(cross_border_trade, df_oar_final) + + df_op_life_trn, df_op_life_out = set_op_life(tech_code_dict, df_iar_final, + df_oar_final, op_life_dict) + + df_op_life = set_op_life_transmission(df_op_life_trn, df_op_life_out, op_life_dict) + + df_max_cap_invest, df_min_cap_invest = cap_investment_constraints(df_iar_final, + no_investment_techs) + + # Creates sets for TECHNOLOGIES and FUELS. + if custom_nodes: + tech_set = create_sets('TECHNOLOGY', df_oar_final, output_data_dir, custom_techs) + else: + tech_set = create_sets('TECHNOLOGY', df_oar_final, output_data_dir, []) + + fuel_set = create_sets('FUEL', df_oar_final, output_data_dir, []) + + # Create sets for YEAR, MODE_OF_OPERATION and REGION + years_set, mode_list_set, regions_set = output_sets(mode_list) + + if not tech_capacity is None: + (tech_set, + fuel_set, + df_max_cap_invest, + df_min_cap_invest, + df_res_cap, + df_iar_final, + df_oar_final, + df_op_life, + df_capact_final, + df_cap_cost_final + ) = set_user_defined_capacity( + tech_capacity, + op_life_dict, + tech_set, + df_min_cap_invest, + df_max_cap_invest, + df_res_cap, + df_iar_final, + df_oar_final, + fuel_set, + df_op_life, + df_capact_final, + df_cap_cost_final, + df_iar_custom_val, + df_oar_custom_val + ) + + df_af_final = availability_factor(availability, tech_set) + + # OUTPUT CSV's + + df_res_cap.to_csv(os.path.join(output_data_dir, "ResidualCapacity.csv"), index=None) + + df_oar_final.to_csv(os.path.join(output_data_dir, "OutputActivityRatio.csv"), index=None) + + df_iar_final.to_csv(os.path.join(output_data_dir, "InputActivityRatio.csv"), index=None) + + df_cap_cost_final.to_csv(os.path.join(output_data_dir, "CapitalCost.csv"), index = None) + + df_fix_cost_final.to_csv(os.path.join(output_data_dir, "FixedCost.csv"), index = None) + + df_capact_final.to_csv(os.path.join(output_data_dir, "CapacityToActivityUnit.csv"), index = None) + + df_crossborder_final.to_csv(os.path.join(output_data_dir, + "TotalTechnologyModelPeriodActivityUpperLimit.csv"), + index = None) + + df_op_life.to_csv(os.path.join(output_data_dir, "OperationalLife.csv"), index = None) + + df_max_cap_invest.to_csv(os.path.join(output_data_dir, + 'TotalAnnualMaxCapacityInvestment.csv'), + index = None) + + df_min_cap_invest.to_csv(os.path.join(output_data_dir, + 'TotalAnnualMinCapacityInvestment.csv'), + index = None) + + df_af_final.to_csv(os.path.join(output_data_dir, 'AvailabilityFactor.csv'), index=None) + + tech_set.to_csv(os.path.join(output_data_dir, "TECHNOLOGY.csv"), index = None) + + fuel_set.to_csv(os.path.join(output_data_dir, "FUEL.csv"), index = None) + + years_set.to_csv(os.path.join(output_data_dir, "YEAR.csv"), index = None) + + mode_list_set.to_csv(os.path.join(output_data_dir, "MODE_OF_OPERATION.csv"), index = None) + + regions_set.to_csv(os.path.join(output_data_dir, "REGION.csv"), index = None) + +if __name__ == "__main__": + + plexos_prop = import_plexos_2015(file_plexos, "prop") + plexos_memb = import_plexos_2015(file_plexos, "memb") + op_life = import_op_life(file_default_op_life) + + op_life_dict = dict(zip(list(op_life['tech']), + list(op_life['years']))) + + tech_code = import_naming_convention_tech(file_naming_convention_tech) + + tech_list = list(tech_code['code']) + + tech_code_dict = dict(zip(list(tech_code['tech']), + list(tech_code['code']))) + + trn_line = import_line_data(file_line_data, "Lines") + trn_interface = import_line_data(file_line_data, "Interface") + + weo_costs = import_weo_costs(file_weo_costs) + weo_regions = import_weo_regions(file_weo_regions) + + custom_res_cap = import_custom_res_cap(file_custom_res_cap) + + availability = import_afs(file_default_av_factors) + + input_data = { + "plexos_prop": plexos_prop, + "plexos_memb": plexos_memb, + "weo_costs": weo_costs, + "weo_regions": weo_regions, + "default_op_life": op_life, + "naming_convention_tech": tech_code, + "line_data": trn_line, + "interface_data": trn_interface, + "custom_res_cap" : custom_res_cap, + "default_av_factors": availability, + } + + main(**input_data) \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/operational_life.py b/workflow/scripts/osemosys_global/powerplant/operational_life.py new file mode 100644 index 00000000..d2075007 --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/operational_life.py @@ -0,0 +1,41 @@ +"""Function to calculate residual capacity for powerplant technologies.""" + +import pandas as pd + +from constants import ( + region_name, +) + +from utils import apply_dtypes + +def set_op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict): + + # Create Operational Life data + tech_code_dict_reverse = dict((v,k) for k,v in tech_code_dict.items()) + op_life_techs = list(set(list(df_iar_final['TECHNOLOGY'].unique()) + list(df_oar_final['TECHNOLOGY'].unique()))) + op_life_pwr = [tech for tech in op_life_techs if (tech[0:3] == 'PWR') and (len(tech) == 13)] + op_life_trn = [tech for tech in op_life_techs if tech[0:3] == 'TRN'] + + op_life_out = [] + + # power generation technologies + for op_life_tech in op_life_pwr: + op_life_tech_name = tech_code_dict_reverse[op_life_tech[3:6]] + op_life_out.append([ + region_name, + op_life_tech, + op_life_dict[op_life_tech_name]]) + + return op_life_trn, op_life_out + +def set_op_life_transmission(op_life_trn, op_life_out, op_life_dict): + + # transmission technologies + for op_life_tech in op_life_trn: + op_life_out.append([region_name, op_life_tech, op_life_dict.get('TRN')]) + + op_life_base = pd.DataFrame(op_life_out, columns = ['REGION', 'TECHNOLOGY', 'VALUE']) + + op_life_base = apply_dtypes(op_life_base, "OperationalLife") + + return op_life_base \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/read.py b/workflow/scripts/osemosys_global/powerplant/read.py new file mode 100644 index 00000000..deea5542 --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/read.py @@ -0,0 +1,89 @@ +"""Module for reading in data sources""" + +import pandas as pd + +from constants import custom_nodes + +def import_plexos_2015(f: str, metric: str) -> dict[str, pd.DataFrame]: + """Imports PLEXOS-World 2015 model file. + + PLEXOS_World_2015_Gold_V1.1.xlsx + """ + if metric.lower() == "memb": + sheet_name = "Memberships" + elif metric.lower() == "prop": + sheet_name = "Properties" + else: + raise NotImplementedError + + return pd.read_excel(f, sheet_name=sheet_name) + +def import_weo_regions(f: str) -> pd.DataFrame: + """Imports WEO to OG region mapping. + + weo_region_mapping.csv + """ + return pd.read_csv(f) + +def import_weo_costs(f: str) -> pd.DataFrame: + """Imports WEO cost data for powerplant technologies. + + weo_2020_powerplant_costs.csv + """ + return pd.read_csv(f) + +def import_op_life(f: str) -> pd.DataFrame: + """Imports default operational life data. + + operational_life.csv + """ + return pd.read_csv(f) + +def import_naming_convention_tech(f: str) -> pd.DataFrame: + """Imports naming convention for powerplant technologies. + + naming_convention_tech.csv + """ + return pd.read_csv(f) + +def import_line_data(f: str, metric: str) -> dict[str, pd.DataFrame]: + """Imports transmission data from PLEXOS-World. + + Costs Line expansion.xlsx + """ + if metric.lower() == "interface": + sheet_name = "Interface" + elif metric.lower() == "lines": + sheet_name = "Lines" + else: + raise NotImplementedError + + return pd.read_excel(f, sheet_name=sheet_name) + +def import_afs(f: str) -> pd.DataFrame: + """Imports default availability factors. + + availability_factors.csv + """ + return pd.read_csv(f) + +def import_custom_res_cap(f: str) -> pd.DataFrame: + """Imports residual capacities for custom nodes. + + custom_nodes\residual_capacity.csv + """ + return pd.read_csv(f) + +def import_tech_set(f: str) -> pd.DataFrame: + """Imports list of technologies. + + TECHNOLOGY.csv + """ + return pd.read_csv(f) + +def import_fuel_set(f: str) -> pd.DataFrame: + """Imports list of fuels. + + FUEL.csv + """ + return pd.read_csv(f) \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/residual_capacity.py b/workflow/scripts/osemosys_global/powerplant/residual_capacity.py new file mode 100644 index 00000000..686df101 --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/residual_capacity.py @@ -0,0 +1,85 @@ +"""Function to calculate residual capacity for powerplant technologies.""" + +import pandas as pd + +from constants import ( + start_year, + end_year, + region_name, + custom_nodes, +) + +from data import( + createPwrTechs, + custom_nodes_csv + ) + + +def res_capacity(df_gen_base, tech_list, df_tech_code, df_custom_res_cap, duplicate_techs): + + # ### Calculate residual capacity + res_cap_cols = [ + "node_code", + "tech_code", + "total_capacity", + "start_year", + "retirement_year_model", + ] + + df_res_cap = df_gen_base[res_cap_cols] + + for each_year in range(start_year, end_year+1): + df_res_cap[str(each_year)] = 0 + + df_res_cap = pd.melt( + df_res_cap, + id_vars=res_cap_cols, + value_vars=[x for x in df_res_cap.columns if x not in res_cap_cols], + var_name="model_year", + value_name="value", + ) + df_res_cap["model_year"] = df_res_cap["model_year"].astype(int) + df_res_cap.loc[ + (df_res_cap["model_year"] >= df_res_cap["start_year"]) + & (df_res_cap["model_year"] <= df_res_cap["retirement_year_model"]), + "value", + ] = df_res_cap["total_capacity"] + + df_res_cap = df_res_cap.groupby( + ["node_code", "tech_code", "model_year"], as_index=False + )["value"].sum() + + # Add column with naming convention + df_res_cap = createPwrTechs(df_res_cap, duplicate_techs) + + # Convert total capacity from MW to GW + df_res_cap['value'] = df_res_cap['value'].div(1000) + + # Rename 'model_year' to 'year' and 'total_capacity' to 'value' + df_res_cap.rename({'model_year': 'YEAR', + 'value': 'VALUE'}, + inplace=True, + axis=1) + # Drop 'tech_code' and 'node_code' + df_res_cap.drop(['tech_code', 'node_code'], + inplace=True, + axis=1) + + # Add 'REGION' column and fill 'GLOBAL' throughout + df_res_cap['REGION'] = region_name + + # Reorder columns + df_res_cap = df_res_cap[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] + + if custom_nodes: + df_res_cap_custom, custom_techs = custom_nodes_csv(df_custom_res_cap, + tech_list) + df_res_cap = pd.concat([df_res_cap, df_res_cap_custom]) + + df_res_cap.drop_duplicates(subset=['REGION','TECHNOLOGY','YEAR'], + keep='last', + inplace=True) + df_res_cap = df_res_cap.loc[(df_res_cap['TECHNOLOGY'].str.startswith('PWR')) & + (~df_res_cap['TECHNOLOGY'].str.endswith('00'))] + + return df_res_cap, custom_techs if custom_nodes else df_res_cap \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/sets.py b/workflow/scripts/osemosys_global/powerplant/sets.py new file mode 100644 index 00000000..1b7729ec --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/sets.py @@ -0,0 +1,42 @@ +"""Function to create sets.""" + +import pandas as pd + +from constants import ( + region_name, + years, + SET_DTYPES +) + +def create_sets(x, df, output_dir, custom_node_elements): + """Creates a formatted otoole set csv + + Arguments: + x = Name of set given by otoole as a string + df = dataframe in extract set information from + output_dir = directory to write file to + + Returns: + None. Writes out csv file + + Example: + create_sets('TECHNOLOGY', df_oar_final, output_dir) + """ + set_elements = list(df[x].unique()) + list(df[x].unique()) + list(custom_node_elements) + set_elements = list(set(set_elements)) + set_elements = [x for x in set_elements if x != 'nan'] + set_elements.sort() + set_elements_df = pd.DataFrame(set_elements, columns = ['VALUE']) + return set_elements_df + +def output_sets(mode_list): + + # ## Create set for YEAR, REGION, MODE_OF_OPERATION + years_df = pd.DataFrame(years, columns = ['VALUE']).astype(SET_DTYPES["YEAR"]) + + mode_list_df = pd.DataFrame(mode_list, columns = ['VALUE']).astype(SET_DTYPES["MODE_OF_OPERATION"]) + + regions_df = pd.DataFrame(columns = ['VALUE']).astype(SET_DTYPES["REGION"]) + regions_df.loc[0] = region_name + + return years_df, mode_list_df, regions_df \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py new file mode 100644 index 00000000..beefa622 --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py @@ -0,0 +1,271 @@ +"""Function to integrate user defined capacities.""" + +import pandas as pd +import itertools + +from constants import ( + region_name, + years +) + +def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, + df_min_cap_invest, df_max_cap_invest, df_res_cap, + df_iar_final, df_oar_final, fuel_set, op_life_base, cap_act_base, + cap_cost_base, df_iar_custom_val, df_oar_custom_val): + """User-defined capacities are used when a specific technology must be + invested, for a given year and capacity. This is applied through the + parameter 'TotalAnnualMinCapacityInvestment'. + + Args: + region: From config file (e.g. 'GLOBAL') + tech_capacity: User-defined capacity in config file + (e.g. TRNAGOXXCODXX: [5, 2030]) + + Returns + None + """ + techCapacity = [] + tech_capacity_dict = {} + first_year_dict = {} + build_rate_dict = {} + capex_dict = {} + build_year_dict = {} + + if not tech_capacity is None: + + for tech, tech_params in tech_capacity.items(): + techCapacity.append([tech, tech_params[0], tech_params[1]]) + tech_capacity_dict[tech] = tech_params[2] + build_year_dict[tech] = tech_params[1] + first_year_dict[tech] = tech_params[3] + build_rate_dict[tech] = tech_params[4] + capex_dict[tech] = tech_params[5] # + tech_capacity_df = pd.DataFrame(techCapacity, + columns=['TECHNOLOGY', 'VALUE', 'YEAR']) + tech_capacity_df['REGION'] = region_name + tech_capacity_df = tech_capacity_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] + + for each_tech in list(tech_capacity_df['TECHNOLOGY'].unique()): + if each_tech not in list(df_tech_set['VALUE']): + df_tech_set = pd.concat([df_tech_set, pd.DataFrame({'VALUE':[each_tech]})]) + + df_tech_set.drop_duplicates(inplace=True) + + df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_df]) + df_min_cap_inv.drop_duplicates(inplace=True) + + df = pd.DataFrame(list(itertools.product(list(tech_capacity_df['TECHNOLOGY'].unique()), + years) + ), + columns = ['TECHNOLOGY', + 'YEAR'] + ) + df['REGION'] = region_name + df = pd.merge(df, df_min_cap_inv, + how='left', + on=['REGION', 'TECHNOLOGY', 'YEAR']) + df['FIRST_YEAR'] = df['TECHNOLOGY'].map(first_year_dict) + df['BUILD_YEAR'] = df['TECHNOLOGY'].map(build_year_dict) + df['MAX_BUILD'] = df['TECHNOLOGY'].map(build_rate_dict) + + # Fill VALUE with MAX_BUILD for YEAR >= FIRST_YEAR + df.loc[(df['YEAR']>=df['FIRST_YEAR']) & + (df['YEAR']>df['BUILD_YEAR']), + 'VALUE'] = df['MAX_BUILD'] + df.fillna(0, + inplace=True) + max_cap_techs_df = df[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + + #Replace VALUE with CAPEX + df['VALUE'] = df['TECHNOLOGY'].map(capex_dict) + + # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD for TRN + df_max_cap_inv = pd.concat([df_max_cap_invest, max_cap_techs_df]).drop_duplicates() + + # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD for TRN + df_max_cap_inv.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'YEAR'], + keep='last', + inplace=True) + + # For technologies with start year before model start year, add to ResidualCapacity + df_res_cap_ud = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] < min(years)] + df_res_cap_ud.rename(columns={'YEAR':'START_YEAR'}, + inplace=True) + df_res_cap_ud_final = pd.DataFrame(list(itertools.product(df_res_cap_ud['TECHNOLOGY'].unique(), + years) + ), + columns = ['TECHNOLOGY', + 'YEAR'] + ) + df_res_cap_ud_final = pd.merge(df_res_cap_ud_final, + df_res_cap_ud, + how='left', + on=['TECHNOLOGY']) + df_res_cap_ud_final['TECH'] = df_res_cap_ud_final['TECHNOLOGY'].str[3:6] + df_res_cap_ud_final.loc[df_res_cap_ud_final['TECHNOLOGY'].str.contains('TRN'), + 'TECH'] = 'TRN' + df_res_cap_ud_final['OP_LIFE'] = df_res_cap_ud_final['TECH'].map(op_life_dict) + df_res_cap_ud_final['END_YEAR'] = (df_res_cap_ud_final['OP_LIFE'] + + df_res_cap_ud_final['START_YEAR']) + df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] + >= df_res_cap_ud_final['START_YEAR']] + df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] + <= df_res_cap_ud_final['END_YEAR']] + df_res_cap_ud_final['REGION'] = region_name + df_res_cap_ud_final = df_res_cap_ud_final[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + + df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final]) + + # For technologies with start year at or after model start year, add to + # TotalAnnualMinCapacityInvestment + df_min_cap_inv = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] >= min(years)] + df_min_cap_inv.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'YEAR'], + keep='last', + inplace=True) + + # Add IAR and OAR for custom technologies + tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) + df_iar_custom = pd.DataFrame(list(itertools.product(tech_list, + [1, 2], + years) + ), + columns = ['TECHNOLOGY', + 'MODE_OF_OPERATION', + 'YEAR'] + ) + df_oar_custom = pd.DataFrame(list(itertools.product(tech_list, + [1, 2], + years) + ), + columns = ['TECHNOLOGY', + 'MODE_OF_OPERATION', + 'YEAR'] + ) + # IAR in modes 1 and 2 are primary electricity commodity ('ELC*01') in + # node_from and node_to, respectively. + # OAR is the inverse of the above + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, + 'FUEL'] = ('ELC' + + df_iar_custom['TECHNOLOGY'].str[3:8] + + '01') + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, + 'FUEL'] = ('ELC' + + df_iar_custom['TECHNOLOGY'].str[8:13] + + '01') + df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, + 'FUEL'] = ('ELC' + + df_oar_custom['TECHNOLOGY'].str[8:13] + + '01') + df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, + 'FUEL'] = ('ELC' + + df_oar_custom['TECHNOLOGY'].str[3:8] + + '01') + df_iar_custom['VALUE'] = df_iar_custom_val + df_oar_custom['VALUE'] = df_oar_custom_val + df_iar_custom['REGION'] = region_name + df_oar_custom['REGION'] = region_name + + df_iar_custom = df_iar_custom[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + df_oar_custom = df_oar_custom[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + + df_iar = pd.concat([df_iar_final, df_iar_custom]) + df_oar = pd.concat([df_oar_final, df_oar_custom]) + + df_iar.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', + inplace=True) + + df_oar.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', + inplace=True) + + # Add new fuels to FUEL set, if not already present + fuel_list = [] + fuel_list = list(df_iar_custom['FUEL'].unique()) + list(df_oar_custom['FUEL'].unique()) + fuel_list = list(set(fuel_list)) + + for each_fuel in fuel_list: + if each_fuel not in list(fuel_set['VALUE']): + fuel_set = pd.concat([fuel_set, pd.DataFrame({'VALUE':[each_fuel]})]) + + op_life_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) + + op_life_custom.loc[op_life_custom['TECHNOLOGY'].str.contains('TRN'), + 'VALUE'] = op_life_dict.get('TRN') + op_life_custom['REGION'] = region_name + op_life_custom = op_life_custom[['REGION', + 'TECHNOLOGY', + 'VALUE']] + + op_life = pd.concat([op_life_base, op_life_custom]) + op_life.drop_duplicates(subset=['REGION', + 'TECHNOLOGY'], + keep='last', + inplace=True) + + cap_act_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) + cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('TRN'), + 'VALUE'] = 31.536 + cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('PWR'), + 'VALUE'] = 31.536 + cap_act_custom['REGION'] = region_name + cap_act_custom = cap_act_custom[['REGION', + 'TECHNOLOGY', + 'VALUE']] + cap_act = pd.concat([cap_act_base, cap_act_custom]) + cap_act.drop_duplicates(inplace=True) + + tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) + cap_cost_trn = pd.DataFrame(list(itertools.product(tech_list, + years)), + columns = ['TECHNOLOGY', + 'YEAR']) + + # Update CapitalCost with user-defined costs by transmission line + for each_trn in tech_list: + cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.startswith(each_trn), + 'VALUE'] = capex_dict[each_trn] + + cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.contains('PWRTRN'), + 'VALUE'] = 300 + cap_cost_trn['REGION'] = region_name + cap_cost_trn = cap_cost_trn[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + cap_cost = pd.concat([cap_cost_base, cap_cost_trn]) + cap_cost.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], + keep="last", + inplace=True) + + return(df_tech_set, fuel_set, df_max_cap_inv, df_min_cap_inv, + df_res_cap, df_iar, df_oar, op_life, cap_act, cap_cost) + \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/utils.py b/workflow/scripts/osemosys_global/powerplant/utils.py new file mode 100644 index 00000000..336101da --- /dev/null +++ b/workflow/scripts/osemosys_global/powerplant/utils.py @@ -0,0 +1,21 @@ +"""Utility Functions""" + +import pandas as pd +from typing import Optional +from constants import SET_DTYPES + +import logging +logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) + +def apply_dtypes(df:pd.DataFrame, name: Optional[str]) -> pd.DataFrame: + """Sets datatypes on dataframe""" + + for col in df.columns: + try: + df[col] = df[col].astype(SET_DTYPES[col]) + except KeyError: + if name: + logging.info(f"Can not set dtype for {name} on {col}") + else: + logging.info(f"Can not set dtype on {col}") + return df diff --git a/workflow/scripts/osemosys_global/powerplant_data.py b/workflow/scripts/osemosys_global/powerplant_data.py deleted file mode 100644 index 57895a93..00000000 --- a/workflow/scripts/osemosys_global/powerplant_data.py +++ /dev/null @@ -1,1990 +0,0 @@ -# Import modules -import warnings -warnings.simplefilter(action='ignore', category=FutureWarning) -import pandas as pd -from datetime import datetime -pd.options.mode.chained_assignment = None -import numpy as np -import itertools -import os -from configuration import ConfigFile, ConfigPaths -from constants import SET_DTYPES -from utils import apply_dtypes -import logging -logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) -from scipy import spatial - -def main(): - - """Main retrieves the script input data, sets user configurations - and calls the different functions as defined within the script.""" - - # CONFIGURATION PARAMETERS - - global config, region_name, input_data_dir, output_data_dir - - config_paths = ConfigPaths() - config = ConfigFile('config') - region_name = config.region_name - - input_data_dir = config_paths.input_data_dir - output_data_dir = config_paths.output_data_dir - - custom_nodes_dir = config_paths.custom_nodes_dir - - # Check for custom nodes directory - try: - os.makedirs(custom_nodes_dir) - except FileExistsError: - pass - - global model_start_year, model_end_year, years, custom_nodes - - tech_capacity = config.get('user_defined_capacity') - cross_border_trade = config.get('crossborderTrade') - model_start_year = config.get('startYear') - model_end_year = config.get('endYear') - years = list(range(model_start_year, model_end_year + 1)) - custom_nodes = config.get('nodes_to_add') - no_investment_techs = config.get('no_invest_technologies') - - # Create output directory - if not os.path.exists(output_data_dir): - os.makedirs(output_data_dir) - - # INPUT FILES - - df_pw_prop = pd.read_excel(os.path.join(input_data_dir, - "PLEXOS_World_2015_Gold_V1.1.xlsx"), - sheet_name = "Properties") - - df_pw_memb = pd.read_excel(os.path.join(input_data_dir, - "PLEXOS_World_2015_Gold_V1.1.xlsx"), - sheet_name = "Memberships") - - df_dict = df_pw_memb[df_pw_memb["parent_class"] == "Generator"].rename( - {"parent_object": "powerplant"}, axis=1 - ) - - df_weo_data = pd.read_csv(os.path.join(input_data_dir, - "weo_2020_powerplant_costs.csv") - ) - df_op_life = pd.read_csv(os.path.join(input_data_dir, - "operational_life.csv") - ) - - op_life_dict = dict(zip(list(df_op_life['tech']), - list(df_op_life['years']))) - - df_tech_code = pd.read_csv(os.path.join(input_data_dir, - "naming_convention_tech.csv") - ) - - tech_list = list(df_tech_code['code']) - - tech_code_dict = dict(zip(list(df_tech_code['tech']), - list(df_tech_code['code']))) - - df_trn_efficiencies = pd.read_excel(os.path.join(input_data_dir, - "Costs Line expansion.xlsx"), - sheet_name = 'Interface' - ) - - df_trn_lines = pd.read_excel(os.path.join(input_data_dir, - "Costs Line expansion.xlsx"), - sheet_name = 'Lines') - - df_weo_regions = pd.read_csv(os.path.join(input_data_dir, - "weo_region_mapping.csv") - ) - - df_tech_set = os.path.join(output_data_dir, 'TECHNOLOGY.csv') - - df_af = pd.read_csv(os.path.join(input_data_dir, - "availability_factors.csv") - ) - - fuel_set = os.path.join(output_data_dir, 'FUEL.csv') - - if custom_nodes: - df_custom_res_cap = pd.read_csv(os.path.join(input_data_dir, - "custom_nodes", - "residual_capacity.csv") - ) - - else: - - df_custom_res_cap = pd.DataFrame() - - # USER INPUTS - - """Change IAR for CSP value taken from PLEXOS to 1.0. - Relevant for the 'average_efficiency' function.""" - avg_csp_eff = 1 - - """Change IAR for URN value taken from PLEXOS to 2.2 (45%). - Relevant for the 'average_efficiency' function.""" - avg_urn_eff = 0.45 - - """Technologies that will have 00 and 01 suffixes to represent PLEXOS - historical values and future values. Relevant for the 'residual_capacity' - and activity functions.""" - duplicate_techs = ['CCG', 'OCG'] - - """Add extra nodes which exist in 2050 but are not in the 2015 PLEXOS-World - data to enable their addition to the workflow. Relevant for the - 'generator_table' and activity functions""" - nodes_extra_list = ['AF-SOM', - 'AF-TCD', - 'AS-TLS', - 'EU-MLT', - 'NA-BLZ', - 'NA-HTI', - 'SA-BRA-J1', - 'SA-BRA-J2', - 'SA-BRA-J3', - 'SA-SUR'] - - """Sets the mode of operations for technologies. - Relevant for the activity_master_start function.""" - mode_list = [1,2] - - """List of technologies (thermal) for which output activity ratios need - to be developed. Relevant for the 'activity_output_pwr' function.""" - thermal_fuel_list_oar = ['COA', - 'COG', - 'OCG', - 'CCG', - 'PET', - 'URN', - 'OIL', - 'OTH', - 'CCS' - ] - - """List of technologies (thermal) for which input activity ratios need - to be developed. Relevant for the 'activity_input_pwr' function.""" - thermal_fuel_list_iar = ['COA', - 'COG', - 'PET', - 'URN', - 'OIL', - 'OTH' - ] - - """List of upstream fuels for which output activity ratios need - to be developed. Relevant for the 'activity_upstream' function.""" - thermal_fuels_list_mining = ['COA', - 'COG', - 'GAS', - 'PET', - 'URN', - 'OIL', - 'OTH' - ] - - """List of technologies (renewable) for which activity ratios need - to be developed. Relevant for the 'activity_input_pwr' and - 'activity_upstream' functions.""" - renewables_list = ['BIO', - 'GEO', - 'HYD', - 'SPV', - 'CSP', - 'WAS', - 'WAV', - 'WON', - 'WOF'] - - """Technology input for the mapping of WEO cost data to OG technologies. - Relevant for the 'costs_pwr' function.""" - costs_dict = {'Biomass - waste incineration - CHP':'WAS', - 'Biomass Power plant':'BIO', - 'CCGT':'CCG', - 'CCGT - CHP':'COG', - 'Concentrating solar power':'CSP', - 'Gas turbine':'OCG', - 'Geothermal':'GEO', - 'Hydropower - large-scale':'HYD', - 'Marine':'WAV', - 'Nuclear':'URN', - 'Solar photovoltaics - Large scale':'SPV', - 'Steam Coal - SUBCRITICAL':'COA', - 'Steam Coal - SUPERCRITICAL':'COA', - 'Steam Coal - ULTRASUPERCRITICAL':'COA', - 'Wind onshore':'WON', - 'Wind offshore':'WOF', - 'Petroleum':'PET', - 'Oil':'OIL', - 'Other':'OTH', - 'IGCC + CCS':'CCS', - 'Coal* + CCS':'CCS',} - - """setings for custom iar values deviating from derived values from the - PLEXOS-World dataset. Relevant for the 'newIar' function.""" - global new_iar_ccg, new_iar_ocg, new_iar_coa, new_iar_default - - new_iar_ccg = 0.5 - new_iar_ocg = 0.35 - new_iar_coa = 0.33 - new_iar_default = 1 - - """Set iar and oar values for custom transmission entries. I.e. oar of 0.9 - assumes 10% losses. Relevant for the user_defined_capacity function.""" - df_iar_custom_val = 1 - df_oar_custom_val = 0.9 - - # CALL FUNCTIONS - - gen_table = generator_table(df_pw_prop, df_dict, tech_code_dict, - op_life_dict, nodes_extra_list) - - df_eff_node, df_eff_tech = average_efficiency(gen_table, avg_csp_eff, - avg_urn_eff) - - df_res_cap, custom_techs = residual_capacity(gen_table, tech_list, df_tech_code, - df_custom_res_cap, duplicate_techs) - - gem_cap(gen_table, tech_code_dict, op_life_dict) - - df_ratios = activity_master_start(gen_table, nodes_extra_list, - duplicate_techs, mode_list) - - df_oar, df_oar_base = activity_output_pwr(df_ratios, thermal_fuel_list_oar) - - df_iar_base = activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, - df_eff_node, df_eff_tech) - - df_oar_upstream, df_oar_int = activity_upstream(df_iar_base, renewables_list, - thermal_fuels_list_mining) - - df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar = activity_transmission(df_oar_base, - df_pw_prop, - df_trn_efficiencies) - - df_oar_final, df_iar_final = activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, - df_oar_trn, df_int_trn_oar, df_iar_base, - df_iar_trn, df_int_trn_iar, duplicate_techs) - - df_costs = costs_pwr(df_weo_data, costs_dict) - - df_trans_capex, df_trans_fix = get_transmission_costs(df_trn_lines, df_oar_final) - - cap_cost_base, fix_cost_base = costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, df_trans_fix) - - cap_act_base = capact(df_oar_final) - - activity_transmission_limit(cross_border_trade, df_oar_final) - - op_life_trn, op_life_out = op_life(tech_code_dict, df_iar_final, - df_oar_final, op_life_dict) - - op_life_base = op_life_transmission(op_life_trn, op_life_out, op_life_dict) - - df_max_cap_invest, df_min_cap_invest = cap_investment_constraints(df_iar_final, no_investment_techs) - - output_sets(df_oar_final, custom_techs, mode_list) - - tech_set_base = user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, - df_min_cap_invest, df_max_cap_invest, df_res_cap, - df_iar_final, df_oar_final, fuel_set, op_life_base, cap_act_base, - cap_cost_base, df_iar_custom_val, df_oar_custom_val) - - availability_factor(df_af, tech_set_base) - -def generator_table(df_pw_prop, df_dict, tech_code_dict, op_life_dict, nodes_extra_list): - - # Create main generator table - gen_cols_1 = ["child_class", "child_object", "property", "value"] - df_gen = df_pw_prop[gen_cols_1] - df_gen = df_gen[df_gen["child_class"] == "Generator"] - df_gen.rename(columns={"child_object": "powerplant"}, inplace=True) - df_gen.drop("child_class", axis=1, inplace=True) - df_gen = pd.pivot_table(df_gen, - index="powerplant", - columns="property", - values="value", - aggfunc=np.sum, - fill_value=0, - ) - df_gen["total_capacity"] = (df_gen["Max Capacity"].astype(float)) * ( - df_gen["Units"].astype(int) - ) - - gen_cols_base = ["Commission Date", "Heat Rate", "Max Capacity", "total_capacity"] - df_gen_base = df_gen[gen_cols_base] - - ## Compile dataframe with powerplants, nodes, and fuels - df_dict_fuel = df_dict[df_dict["collection"] == "Fuels"] - df_dict_fuel = df_dict_fuel[["powerplant", "child_object"]] - df_dict_nodes = df_dict[df_dict["collection"] == "Nodes"] - df_dict_nodes = df_dict_nodes[["powerplant", "child_object"]] - df_dict_2 = pd.merge(df_dict_fuel, df_dict_nodes, how="outer", on="powerplant") - - - ## Merge original generator dataframe with nodes and fuels - df_gen_base = pd.merge(df_gen_base, df_dict_2, how="outer", on="powerplant") - df_gen_base.rename( - {"child_object_x": "fuel", "child_object_y": "node"}, axis=1, inplace=True - ) - - ## Extract start year from Commission Date - df_gen_base["Commission Date"] = (pd.TimedeltaIndex(df_gen_base["Commission Date"].astype(int), - unit='d') + - datetime(1900, 1, 1)) - df_gen_base["start_year"] = df_gen_base["Commission Date"].dt.year - df_gen_base.drop("Commission Date", axis=1, inplace=True) - - ## Calculate efficiency from heat rate. Units of heat rate in MJ/kWh - df_gen_base["efficiency"] = 3.6 / df_gen_base["Heat Rate"].astype(float) - df_gen_base.drop("Heat Rate", axis=1, inplace=True) - - ## Calcluate years of operation from start year - df_gen_base["years_of_operation"] = model_start_year - df_gen_base["start_year"] - - ## Fix blank spaces in 'fuels' columns. Appearing for 'Oil' powerplants in certain countries - df_gen_base.loc[df_gen_base["fuel"].isna(), "fuel"] = ( - df_gen_base["node"].str.split("-").str[:2].str.join("-") - + " " - + df_gen_base["powerplant"].str.split("_", expand=True)[1] - ) - - ## Create column for technology - df_gen_base["technology"] = df_gen_base["powerplant"].str.split("_").str[1] - df_gen_base["technology"] = df_gen_base["technology"].str.title() - - ## Divide Gas into CCGT and OCGT based on max capacity - df_gen_base.loc[ - (df_gen_base["technology"] == "Gas") & (df_gen_base["Max Capacity"].astype(float) > 130), - "technology", - ] = "Gas-CCGT" - df_gen_base.loc[ - (df_gen_base["technology"] == "Gas") & (df_gen_base["Max Capacity"].astype(float) <= 130), - "technology", - ] = "Gas-OCGT" - - # Create table with aggregated capacity - df_gen_agg_node = df_gen_base[df_gen_base['start_year']<=model_start_year] - df_gen_agg_node = df_gen_agg_node.groupby(['node', 'technology'], - as_index=False)['total_capacity'].sum() - df_gen_agg_node = df_gen_agg_node.pivot(index='node', - columns='technology', - values='total_capacity').fillna(0).reset_index() - - df_gen_agg_node.drop('Sto', axis=1, inplace=True) # Drop 'Sto' technology. Only for USA. - - nodes_extra_df = pd.DataFrame(columns=['node']) - - nodes_extra_df['node'] = nodes_extra_list - - df_gen_agg_node = pd.concat( - [df_gen_agg_node,nodes_extra_df], - ignore_index=True, - sort=False, - ).fillna(0).sort_values(by='node').set_index('node').round(2) - - # Add region and country code columns - df_gen_base['region_code'] = df_gen_base['node'].str[:2] - df_gen_base['country_code'] = df_gen_base['node'].str[3:] - - df_gen_base['operational_life'] = df_gen_base['technology'].map(op_life_dict) - df_gen_base['retirement_year_data'] = (df_gen_base['operational_life'] - + df_gen_base['start_year']) - df_gen_base['retirement_diff'] = ((df_gen_base['years_of_operation'] - - df_gen_base['operational_life'])/ - df_gen_base['operational_life']) - - ''' Set retirement year based on years of operation. - If (years of operation - operational life) is more than 50% of - operational life, set retirement year - ''' - df_gen_base.loc[df_gen_base['retirement_diff'] >= 0.5, - 'retirement_year_model'] = 2028 - df_gen_base.loc[(df_gen_base['retirement_diff'] < 0.5) & - (df_gen_base['retirement_diff'] > 0), - 'retirement_year_model'] = 2033 - df_gen_base.loc[df_gen_base['retirement_diff'] <= 0, - 'retirement_year_model'] = df_gen_base['retirement_year_data'] - - df_gen_base['tech_code'] = df_gen_base['technology'].map(tech_code_dict) - - df_gen_base.loc[df_gen_base['node'].str.len() <= 6, - 'node_code'] = (df_gen_base['node']. - str.split('-'). - str[1:]. - str.join("") + - 'XX') - df_gen_base.loc[df_gen_base['node'].str.len() > 6, - 'node_code'] = (df_gen_base['node']. - str.split('-'). - str[1:]. - str.join("") - ) - - df_gen_base = df_gen_base.loc[~df_gen_base['tech_code'].isna()] - - return df_gen_base - -def average_efficiency(df_gen_base, avg_csp_eff, avg_urn_eff): - - # ### Calculate average InputActivityRatio by node+technology and only by technology - df_eff = df_gen_base[['node_code', - 'efficiency', - 'tech_code']] - - # Change IAR for CSP value taken from PLEXOS - df_eff.loc[df_eff['tech_code']=='CSP', 'efficiency'] = avg_csp_eff - - # Change IAR for URN value taken from PLEXOS - df_eff.loc[df_eff['tech_code']=='URN', 'efficiency'] = avg_urn_eff - - # Average efficiency by node and technology - df_eff_node = df_eff.groupby(['tech_code', - 'node_code'], - as_index = False).mean() - - df_eff_node['node_average_iar'] = ((1 / df_eff_node['efficiency']). - round(2)) - - df_eff_node.drop('efficiency', - axis = 1, - inplace = True) - - # Average efficiency by technology - df_eff_tech = df_eff.drop(columns="node_code").groupby('tech_code', as_index = False).mean() - - df_eff_tech['tech_average_iar'] = ((1 / df_eff_tech['efficiency']). - round(2)) - - df_eff_tech.drop('efficiency', - axis = 1, - inplace = True) - - return df_eff_node, df_eff_tech - -def residual_capacity(df_gen_base, tech_list, df_tech_code, df_custom_res_cap, duplicate_techs): - - # ### Calculate residual capacity - res_cap_cols = [ - "node_code", - "tech_code", - "total_capacity", - "start_year", - "retirement_year_model", - ] - - df_res_cap = df_gen_base[res_cap_cols] - - for each_year in range(model_start_year, model_end_year+1): - df_res_cap[str(each_year)] = 0 - - df_res_cap = pd.melt( - df_res_cap, - id_vars=res_cap_cols, - value_vars=[x for x in df_res_cap.columns if x not in res_cap_cols], - var_name="model_year", - value_name="value", - ) - df_res_cap["model_year"] = df_res_cap["model_year"].astype(int) - df_res_cap.loc[ - (df_res_cap["model_year"] >= df_res_cap["start_year"]) - & (df_res_cap["model_year"] <= df_res_cap["retirement_year_model"]), - "value", - ] = df_res_cap["total_capacity"] - - df_res_cap = df_res_cap.groupby( - ["node_code", "tech_code", "model_year"], as_index=False - )["value"].sum() - - # Add column with naming convention - df_res_cap = createPwrTechs(df_res_cap, duplicate_techs) - - # Convert total capacity from MW to GW - df_res_cap['value'] = df_res_cap['value'].div(1000) - - # Rename 'model_year' to 'year' and 'total_capacity' to 'value' - df_res_cap.rename({'model_year': 'YEAR', - 'value': 'VALUE'}, - inplace=True, - axis=1) - # Drop 'tech_code' and 'node_code' - df_res_cap.drop(['tech_code', 'node_code'], - inplace=True, - axis=1) - - # Add 'REGION' column and fill 'GLOBAL' throughout - df_res_cap['REGION'] = region_name - - # Reorder columns - df_res_cap = df_res_cap[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - - if custom_nodes: - df_res_cap_custom, custom_techs = custom_nodes_csv(custom_nodes, - df_custom_res_cap, - region_name, - years, - tech_list) - df_res_cap = pd.concat([df_res_cap, df_res_cap_custom]) - - df_res_cap.drop_duplicates(subset=['REGION','TECHNOLOGY','YEAR'], - keep='last', - inplace=True) - df_res_cap = df_res_cap.loc[(df_res_cap['TECHNOLOGY'].str.startswith('PWR')) & - (~df_res_cap['TECHNOLOGY'].str.endswith('00'))] - - df_res_cap.to_csv(os.path.join(output_data_dir, - "ResidualCapacity.csv"), - index=None) - - return df_res_cap, custom_techs if custom_nodes else df_res_cap - -"""The output of the gem_cap function is currently unused. Implementation is being discussed so kept as standalone -function for the time being.""" -def gem_cap(df_gen_base, tech_code_dict, op_life_dict): - - # ### Calculate planned capacities based on the Global Energy Observatory datasets - - # Add spatial mapping to link Global Energy Observatory region naming conventions with OSeMOSYS Global - df_gem_regions = pd.read_csv(os.path.join(input_data_dir, - "gem_region_mapping.csv"), encoding = "ISO-8859-1") - - # pull locations from existing powerplants in PLEXOS-World dataset - gen_locationsinput = pd.read_excel(os.path.join(input_data_dir, - "PLEXOS_World_2015_Gold_V1.1.xlsx"), - sheet_name = "Attributes") - - gen_lat = gen_locationsinput.loc[(gen_locationsinput['class'].isin(['Battery', 'Generator'])) & - (gen_locationsinput['attribute'] == 'Latitude')].set_index('name') - - gen_long = gen_locationsinput.loc[(gen_locationsinput['class'].isin(['Battery', 'Generator'])) & - (gen_locationsinput['attribute'] == 'Longitude')].set_index('name') - - gen_locations = pd.DataFrame(gen_long.index).merge( - gen_lat['value'], left_on = 'name', right_index = True).merge( - gen_long['value'], left_on = 'name', right_index = True) - - gen_locations = pd.merge(gen_locations, df_gen_base[['node', 'country_code', 'node_code', 'powerplant']], - left_on = 'name', right_on = 'powerplant', how = 'inner') - - subcountries = gen_locations.loc[~gen_locations['node_code'].str.contains('XX')]['country_code'].str[:3].unique() - - # Set column name dictionaries for different Global Energy Monitor (gem) input datasets - gem_coal_col = {'Country' : 'Country', 'Capacity (MW)' : 'VALUE', - 'Status' : 'Status', 'Year' : 'Year_built', - 'RETIRED' : 'Year_retired', 'Planned Retire' : 'Year_retired_planned', - 'Latitude' : 'Latitude', 'Longitude' : 'Longitude'} - - gem_gas_col = {'Country' : 'Country', 'Capacity elec. (MW)' : 'VALUE', - 'Status' : 'Status', 'Start year' : 'Year_built', - 'Retired year' : 'Year_retired', 'Planned retire' : 'Year_retired_planned', - 'Latitude' : 'Latitude', 'Longitude' : 'Longitude', - 'Technology' : 'Technology'} - - # Set technology dictionary to match with OSeMOSYS global technologies - gem_gas_dict = {'CC' : 'CCG', - 'GT' : 'OCG', - 'ICCC' : 'CCG', - 'ISCC' : 'CCG', - 'ST' : 'OCG', - 'AFC' : 'CCG'} - - # Set which status criteria are used to filter datasets - new_criteria = ['operating', 'proposed', 'announced', 'pre-permit', 'permitted', 'construction']# 'shelved' & cancelled' not included - old_criteria = ['mothballed', 'retired', 'operating']# operating added because currently operating plants can already have an intended retirement year added - - # Import gem Datasets - gem_coal = pd.read_excel(os.path.join(input_data_dir, - 'Global-Coal-Plant-Tracker-Jan-2022.xlsx'), - sheet_name = 'Units', usecols = gem_coal_col.keys()) - - - gem_gas = pd.read_excel(os.path.join(input_data_dir, - 'Global-Gas-Plant-Tracker-Feb-2022.xlsx'), - sheet_name = 'Gas Units', usecols = gem_gas_col.keys()) - - # Add Technology columns and drop entries with no capacity values - gem_coal.rename(columns = gem_coal_col, inplace = True) - gem_coal['Technology'] = 'COA' - gem_coal = gem_coal[pd.to_numeric(gem_coal['VALUE'], errors='coerce').notnull()] - - gem_gas.rename(columns = gem_gas_col, inplace = True) - gem_gas['Technology'] = gem_gas['Technology'].map(gem_gas_dict) - gem_gas = gem_gas[pd.to_numeric(gem_gas['VALUE'], errors='coerce').notnull()] - - # For entries in the gem dataset with no specified gas technology same assumptions are applied as with OSeMOSYS Global - gem_gas.loc[ - (gem_gas["Technology"].isna()) & (gem_gas["VALUE"].astype(float) > 130), - "Technology", - ] = "CCG" - gem_gas.loc[ - (gem_gas["Technology"].isna()) & (gem_gas["VALUE"].astype(float) <= 130), - "Technology", - ] = "OCG" - - # Combine different datasets - gem_all = pd.concat([gem_coal, gem_gas]) - - # Add spatial mapping - gem_all = gem_all.merge(df_gem_regions, left_on = 'Country', right_on = 'gem_region') - - gem_concat = pd.DataFrame(columns = gem_all.columns) - - # Matches the lat/longs of plants in the gem datasets with lat/longs of the nearest plants in PLEXOS-World. - # The associated sub-country node of the nearest plant is assumed to be the node for the gem dataset entry. - for a in subcountries: - - gen_locations_sc = gen_locations.loc[gen_locations['country_code'].str.contains(a)].reset_index(drop = True) - gem_all_sc = gem_all.loc[gem_all['country_code'] == a].reset_index(drop = True) - - source = spatial.KDTree(gen_locations_sc[['value_x', 'value_y']].to_numpy()) - - output = pd.DataFrame(source.query([gem_all_sc[['Latitude', 'Longitude'] - ].to_numpy()])[1].transpose()) - - gem_all_sc = gem_all_sc.merge(output, left_index = True, right_index = True).set_index(0) - gem_all_sc = gem_all_sc.merge(gen_locations_sc[['node_code']], left_index = True, - right_index = True) - - gem_concat = pd.concat([gem_concat, gem_all_sc]) - - # Adds matched sub-country entries to original df and sets node codes. - gem_all = gem_all.loc[~gem_all['country_code'].isin(subcountries)] - gem_all = pd.concat([gem_all, gem_concat], axis = 0).reset_index(drop = True) - gem_all['node_code'].fillna(gem_all['country_code'] + 'XX', inplace = True) - - # Filters datframe for new plants by year entry - gem_all_new = gem_all.loc[(gem_all['Status'].isin(new_criteria)) & (gem_all['Year_built'].notna()) - ].reset_index(drop = True) - - # Strips year entry to single value (last year taken if range is given e.g. 2025-) - gem_all_new['YEAR'] = np.where(gem_all_new['Year_built'].astype(str).str.len() == 4, gem_all_new['Year_built'], - gem_all_new['Year_built'].str[-4:]) - - # Drops non-existing or non-numerical entries (e.g. 'no year found') and entries from < model_start_year - gem_all_new['YEAR'] = gem_all_new['YEAR'].apply(pd.to_numeric, errors = 'coerce') - gem_all_new = gem_all_new[(gem_all_new['YEAR'].notna()) & - (gem_all_new['YEAR'] > model_start_year)].reset_index() - - gem_all_retired = gem_all.loc[(gem_all['Status'].isin(old_criteria)) & (gem_all['Year_retired'].notna()) | - (gem_all['Year_retired_planned'].notna())] - - # Pulls retirement OR planned retirement year - gem_all_retired['YEAR'] = np.where(gem_all_retired['Year_retired'].notna(), gem_all_retired['Year_retired'], - gem_all_retired['Year_retired_planned']) - - # Strips year entry to single value (last year taken if range is given e.g. 2025-2030) - gem_all_retired['YEAR'] = np.where(gem_all_retired['YEAR'].astype(str).str.len().isin({4,6}), gem_all_retired['YEAR'], - gem_all_retired['YEAR'].str[-4:]) - - # Drops non-existing or non-numerical entries (e.g. 'no year found') and entries from < model_start_year - gem_all_retired['YEAR'] = gem_all_retired['YEAR'].apply(pd.to_numeric, errors = 'coerce') - gem_all_retired = gem_all_retired[(gem_all_retired ['YEAR'].notna()) & - (gem_all_retired['YEAR'] > model_start_year)].reset_index() - - # Group values by technology, node & year - gem_all_new_agg = gem_all_new.groupby(['node_code', 'Technology', 'YEAR'], - as_index=False)['VALUE'].sum() - - # Adds lifetime to planned capacities, calculates future retirement year and adds to retirement dataframe. - tech_code_dict_inv = {v: k for k, v in tech_code_dict.items()} - gem_all_new_agg_oplife = gem_all_new_agg.copy() - gem_all_new_agg_oplife['operational_life'] = gem_all_new_agg_oplife['Technology' - ].map(tech_code_dict_inv).map(op_life_dict) - - gem_all_new_agg_oplife['YEAR'] = gem_all_new_agg_oplife['YEAR'] + gem_all_new_agg_oplife['operational_life'] - - gem_all_retired = pd.concat([gem_all_retired, gem_all_new_agg_oplife]) - - gem_all_retired_agg = gem_all_retired.groupby(['node_code', 'Technology', 'YEAR'], - as_index=False)['VALUE'].sum() - -def activity_master_start(df_gen_base, nodes_extra_list, duplicate_techs, mode_list): - - # ### Add input and output activity ratios - - # Create master table for activity ratios - if custom_nodes: - node_list = list(df_gen_base['node_code'].unique()) + custom_nodes - else: - node_list = list(df_gen_base['node_code'].unique()) - # Add extra nodes which are not present in 2015 but will be by 2050 - for each_node in nodes_extra_list: - if len(each_node) <= 6: - node_list.append("".join(each_node.split('-')[1:]) + 'XX') - else: - node_list.append("".join(each_node.split('-')[1:])) - - master_fuel_list = list(df_gen_base['tech_code'].unique()) - master_fuel_list.append('CCS') - - df_ratios = pd.DataFrame(list(itertools.product(node_list, - master_fuel_list, - mode_list, - years) - ), - columns = ['node_code', 'tech_code', 'MODE_OF_OPERATION', 'YEAR'] - ) - - df_ratios = createPwrTechs(df_ratios, duplicate_techs) - - return df_ratios - -def activity_output_pwr(df_ratios, thermal_fuel_list_oar): - - # #### OutputActivityRatio - Power Generation Technologies - df_oar = df_ratios.copy() - mask = df_oar['TECHNOLOGY'].apply(lambda x: x[3:6] in thermal_fuel_list_oar) - df_oar['FUEL'] = 0 - df_oar['FUEL'][mask] = 1 - df_oar = df_oar.loc[~((df_oar['MODE_OF_OPERATION'] > 1) & - (df_oar['FUEL'] == 0))] - df_oar['FUEL'] = ('ELC' + - df_oar['TECHNOLOGY'].str[6:11] + - '01' - ) - df_oar['VALUE'] = 1 - - # Add 'REGION' column and fill 'GLOBAL' throughout - df_oar['REGION'] = region_name - - # Select columns for final output table - df_oar_base = df_oar[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] - - return df_oar, df_oar_base - -def activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, - df_eff_node, df_eff_tech): - - # #### InputActivityRatio - Power Generation Technologies - # Copy OAR table with all columns to IAR - df_iar = df_oar.copy() - - df_iar['FUEL'] = 0 - - # Deal with GAS techs first... OCG and CCG - # OCG Mode 1: Domestic GAS - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['OCG'])), - 'FUEL'] = 'GAS'+df_iar['TECHNOLOGY'].str[6:9] - # OCG Mode 2: International GAS - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['OCG'])), - 'FUEL'] = 'GASINT' - - # CCG Mode 1: Domestic GAS - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['CCG'])), - 'FUEL'] = 'GAS'+df_iar['TECHNOLOGY'].str[6:9] - - # CCG Mode 2: International GAS - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['CCG'])), - 'FUEL'] = 'GASINT' - - # CCS Mode 1: Domestic COA - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['CCS'])), - 'FUEL'] = 'COA'+df_iar['TECHNOLOGY'].str[6:9] - - # CCS Mode 2: International COA - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['CCS'])), - 'FUEL'] = 'COAINT' - - # For non-GAS thermal fuels, domestic fuel input by country in mode 1 and - # 'international' fuel input in mode 2 - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(thermal_fuel_list_iar)), - 'FUEL'] = df_iar['TECHNOLOGY'].str[3:9] - - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & - (df_iar['TECHNOLOGY'].str[3:6].isin(thermal_fuel_list_iar)), - 'FUEL'] = df_iar['TECHNOLOGY'].str[3:6] + 'INT' - - # For renewable fuels, input by node in mode 1 - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(renewables_list)), - 'FUEL'] = df_iar['TECHNOLOGY'].str[3:11] - - # Remove mode 2 when not used - df_iar = df_iar.loc[df_iar['FUEL'] != 0] - - # Join efficiency columns: one with node and technology average, and the - # other with technology average - df_iar = df_iar.join(df_eff_node.set_index(['tech_code', 'node_code']), - on=['tech_code', 'node_code']) - - df_iar = df_iar.join(df_eff_tech.set_index('tech_code'), - on='tech_code') - - # When available, choose node and technology average. Else, - # choose technology average - df_iar['VALUE'] = df_iar['node_average_iar'] - - df_iar.loc[df_iar['TECHNOLOGY'].str.startswith('PWRCCS'), - 'tech_average_iar'] = 3 - - df_iar.loc[df_iar['VALUE'].isna(), - 'VALUE'] = df_iar['tech_average_iar'] - df_iar.drop_duplicates(inplace=True) - - # Add 'REGION' column and fill 'GLOBAL' throughout - df_iar['REGION'] = region_name - - # Select columns for final output table - df_iar_base = df_iar[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] - - return df_iar_base - -def activity_upstream(df_iar_base, renewables_list, thermal_fuels_list_mining): - - # #### OutputActivityRatios - Upstream - - # We have to create a technology to produce every fuel that is input into any of the power technologies: - - df_oar_upstream = df_iar_base.copy() - - # All mining and resource technologies have an OAR of 1... - df_oar_upstream['VALUE'] = 1 - - # Renewables - set the technology as RNW + FUEL - df_oar_upstream.loc[df_oar_upstream['FUEL'].str[0:3].isin(renewables_list), - 'TECHNOLOGY'] = 'RNW'+df_oar_upstream['FUEL'] - - # If the fuel is a thermal fuel, we need to create the OAR for the mining technology... BUT NOT FOR THE INT FUELS... - df_oar_upstream.loc[df_oar_upstream['FUEL'].str[0:3].isin(thermal_fuels_list_mining) & - ~(df_oar_upstream['FUEL'].str[3:6] == "INT"), - 'TECHNOLOGY'] = 'MIN'+df_oar_upstream['FUEL'] - - # Above should get all the outputs for the MIN technologies, but we need to adjust the mode 2 ones to just the fuel code (rather than MINCOAINT) - df_oar_upstream.loc[df_oar_upstream['MODE_OF_OPERATION']==2, - 'TECHNOLOGY'] = 'MIN'+df_oar_upstream['FUEL'].str[0:3]+df_oar_upstream['TECHNOLOGY'].str[6:9] - df_oar_upstream.loc[df_oar_upstream['MODE_OF_OPERATION']==2, - 'FUEL'] = df_oar_upstream['FUEL'].str[0:3] - - # Now remove the duplicate fuels that the above created (because there's now a COA for each country, not each region, and GAS is repeated twice for each region as well): - df_oar_upstream.drop_duplicates(keep='first',inplace=True) - - # Now we have to create the MINXXXINT technologies. They are all based on the MODE_OF_OPERATION == 2: - df_oar_int = pd.DataFrame(df_oar_upstream.loc[df_oar_upstream['MODE_OF_OPERATION'] == 2, :]) - - # At this point we should have only the internationally traded fuels since they're all mode 2. So we can make the tech MINXXXINT and that's that. - df_oar_int['TECHNOLOGY'] = 'MIN'+df_oar_int['FUEL']+'INT' - # And rename the fuel to XXXINT - df_oar_int['FUEL'] = df_oar_int['FUEL']+'INT' - df_oar_int['MODE_OF_OPERATION'] = 1 # This is probably not strictly necessary as long as they're always the same in and out... - - # and de-duplicate this list: - df_oar_int.drop_duplicates(keep='first',inplace=True) - - # #### Input Activity Ratios - Upstream - - # All we need to do is take in the thermal fuels for the MINXXXINT technologies. This already exists as df_oar_int with the XXINT fuel so we can simply copy that: - df_iar_int = df_oar_int.copy() - df_iar_int['FUEL'] = df_iar_int['FUEL'].str[0:3] - - return df_oar_upstream, df_oar_int - -def activity_transmission(df_oar_base, df_pw_prop, df_trn_efficiencies): - - # #### Downstream Activity Ratios - - # Build transmission system outputs - - df_iar_trn = df_oar_base.copy() - - # Change the technology name to PWRTRNXXXXX - df_iar_trn["TECHNOLOGY"] = "PWRTRN" + df_iar_trn["FUEL"].str[3:8] - # Make all modes of operation 1 - df_iar_trn["MODE_OF_OPERATION"] = 1 - # And remove all the duplicate entries - df_iar_trn.drop_duplicates(keep="first", inplace=True) - - # OAR for transmission technologies is IAR, but the fuel is 02 instead of 01: - df_oar_trn = df_iar_trn.copy() - df_oar_trn["FUEL"] = df_oar_trn["FUEL"].str[0:8] + "02" - - # Build international transmission system from original input data, but for Line rather than Generator: - int_trn_cols = ["child_class", "child_object", "property", "value"] - df_int_trn = df_pw_prop[int_trn_cols] - df_int_trn = df_int_trn[df_int_trn["child_class"] == "Line"] - - # For IAR and OAR we can drop the value: - df_int_trn = df_int_trn.drop(["child_class", "value"], axis=1) - - # Create MofO column based on property: - df_int_trn["MODE_OF_OPERATION"] = 1 - df_int_trn.loc[df_int_trn["property"] == "Min Flow", "MODE_OF_OPERATION"] = 2 - - # Use the child_object column to build the technology names: - df_int_trn["codes"] = df_int_trn["child_object"].str.split(pat="-") - - # If there are only two locations, then the node is XX - df_int_trn.loc[df_int_trn["codes"].str.len() == 2, "TECHNOLOGY"] = ( - "TRN" + df_int_trn["codes"].str[0] + "XX" + df_int_trn["codes"].str[1] + "XX" - ) - # If there are four locations, the node is already included - df_int_trn.loc[df_int_trn["codes"].str.len() == 4, "TECHNOLOGY"] = ( - "TRN" - + df_int_trn["codes"].str[0] - + df_int_trn["codes"].str[1] - + df_int_trn["codes"].str[2] - + df_int_trn["codes"].str[3] - ) - # If there are three items, and the last item is two characters, then the second item is an XX: - df_int_trn.loc[ - (df_int_trn["codes"].str.len() == 3) & (df_int_trn["codes"].str[2].str.len() == 2), - "TECHNOLOGY", - ] = ( - "TRN" - + df_int_trn["codes"].str[0] - + "XX" - + df_int_trn["codes"].str[1] - + df_int_trn["codes"].str[2] - ) - # If there are three items, and the last item is three characters, then the last item is an XX: - df_int_trn.loc[ - (df_int_trn["codes"].str.len() == 3) & (df_int_trn["codes"].str[2].str.len() == 3), - "TECHNOLOGY", - ] = ( - "TRN" - + df_int_trn["codes"].str[0] - + df_int_trn["codes"].str[1] - + df_int_trn["codes"].str[2] - + "XX" - ) - - # Set the value (of either IAR or OAR) to 1 - df_int_trn["VALUE"] = 1 - df_int_trn["REGION"] = region_name - - df_int_trn = df_int_trn.drop(["property", "child_object", "codes"], axis=1) - df_int_trn["YEAR"] = model_start_year - - # add in future years - df_int_trn_new = df_int_trn.copy() - df_int_trn_new["YEAR"] = [range(model_start_year + 1, model_end_year + 1)] * len(df_int_trn_new) - df_int_trn_new = df_int_trn_new.explode(column="YEAR") - - df_int_trn = pd.concat([df_int_trn, df_int_trn_new]).reset_index(drop=True) - - # Now create the input and output activity ratios - df_int_trn_oar = df_int_trn.copy() - df_int_trn_iar = df_int_trn.copy() - - # IAR Mode 1 is input from first country: - df_int_trn_iar.loc[df_int_trn_iar["MODE_OF_OPERATION"] == 1, "FUEL"] = ( - "ELC" + df_int_trn_iar["TECHNOLOGY"].str[3:8] + "02" - ) - # IAR Mode 2 is input from second country: - df_int_trn_iar.loc[df_int_trn_iar["MODE_OF_OPERATION"] == 2, "FUEL"] = ( - "ELC" + df_int_trn_iar["TECHNOLOGY"].str[8:13] + "02" - ) - - # OAR Mode 2 is output to first country: - df_int_trn_oar.loc[df_int_trn_oar["MODE_OF_OPERATION"] == 2, "FUEL"] = ( - "ELC" + df_int_trn_oar["TECHNOLOGY"].str[3:8] + "01" - ) - # OAR Mode 1 is out to the second country: - df_int_trn_oar.loc[df_int_trn_oar["MODE_OF_OPERATION"] == 1, "FUEL"] = ( - "ELC" + df_int_trn_oar["TECHNOLOGY"].str[8:13] + "01" - ) - - # Drop unneeded columns - df_trn_efficiencies = df_trn_efficiencies.drop( - [ - "Interface", - "KM distance", - "HVAC/HVDC/Subsea", - "Build Cost ($2010 in $000)", - "Build cost + FOM (3.5% of Capex per year)", - "Unnamed: 8", - "Interface is per MW", - "Unnamed: 10", - "Unnamed: 11", - "Unnamed: 12", - "Subsea lines", - "Unnamed: 14" - ], - axis=1, - ) - - # Drop NaN values - df_trn_efficiencies = df_trn_efficiencies.dropna(subset=["From"]) - - # Create tech column from To and From Codes: - df_trn_efficiencies = format_transmission_name(df_trn_efficiencies) - - # Rename column 'VALUES' - df_trn_efficiencies = df_trn_efficiencies.rename(columns={"Losses": "VALUE"}) - - # And adjust OAR values to be output amounts vs. losses: - df_trn_efficiencies['VALUE'] = 1.0 - df_trn_efficiencies['VALUE'] - - # and add values into OAR matrix - df_int_trn_oar = df_int_trn_oar.drop(["VALUE"], axis=1) - df_int_trn_oar = pd.merge( - df_int_trn_oar, df_trn_efficiencies, how="outer", on="TECHNOLOGY" - ) - - return df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar - -def activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, - df_int_trn_oar, df_iar_final, df_iar_trn, df_int_trn_iar, - duplicate_techs): - - # #### Output IAR and OAR - - # Combine the pieces from above and output to csv: - - df_oar_final = pd.concat( - [ - df_oar_base, # If we want to split transmission to different script might have to not used df_oar_final as input to this function. - df_oar_upstream, - df_oar_int, - df_oar_trn, - df_int_trn_oar, - ] - ).dropna() - - # Select columns for final output table - df_oar_final = df_oar_final[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] - - df_iar_final = pd.concat( - [ - df_iar_final, - df_iar_trn, - df_int_trn_iar, - ] - ).dropna() - - # Select columns for final output table - df_iar_final = df_iar_final[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE']] - - # Add iar for techs not using PLEXOS values - df_iar_newTechs = duplicatePlexosTechs(df_iar_final, duplicate_techs) - for duplicate_tech in duplicate_techs: - df_new_iar = newIar(df_iar_newTechs, duplicate_tech,new_iar_ccg, - new_iar_ocg, new_iar_coa, new_iar_default) - df_iar_final = pd.concat([df_iar_final, df_new_iar]) - - # Add oar for techs not using PLEXOS values - df_oar_newTechs = duplicatePlexosTechs(df_oar_final, duplicate_techs) - df_oar_final = pd.concat([df_oar_final, df_oar_newTechs]) - - df_oar_final.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR'], - keep='last', - inplace=True) - - df_oar_final.to_csv(os.path.join(output_data_dir, - "OutputActivityRatio.csv"), - index=None) - - df_iar_final = apply_dtypes(df_iar_final, "InputActivityRatio") - df_iar_final.to_csv(os.path.join(output_data_dir, - "InputActivityRatio.csv"), - index=None) - - return df_oar_final, df_iar_final - -def costs_pwr(df_weo_data, costs_dict): - - # ### Costs: Capital, fixed, and variable - - df_costs = pd.melt(df_weo_data, - id_vars = ['technology', 'weo_region', 'parameter'], - value_vars = ['2019', '2030', '2040'], - var_name = 'YEAR') - df_costs['parameter'] = df_costs['parameter'].str.split('\r\n').str[0] - df_costs['value'] = df_costs['value'].replace({'n.a.':0}) - df_costs['value'] = df_costs['value'].astype(float) - df_costs = df_costs.pivot_table(index = ['technology', 'parameter', 'YEAR'], - columns = 'weo_region', - values = 'value').reset_index() - df_costs['AS_average'] = (df_costs['China'] + - df_costs['India'] + - df_costs['Japan'] + - df_costs['Middle East']).div(4) - df_costs['SEA_average'] = (df_costs['Indonesia'] + - df_costs['Vietnam']).div(2) - df_costs['NA_average'] = (df_costs['United States']) - df_costs['SA_average'] = (df_costs['Brazil']) - df_costs['Global_average'] = (df_costs['Africa'] + - df_costs['Brazil'] + - df_costs['Europe'] + - df_costs['China'] + - df_costs['India'] + - df_costs['Japan'] + - df_costs['Middle East'] + - df_costs['Russia'] + - df_costs['United States']).div(9) - df_costs = pd.melt(df_costs, - id_vars = ['technology', 'parameter', 'YEAR'], - value_vars = [x - for x - in df_costs.columns - if x not in ['technology', 'parameter', 'YEAR'] - ] - ) - df_costs['YEAR'] = df_costs['YEAR'].astype(int) - - df_costs = df_costs.loc[df_costs['technology'].isin(costs_dict.keys())] - df_costs['technology_code'] = df_costs['technology'].replace(costs_dict) - - return df_costs - -def format_transmission_name(df): - '''Formats PLEXOS transmission names into OSeMOSYS Global names. - - Args: - :param df: Pandas DataFrame with columns 'From' and 'To' describing the - transmission from and to contries. ie. - - Returns: - :param df: Same as df_in, except the 'From' and 'To' columns are replaced - with a single 'TECHNOLOGY' column holding OSeMOSYS Global - naming conventions - - Example: - df = pd.DataFrame( - [[AF-COD, AF-COG, 0.001], - [EU-AUT, EU-SVK, 0.004], - [AS-LBN, AS-SYR, 0.006]], - columns = ['From', 'To', 'Losses'] - ) - pd.DataFrame( - [[0.001,TRNCODXXCOGXX], - [0.004,TRNAUTXXSVKXX], - [0.006,TRNLBNXXSYRXX]] - columns = ['Losses','TECHNOLOGY'])''' - - # If from column has length 6 then it's the last three chars plus XX - df.loc[df["From"].str.len() == 6, "From"] = (df["From"].str[3:6] + "XX") - - # If from column has length 9 then it's the 3:6 and 7:9 three chars plus XX - df.loc[df["From"].str.len() == 9, "From"] = ( - df["From"].str[3:6] + df["From"].str[7:9]) - - # If to column has length 6 then it's the last three chars plus XX - df.loc[df["To"].str.len() == 6, "To"] = (df["To"].str[3:6] + "XX") - - # If to column has length 9 then it's the 3:6 and 7:9 three chars plus XX - df.loc[df["To"].str.len() == 9, "To"] = ( - df["To"].str[3:6] + df["To"].str[7:9]) - - # Combine From and To columns. - df["TECHNOLOGY"] = ("TRN" + df["From"] + df["To"]) - - # Drop to and from columns - df = df.drop(["From", "To"], axis=1) - - return df - -def get_transmission_costs(df_trn_lines, df_oar_final): - '''Gets electrical transmission capital and fixed cost per technology. - - Both the capital costs and fixed cost are written out to avoid having - to read in the excel file twice - - Returns: - df_trans_capex: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' - representing CAPITAL cost in millions of dollars per year. - df_trans_fix: DataFrame with the columns 'TECHNOLOGY' and 'VALUE' - representing FIXED cost in millions of dollars per year. - ''' - - # Drop unneeded columns - df = df_trn_lines.drop( - [ - "Line", - "KM distance", - "HVAC/HVDC/Subsea", - "Losses", - "Unnamed: 8", - "Line Max Size (MW)", - "Unnamed: 10", - "Unnamed: 11", - "Unnamed: 12", - "Subsea lines", - "Unnamed: 14" - ], - axis=1, - ) - - # Use to/from codes to create a TECHNOLOGY columns - df = format_transmission_name(df) - - # Changes units - # Raw given in dollars -> model units in million dollars - df = df.rename(columns={'Annual FO&M (3.5% of CAPEX) ($2010 in $000)':'O&M'}) - df = df.rename(columns={'Build Cost ($2010 in $000)':'CAPEX'}) - df['O&M'] = df['O&M'].astype(float) / 1000 - df['CAPEX'] = df['CAPEX'].astype(float) / 1000 - df = df.round({'O&M': 3, 'CAPEX': 3, }) - - # Separate out fixed and capex and return values - df_trans_capex = df.drop(['O&M'], axis=1) - df_trans_capex = df_trans_capex.rename(columns={'CAPEX':'VALUE'}) - df_trans_fix = df.drop(['CAPEX'], axis=1) - df_trans_fix = df_trans_fix.rename(columns={'O&M':'VALUE'}) - - df_trans_capex['REGION'] = region_name - df_trans_capex['YEAR'] = [years] * len(df_trans_capex) - df_trans_capex = df_trans_capex.explode('YEAR') - - df_trans_fix['REGION'] = region_name - df_trans_fix['YEAR'] = [years] * len(df_trans_fix) - df_trans_fix = df_trans_fix.explode('YEAR') - - # Filter out techs that don't have activity ratios - df_trans_capex = df_trans_capex.loc[ - df_trans_capex['TECHNOLOGY'].isin(df_oar_final['TECHNOLOGY'])] - df_trans_fix = df_trans_fix.loc[ - df_trans_fix['TECHNOLOGY'].isin(df_oar_final['TECHNOLOGY'])] - - return df_trans_capex, df_trans_fix - -def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, - df_trans_fix): - - weo_regions_dict = dict([(k, v) - for k, v - in zip(df_weo_regions['technology_code'], - df_weo_regions['weo_region'] - ) - ] - ) - - # Create formatted CSVs - for each_cost in ['Capital', 'O&M']: - df_costs_temp = df_costs.loc[df_costs['parameter'].str.contains(each_cost)] - - df_costs_temp.drop(['technology', 'parameter'], - axis = 1, - inplace = True) - df_costs_final = df_oar_final[['REGION', - 'TECHNOLOGY', - 'YEAR' - ]] - df_costs_final['YEAR'] = df_costs_final['YEAR'].astype(int) - df_costs_final = df_costs_final.drop_duplicates() - df_costs_final = (df_costs_final - .loc[(df_costs_final['TECHNOLOGY'] - .str.startswith('PWR') - ) & - (~df_costs_final['TECHNOLOGY'] - .str.contains('TRN') - ) - ] - ) - df_costs_final['technology_code'] = df_costs_final['TECHNOLOGY'].str[3:6] - df_costs_final['weo_region'] = df_costs_final['TECHNOLOGY'].str[6:9] - df_costs_final['weo_region'] = (df_costs_final['weo_region'] - .replace(weo_regions_dict)) - - df_costs_final = pd.merge(df_costs_final, - df_costs_temp, - on = ['technology_code', 'weo_region', 'YEAR'], - how = 'left' - ) - df_costs_final.drop(['technology_code', 'weo_region'], - axis = 1, - inplace = True) - df_costs_final = df_costs_final.fillna(-9) - df_costs_final = pd.pivot_table(df_costs_final, - index = ['REGION', 'YEAR'], - columns = 'TECHNOLOGY', - values = 'value').reset_index() - df_costs_final = df_costs_final.replace([-9],[np.nan]) - - df_costs_final = df_costs_final.interpolate(method = 'linear', - limit_direction='forward').round(2) - df_costs_final = df_costs_final.interpolate(method = 'linear', - limit_direction='backward').round(2) - df_costs_final = pd.melt(df_costs_final, - id_vars = ['REGION', 'YEAR'], - value_vars = [x for x in df_costs_final.columns - if x not in ['REGION', 'YEAR'] - ], - var_name = 'TECHNOLOGY', - value_name = 'VALUE' - ) - df_costs_final = df_costs_final[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - df_costs_final = df_costs_final[~df_costs_final['VALUE'].isnull()] - - if each_cost in ['Capital']: - df_costs_final_capital = df_costs_final.merge(df_trans_capex, how='outer') - df_costs_final_capital = apply_dtypes(df_costs_final_capital, "CapitalCost") - df_costs_final_capital.to_csv(os.path.join(output_data_dir, - "CapitalCost.csv"), - index = None) - - if each_cost in ['O&M']: - df_costs_final_fixed = df_costs_final.merge(df_trans_fix, how='outer') - df_costs_final_fixed = apply_dtypes(df_costs_final_fixed, "FixedCost") - df_costs_final_fixed.to_csv(os.path.join(output_data_dir, - "FixedCost.csv"), - index = None) - - return df_costs_final_capital, df_costs_final_fixed - -def capact(df_oar_final): - - # Create CapacityToActivityUnit csv - df_capact_final = df_oar_final[['REGION', - 'TECHNOLOGY' - ]] - df_capact_final = df_capact_final.drop_duplicates() - df_capact_final = (df_capact_final - .loc[(df_capact_final['TECHNOLOGY'] - .str.startswith('PWR') - ) | - (df_capact_final['TECHNOLOGY'] - .str.contains('TRN') - ) - ] - ) - - df_capact_final['VALUE'] = 31.536 - df_capact_final.drop_duplicates(inplace=True) - df_capact_final.to_csv(os.path.join(output_data_dir, - "CapacityToActivityUnit.csv"), - index = None) - - return df_capact_final - -def activity_transmission_limit(cross_border_trade, df_oar_final): - - # Set cross-border trade to 0 if False - if not cross_border_trade: - df_crossborder_final = df_oar_final[['REGION', - 'TECHNOLOGY' - ]] - df_crossborder_final = df_crossborder_final.drop_duplicates() - df_crossborder_final = (df_crossborder_final - .loc[df_crossborder_final['TECHNOLOGY'] - .str.startswith('TRN') - ] - ) - df_crossborder_final['VALUE'] = 0 - else: - df_crossborder_final = pd.DataFrame(columns=['REGION', - 'TECHNOLOGY', - 'VALUE']) - - df_crossborder_final = apply_dtypes(df_crossborder_final, "TotalTechnologyModelPeriodActivityUpperLimit") - df_crossborder_final.to_csv(os.path.join(output_data_dir, - "TotalTechnologyModelPeriodActivityUpperLimit.csv"), - index = None) - -def op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict): - - # Create Operational Life data - tech_code_dict_reverse = dict((v,k) for k,v in tech_code_dict.items()) - op_life_techs = list(set(list(df_iar_final['TECHNOLOGY'].unique()) + list(df_oar_final['TECHNOLOGY'].unique()))) - op_life_pwr = [tech for tech in op_life_techs if (tech[0:3] == 'PWR') and (len(tech) == 13)] - op_life_trn = [tech for tech in op_life_techs if tech[0:3] == 'TRN'] - - op_life_out = [] - - # power generation technologies - for op_life_tech in op_life_pwr: - op_life_tech_name = tech_code_dict_reverse[op_life_tech[3:6]] - op_life_out.append([ - region_name, - op_life_tech, - op_life_dict[op_life_tech_name]]) - - return op_life_trn, op_life_out - -def op_life_transmission(op_life_trn, op_life_out, op_life_dict): - - # transmission technologies - for op_life_tech in op_life_trn: - op_life_out.append([region_name, op_life_tech, op_life_dict.get('TRN')]) - - op_life_base = pd.DataFrame(op_life_out, columns = ['REGION', 'TECHNOLOGY', 'VALUE']) - - op_life_base = apply_dtypes(op_life_base, "OperationalLife") - op_life_base.to_csv(os.path.join(output_data_dir, - "OperationalLife.csv"), - index = None) - - return op_life_base - -def cap_investment_constraints(df_iar_final, no_investment_techs): - - # Create totalAnnualMaxCapacityInvestment data - - # Do not allow capacity investment for all PWRxxxxxxxx00 technolgoies - max_cap_invest_techs = list(set( - df_iar_final.loc[df_iar_final['TECHNOLOGY'].str.endswith('00')]['TECHNOLOGY'].tolist())) - max_cap_invest_data = [] - for tech in max_cap_invest_techs: - for year in years: - max_cap_invest_data.append([region_name, tech, year, 0]) - - # Do not allow investment for all xxxABCxxxxxxx technologies - - if not no_investment_techs: - no_investment_techs = [] # Change from None type to empty list - max_cap_invest_techs = list(set(df_iar_final.loc[ - df_iar_final['TECHNOLOGY'].str[3:6].isin(no_investment_techs)][ - 'TECHNOLOGY'].tolist())) - for tech in max_cap_invest_techs: - for year in years: - max_cap_invest_data.append([region_name, tech, year, 0]) - - # Save totalAnnualMaxCapacityInvestment - df_max_cap_invest = pd.DataFrame(max_cap_invest_data, - columns = ['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE'] - ) - df_max_cap_invest = apply_dtypes(df_max_cap_invest, "TotalAnnualMaxCapacityInvestment") - df_max_cap_invest.to_csv(os.path.join(output_data_dir, - 'TotalAnnualMaxCapacityInvestment.csv'), - index = None) - - df_min_cap_invest = pd.DataFrame(columns = ['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE'] - ) - df_min_cap_invest = apply_dtypes(df_min_cap_invest, "TotalAnnualMinCapacityInvestment") - df_min_cap_invest.to_csv(os.path.join(output_data_dir, - 'TotalAnnualMinCapacityInvestment.csv'), - index = None) - - return df_max_cap_invest, df_min_cap_invest - -def create_sets(x, df, output_dir, custom_node_elements): - """Creates a formatted otoole set csv - - Arguments: - x = Name of set given by otoole as a string - df = dataframe in extract set information from - output_dir = directory to write file to - - Returns: - None. Writes out csv file - - Example: - create_sets('TECHNOLOGY', df_oar_final, output_dir) - """ - set_elements = list(df[x].unique()) + list(df[x].unique()) + list(custom_node_elements) - set_elements = list(set(set_elements)) - set_elements = [x for x in set_elements if x != 'nan'] - set_elements.sort() - set_elements_df = pd.DataFrame(set_elements, columns = ['VALUE']) - return set_elements_df.to_csv(os.path.join(output_dir, - str(x) + '.csv' - ), - index = None - ) - -def output_sets(df_oar_final, custom_techs, mode_list): - - # ## Create sets for TECHNOLOGIES, FUELS - if custom_nodes: - create_sets('TECHNOLOGY', df_oar_final, output_data_dir, custom_techs) - else: - create_sets('TECHNOLOGY', df_oar_final, output_data_dir, []) - create_sets('FUEL', df_oar_final, output_data_dir, []) - - # ## Create set for YEAR, REGION, MODE_OF_OPERATION - - years_df = pd.DataFrame(years, columns = ['VALUE']).astype(SET_DTYPES["YEAR"]) - years_df.to_csv(os.path.join(output_data_dir, - "YEAR.csv"), - index = None) - - mode_list_df = pd.DataFrame(mode_list, columns = ['VALUE']).astype(SET_DTYPES["MODE_OF_OPERATION"]) - mode_list_df.to_csv(os.path.join(output_data_dir, - "MODE_OF_OPERATION.csv"), - index = None) - - regions_df = pd.DataFrame(columns = ['VALUE']).astype(SET_DTYPES["REGION"]) - regions_df.loc[0] = region_name - regions_df.to_csv(os.path.join(output_data_dir, - "REGION.csv"), - index = None) - -def duplicatePlexosTechs(df_in, techs): - """Creates new technologies to replace PLEXOS technolgoies. - - New technologies will end in '01', while historical ones end in '00' - - Arguments: - df_in = dataframe in otoole and og formatting with a TECHNOLOGY column - techs = List of technology triads to duplicate [CCG, HYD, ...] - - Returns: - df_out = dataframe with same columns as df_in. All tech names that include - techs values will be returned with updated naming. Remaining technologies - are deleted - - Example: - df_out = duplicatePlexosTechs(df_in, ['CCG', 'OCG']) - df_in['TECHNOLOGY'] = [PWRCCGAFGXX01, PWROCGAFGXX01, PWRHYDAFGXX01] - df_out['TECHNOLOGY'] = [PWRCCGAFGXX02, PWROCGAFGXX02] - """ - df_out = df_in.copy() - df_out = df_out.loc[(df_out['TECHNOLOGY'].str[3:6].isin(techs)) & - ~(df_out['TECHNOLOGY'].str.startswith('MIN'))] - df_out['TECHNOLOGY'] = df_out['TECHNOLOGY'].str.slice_replace(start=11, - stop=13, - repl='01') - return df_out - -def createPwrTechs(df_in, techs): - """Formats power generation technology name - - Adds a 'TECHNOLOGY' column to a dataframe with formatted power - generation technology (PWR) names. Names are formatted so the suffix - for plants in the 'techs' argument list will have 00, while everything - else will have an 01 suffix - - Arguments: - df_in = dataframe with a 'tech_codes' and 'node_codes' column - tList = List of technology triads to have 00 suffix [CCG, HYD, ...] - - Returns: - df_out = same dataframe as df_in, except with a 'TECHNOLOGY' column added - to the end - - Example: - df_in['tech_code'] = ('CCG', SPV, 'OCG', 'HYD') - df_in['node_code'] = ('AGOXX', AGOXX, 'INDNP', 'INDNP') - df_out = createPwrTechs(df_in, ['CCG', 'OCG']) - df_out['TECHNOLOGY'] = [PWRCCGAFGXX00, PWRSPVAFGXX01, PWROCGINDNP00, PWRHYDINDNP01] - """ - df_out = df_in.copy() - for t in techs: - df_out.loc[df_out['tech_code'] == t, 'tech_suffix'] = '00' - df_out['tech_suffix'] = df_out['tech_suffix'].fillna('01') - df_out['TECHNOLOGY'] = ('PWR' + - df_out['tech_code'] + - df_out['node_code'] + - df_out['tech_suffix'] - ) - df_out = df_out.drop('tech_suffix', axis = 1) - return df_out - -def newIar(df_in, tech, new_iar_ccg, - new_iar_ocg, new_iar_coa, new_iar_default): - """Replaces the input activity ratio value with a hardcoded value - - Arguments: - df = dataframe with a 'TECHNOLOGY' and 'VALUE' column - tech = technology to replace iar for (CCG, HYD, SPV...) - - Returns: - df_out = same dataframe as df_in with a new values in 'VALUE' - - Example: - df_out = newIar(df_in, 'CCG') - df_out['TECHNOLOGY'] = [PWRCCGINDNP01, PWRCCGINDNW01] - df_out['VALUE'] = [2, 2] - """ - - df_out = df_in.loc[df_in['TECHNOLOGY'].str[3:6] == tech] - if tech == 'CCG': - iar = new_iar_ccg - elif tech == 'OCG': - iar = new_iar_ocg - elif tech == 'COA': - iar = new_iar_coa - else: - logging.warning(f'Default IAR used for new {tech} power plants') - iar = new_iar_default - df_out['VALUE'] = round(1/iar, 3) - return df_out - -def custom_nodes_csv(df_custom, tech_list): - '''Add custom nodes to the model for each relevant input parameter data csv. - - Args: - df : Pandas DataFrame with columns 'From' and 'To' describing the - transmission from and to contries. ie. - - Returns: - df_out : - - ''' - df_param = pd.DataFrame(list(itertools.product(custom_nodes, - tech_list, - years) - ), - columns = ['CUSTOM_NODE', - 'FUEL_TYPE', - 'YEAR'] - ) - df_param['REGION'] = region_name - df_custom = df_custom.groupby(['CUSTOM_NODE', - 'FUEL_TYPE', - 'START_YEAR', - 'END_YEAR'], - as_index=False)['CAPACITY'].sum() - df_param = pd.merge(df_param, - df_custom, - how='left', - on=['CUSTOM_NODE', - 'FUEL_TYPE']) - df_param['TECHNOLOGY'] = ('PWR' + - df_param['FUEL_TYPE'] + - df_param['CUSTOM_NODE'] + - '01') - technologies = df_param['TECHNOLOGY'].unique() - df_param.dropna(inplace=True) - df_param.drop_duplicates(inplace=True) - df_param = df_param.loc[df_param['YEAR'] >= df_param['START_YEAR']] - df_param = df_param.loc[df_param['YEAR'] <= df_param['END_YEAR']] - df_param['VALUE'] = df_param['CAPACITY'].div(1000) - df_param['REGION'] = region_name - df_param = df_param[['REGION','TECHNOLOGY','YEAR','VALUE']] - df_param = df_param.groupby(['REGION', - 'TECHNOLOGY', - 'YEAR'], - as_index=False)['VALUE'].sum() - - return df_param, technologies - -def user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, - df_min_cap_invest, df_max_cap_invest, df_res_cap, - df_iar_final, df_oar_final, fuel_set, op_life_base, cap_act_base, - cap_cost_base, df_iar_custom_val, df_oar_custom_val): - """User-defined capacities are used when a specific technology must be - invested, for a given year and capacity. This is applied through the - parameter 'TotalAnnualMinCapacityInvestment'. - - Args: - region: From config file (e.g. 'GLOBAL') - tech_capacity: User-defined capacity in config file - (e.g. TRNAGOXXCODXX: [5, 2030]) - - Returns - None - """ - techCapacity = [] - tech_capacity_dict = {} - first_year_dict = {} - build_rate_dict = {} - capex_dict = {} - build_year_dict = {} - - if not tech_capacity is None: - - for tech, tech_params in tech_capacity.items(): - techCapacity.append([tech, tech_params[0], tech_params[1]]) - tech_capacity_dict[tech] = tech_params[2] - build_year_dict[tech] = tech_params[1] - first_year_dict[tech] = tech_params[3] - build_rate_dict[tech] = tech_params[4] - capex_dict[tech] = tech_params[5] # - tech_capacity_df = pd.DataFrame(techCapacity, - columns=['TECHNOLOGY', 'VALUE', 'YEAR']) - tech_capacity_df['REGION'] = region_name - tech_capacity_df = tech_capacity_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - - df_tech_set = pd.read_csv(df_tech_set) - - for each_tech in list(tech_capacity_df['TECHNOLOGY'].unique()): - if each_tech not in list(df_tech_set['VALUE']): - df_tech_set = pd.concat([df_tech_set, pd.DataFrame({'VALUE':[each_tech]})]) - - df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_df]) - df_min_cap_inv.drop_duplicates(inplace=True) - - df = pd.DataFrame(list(itertools.product(list(tech_capacity_df['TECHNOLOGY'].unique()), - years) - ), - columns = ['TECHNOLOGY', - 'YEAR'] - ) - df['REGION'] = region_name - df = pd.merge(df, df_min_cap_inv, - how='left', - on=['REGION', 'TECHNOLOGY', 'YEAR']) - df['FIRST_YEAR'] = df['TECHNOLOGY'].map(first_year_dict) - df['BUILD_YEAR'] = df['TECHNOLOGY'].map(build_year_dict) - df['MAX_BUILD'] = df['TECHNOLOGY'].map(build_rate_dict) - - # Fill VALUE with MAX_BUILD for YEAR >= FIRST_YEAR - df.loc[(df['YEAR']>=df['FIRST_YEAR']) & - (df['YEAR']>df['BUILD_YEAR']), - 'VALUE'] = df['MAX_BUILD'] - df.fillna(0, - inplace=True) - max_cap_techs_df = df[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] - - #Replace VALUE with CAPEX - df['VALUE'] = df['TECHNOLOGY'].map(capex_dict) - - # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD for TRN - df_max_cap_inv = pd.concat([df_max_cap_invest, max_cap_techs_df]).drop_duplicates() - - # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD for TRN - df_max_cap_inv.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'YEAR'], - keep='last', - inplace=True) - df_max_cap_inv.to_csv(os.path.join(output_data_dir, - "TotalAnnualMaxCapacityInvestment.csv"), - index=None) - - # For technologies with start year before model start year, add to ResidualCapacity - df_res_cap_ud = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] < min(years)] - df_res_cap_ud.rename(columns={'YEAR':'START_YEAR'}, - inplace=True) - df_res_cap_ud_final = pd.DataFrame(list(itertools.product(df_res_cap_ud['TECHNOLOGY'].unique(), - years) - ), - columns = ['TECHNOLOGY', - 'YEAR'] - ) - df_res_cap_ud_final = pd.merge(df_res_cap_ud_final, - df_res_cap_ud, - how='left', - on=['TECHNOLOGY']) - df_res_cap_ud_final['TECH'] = df_res_cap_ud_final['TECHNOLOGY'].str[3:6] - df_res_cap_ud_final.loc[df_res_cap_ud_final['TECHNOLOGY'].str.contains('TRN'), - 'TECH'] = 'TRN' - df_res_cap_ud_final['OP_LIFE'] = df_res_cap_ud_final['TECH'].map(op_life_dict) - df_res_cap_ud_final['END_YEAR'] = (df_res_cap_ud_final['OP_LIFE'] - + df_res_cap_ud_final['START_YEAR']) - df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] - >= df_res_cap_ud_final['START_YEAR']] - df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] - <= df_res_cap_ud_final['END_YEAR']] - df_res_cap_ud_final['REGION'] = region_name - df_res_cap_ud_final = df_res_cap_ud_final[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] - - df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final]) - df_res_cap.to_csv(os.path.join(output_data_dir, - 'ResidualCapacity.csv'), - index=None) - - # For technologies with start year at or after model start year, add to - # TotalAnnualMinCapacityInvestment - df_min_cap_inv = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] >= min(years)] - df_min_cap_inv.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'YEAR'], - keep='last', - inplace=True) - df_min_cap_inv.to_csv(os.path.join(output_data_dir, - "TotalAnnualMinCapacityInvestment.csv"), - index=None) - df_tech_set.drop_duplicates(inplace=True) - df_tech_set.to_csv(os.path.join(output_data_dir, - "TECHNOLOGY.csv"), - index=None) - - # Add IAR and OAR for custom technologies - tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) - df_iar_custom = pd.DataFrame(list(itertools.product(tech_list, - [1, 2], - years) - ), - columns = ['TECHNOLOGY', - 'MODE_OF_OPERATION', - 'YEAR'] - ) - df_oar_custom = pd.DataFrame(list(itertools.product(tech_list, - [1, 2], - years) - ), - columns = ['TECHNOLOGY', - 'MODE_OF_OPERATION', - 'YEAR'] - ) - # IAR in modes 1 and 2 are primary electricity commodity ('ELC*01') in - # node_from and node_to, respectively. - # OAR is the inverse of the above - df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, - 'FUEL'] = ('ELC' + - df_iar_custom['TECHNOLOGY'].str[3:8] + - '01') - df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, - 'FUEL'] = ('ELC' + - df_iar_custom['TECHNOLOGY'].str[8:13] + - '01') - df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, - 'FUEL'] = ('ELC' + - df_oar_custom['TECHNOLOGY'].str[8:13] + - '01') - df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, - 'FUEL'] = ('ELC' + - df_oar_custom['TECHNOLOGY'].str[3:8] + - '01') - df_iar_custom['VALUE'] = df_iar_custom_val - df_oar_custom['VALUE'] = df_oar_custom_val - df_iar_custom['REGION'] = region_name - df_oar_custom['REGION'] = region_name - - df_iar_custom = df_iar_custom[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] - df_oar_custom = df_oar_custom[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] - - df_iar = pd.concat([df_iar_final, df_iar_custom]) - df_oar = pd.concat([df_oar_final, df_oar_custom]) - - df_iar.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR'], - keep='last', - inplace=True) - df_iar.to_csv(os.path.join(output_data_dir, - 'InputActivityRatio.csv'), - index=None) - df_oar.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR'], - keep='last', - inplace=True) - df_oar.to_csv(os.path.join(output_data_dir, - 'OutputActivityRatio.csv'), - index=None) - # Add new fuels to FUEL set, if not already present - fuel_list = [] - fuel_list = list(df_iar_custom['FUEL'].unique()) + list(df_oar_custom['FUEL'].unique()) - fuel_list = list(set(fuel_list)) - - fuel_set = pd.read_csv(fuel_set) - - for each_fuel in fuel_list: - if each_fuel not in list(fuel_set['VALUE']): - fuel_set = pd.concat([fuel_set, pd.DataFrame({'VALUE':[each_fuel]})]) - - fuel_set.to_csv(os.path.join(output_data_dir, - "FUEL.csv"), - index=None) - - op_life_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) - - op_life_custom.loc[op_life_custom['TECHNOLOGY'].str.contains('TRN'), - 'VALUE'] = op_life_dict.get('TRN') - op_life_custom['REGION'] = region_name - op_life_custom = op_life_custom[['REGION', - 'TECHNOLOGY', - 'VALUE']] - - op_life = pd.concat([op_life_base, op_life_custom]) - op_life.drop_duplicates(subset=['REGION', - 'TECHNOLOGY'], - keep='last', - inplace=True) - op_life.to_csv(os.path.join(output_data_dir, - 'OperationalLife.csv'), - index=None) - - cap_act_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) - cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('TRN'), - 'VALUE'] = 31.536 - cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('PWR'), - 'VALUE'] = 31.536 - cap_act_custom['REGION'] = region_name - cap_act_custom = cap_act_custom[['REGION', - 'TECHNOLOGY', - 'VALUE']] - cap_act = pd.concat([cap_act_base, cap_act_custom]) - cap_act.drop_duplicates(inplace=True) - cap_act.to_csv(os.path.join(output_data_dir, - 'CapacityToActivityUnit.csv'), - index=None) - - tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) - cap_cost_trn = pd.DataFrame(list(itertools.product(tech_list, - years)), - columns = ['TECHNOLOGY', - 'YEAR']) - - # Update CapitalCost with user-defined costs by transmission line - for each_trn in tech_list: - cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.startswith(each_trn), - 'VALUE'] = capex_dict[each_trn] - - cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.contains('PWRTRN'), - 'VALUE'] = 300 - cap_cost_trn['REGION'] = region_name - cap_cost_trn = cap_cost_trn[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] - cap_cost = pd.concat([cap_cost_base, cap_cost_trn]) - cap_cost.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], - keep="last", - inplace=True) - cap_cost.to_csv(os.path.join(output_data_dir, - 'CapitalCost.csv'), - index=None) - - return df_tech_set - -def availability_factor(availability, tech_set_base): - - af_dict = dict(zip(list(availability['technology']), - list(availability['value']))) - - tech_list = [x for x in tech_set_base['VALUE'] - if x.startswith('PWR')] - df_af_final = pd.DataFrame(list(itertools.product(tech_list, - years) - ), - columns = ['TECHNOLOGY', 'YEAR'] - ) - df_af_final['TECH'] = df_af_final['TECHNOLOGY'].str[3:6] - df_af_final['VALUE'] = df_af_final['TECH'].map(af_dict) - df_af_final.dropna(inplace=True) - df_af_final['REGION'] = region_name - df_af_final = df_af_final[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] - df_af_final.to_csv(os.path.join(output_data_dir, - 'AvailabilityFactor.csv'), - index=None) - -if __name__ == "__main__": - main() - logging.info('Powerplant Data Created') \ No newline at end of file From ce91ebf80ab3cbf81dce149e8a50939ee8ff61d1 Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Tue, 24 Sep 2024 11:30:37 -0400 Subject: [PATCH 05/12] Updates to avoid future depreciation warnings --- .../osemosys_global/powerplant/activity.py | 4 +++- .../osemosys_global/powerplant/costs.py | 19 ++++++++++++------- .../osemosys_global/powerplant/data.py | 9 +++++---- .../osemosys_global/powerplant/main.py | 10 +++++----- .../powerplant/residual_capacity.py | 8 +++++--- .../powerplant/user_defined_capacity.py | 8 +++++--- 6 files changed, 35 insertions(+), 23 deletions(-) diff --git a/workflow/scripts/osemosys_global/powerplant/activity.py b/workflow/scripts/osemosys_global/powerplant/activity.py index 05be7c75..82289aa4 100644 --- a/workflow/scripts/osemosys_global/powerplant/activity.py +++ b/workflow/scripts/osemosys_global/powerplant/activity.py @@ -58,7 +58,8 @@ def activity_output_pwr(df_ratios, thermal_fuel_list_oar): df_oar = df_ratios.copy() mask = df_oar['TECHNOLOGY'].apply(lambda x: x[3:6] in thermal_fuel_list_oar) df_oar['FUEL'] = 0 - df_oar['FUEL'][mask] = 1 + df_oar.loc[mask, "FUEL"] = 1 + df_oar = df_oar.loc[~((df_oar['MODE_OF_OPERATION'] > 1) & (df_oar['FUEL'] == 0))] df_oar['FUEL'] = ('ELC' + @@ -88,6 +89,7 @@ def activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, df_iar = df_oar.copy() df_iar['FUEL'] = 0 + df_iar['FUEL'] = df_iar['FUEL'].astype(str) # Deal with GAS techs first... OCG and CCG # OCG Mode 1: Domestic GAS diff --git a/workflow/scripts/osemosys_global/powerplant/costs.py b/workflow/scripts/osemosys_global/powerplant/costs.py index cf9426dd..7b22d23c 100644 --- a/workflow/scripts/osemosys_global/powerplant/costs.py +++ b/workflow/scripts/osemosys_global/powerplant/costs.py @@ -63,7 +63,7 @@ def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, # Create formatted CSVs for each_cost in ['Capital', 'O&M']: - df_costs_temp = df_costs.loc[df_costs['parameter'].str.contains(each_cost)] + df_costs_temp = df_costs.copy().loc[df_costs.copy()['parameter'].str.contains(each_cost)] df_costs_temp.drop(['technology', 'parameter'], axis = 1, @@ -72,7 +72,8 @@ def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, 'TECHNOLOGY', 'YEAR' ]] - df_costs_final['YEAR'] = df_costs_final['YEAR'].astype(int) + df_costs_final.loc[:,'YEAR'] = df_costs_final['YEAR'].astype(int) + df_costs_final = df_costs_final.drop_duplicates() df_costs_final = (df_costs_final .loc[(df_costs_final['TECHNOLOGY'] @@ -96,17 +97,21 @@ def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, df_costs_final.drop(['technology_code', 'weo_region'], axis = 1, inplace = True) - df_costs_final = df_costs_final.fillna(-9) + df_costs_final = df_costs_final.infer_objects().fillna(-9) df_costs_final = pd.pivot_table(df_costs_final, index = ['REGION', 'YEAR'], columns = 'TECHNOLOGY', values = 'value').reset_index() df_costs_final = df_costs_final.replace([-9],[np.nan]) - df_costs_final = df_costs_final.interpolate(method = 'linear', - limit_direction='forward').round(2) - df_costs_final = df_costs_final.interpolate(method = 'linear', - limit_direction='backward').round(2) + for col in df_costs_final.columns: + if df_costs_final[col].dtype != 'object': + df_costs_final[col] = df_costs_final[col].interpolate(method = 'linear', + limit_direction='forward').round(2) + + df_costs_final[col] = df_costs_final[col].interpolate(method = 'linear', + limit_direction='backward').round(2) + df_costs_final = pd.melt(df_costs_final, id_vars = ['REGION', 'YEAR'], value_vars = [x for x in df_costs_final.columns diff --git a/workflow/scripts/osemosys_global/powerplant/data.py b/workflow/scripts/osemosys_global/powerplant/data.py index d7eb94c3..18d2fe0b 100644 --- a/workflow/scripts/osemosys_global/powerplant/data.py +++ b/workflow/scripts/osemosys_global/powerplant/data.py @@ -1,7 +1,6 @@ """Functions to extract relevent data""" import pandas as pd -import numpy as np from datetime import datetime import itertools import logging @@ -30,7 +29,7 @@ def set_generator_table(plexos_prop: pd.DataFrame, plexos_memb: pd.DataFrame, index="powerplant", columns="property", values="value", - aggfunc=np.sum, + aggfunc='sum', fill_value=0, ) df_gen["total_capacity"] = (df_gen["Max Capacity"].astype(float)) * ( @@ -59,9 +58,10 @@ def set_generator_table(plexos_prop: pd.DataFrame, plexos_memb: pd.DataFrame, ) ## Extract start year from Commission Date - df_gen_base["Commission Date"] = (pd.TimedeltaIndex(df_gen_base["Commission Date"].astype(int), + df_gen_base["Commission Date"] = (pd.to_timedelta(df_gen_base["Commission Date"].astype(int), unit='d') + datetime(1900, 1, 1)) + df_gen_base["start_year"] = df_gen_base["Commission Date"].dt.year df_gen_base.drop("Commission Date", axis=1, inplace=True) @@ -329,5 +329,6 @@ def newIar(df_in, tech, new_iar_ccg, else: logging.warning(f'Default IAR used for new {tech} power plants') iar = new_iar_default - df_out['VALUE'] = round(1/iar, 3) + df_out.loc[:,'VALUE'] = round(1/iar, 3) + return df_out \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/main.py b/workflow/scripts/osemosys_global/powerplant/main.py index 295ecfce..843d7ca3 100644 --- a/workflow/scripts/osemosys_global/powerplant/main.py +++ b/workflow/scripts/osemosys_global/powerplant/main.py @@ -104,7 +104,7 @@ def main( # return generator_table gen_table = set_generator_table(plexos_prop, plexos_memb, op_life_dict, tech_code_dict) - + # Calculate average technology efficiencies. df_eff_node, df_eff_tech = average_efficiency(gen_table, avg_csp_eff, avg_urn_eff) @@ -112,7 +112,7 @@ def main( # Calculate residual capacity. df_res_cap, custom_techs = res_capacity(gen_table, tech_list, tech_code, custom_res_cap, duplicate_techs) - + df_ratios = activity_master_start(gen_table, nodes_extra_list, duplicate_techs, mode_list) @@ -123,7 +123,7 @@ def main( df_oar_upstream, df_oar_int = activity_upstream(df_iar_base, renewables_list, thermal_fuel_list_mining) - + df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar = activity_transmission(df_oar_base, plexos_prop, interface_data) @@ -142,7 +142,7 @@ def main( df_capact_final = capact(df_oar_final) df_crossborder_final = activity_transmission_limit(cross_border_trade, df_oar_final) - + df_op_life_trn, df_op_life_out = set_op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict) @@ -191,7 +191,7 @@ def main( ) df_af_final = availability_factor(availability, tech_set) - + # OUTPUT CSV's df_res_cap.to_csv(os.path.join(output_data_dir, "ResidualCapacity.csv"), index=None) diff --git a/workflow/scripts/osemosys_global/powerplant/residual_capacity.py b/workflow/scripts/osemosys_global/powerplant/residual_capacity.py index 686df101..32671c1f 100644 --- a/workflow/scripts/osemosys_global/powerplant/residual_capacity.py +++ b/workflow/scripts/osemosys_global/powerplant/residual_capacity.py @@ -27,9 +27,8 @@ def res_capacity(df_gen_base, tech_list, df_tech_code, df_custom_res_cap, duplic ] df_res_cap = df_gen_base[res_cap_cols] - - for each_year in range(start_year, end_year+1): - df_res_cap[str(each_year)] = 0 + + df_res_cap = df_res_cap.reindex(columns=[*df_res_cap.columns.tolist(), *list(range(start_year, end_year+1))], fill_value=0) df_res_cap = pd.melt( df_res_cap, @@ -38,7 +37,10 @@ def res_capacity(df_gen_base, tech_list, df_tech_code, df_custom_res_cap, duplic var_name="model_year", value_name="value", ) + df_res_cap["model_year"] = df_res_cap["model_year"].astype(int) + df_res_cap["value"] = df_res_cap["value"].astype(float) + df_res_cap.loc[ (df_res_cap["model_year"] >= df_res_cap["start_year"]) & (df_res_cap["model_year"] <= df_res_cap["retirement_year_model"]), diff --git a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py index beefa622..b2eb8a48 100644 --- a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py +++ b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py @@ -72,7 +72,7 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, df.loc[(df['YEAR']>=df['FIRST_YEAR']) & (df['YEAR']>df['BUILD_YEAR']), 'VALUE'] = df['MAX_BUILD'] - df.fillna(0, + df.infer_objects().fillna(0, inplace=True) max_cap_techs_df = df[['REGION', 'TECHNOLOGY', @@ -93,7 +93,7 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, inplace=True) # For technologies with start year before model start year, add to ResidualCapacity - df_res_cap_ud = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] < min(years)] + df_res_cap_ud = df_min_cap_inv.copy().loc[df_min_cap_inv.copy()['YEAR'] < min(years)] df_res_cap_ud.rename(columns={'YEAR':'START_YEAR'}, inplace=True) df_res_cap_ud_final = pd.DataFrame(list(itertools.product(df_res_cap_ud['TECHNOLOGY'].unique(), @@ -122,7 +122,9 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, 'YEAR', 'VALUE']] - df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final]) + + df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final + if not df_res_cap_ud_final.empty else None]) # For technologies with start year at or after model start year, add to # TotalAnnualMinCapacityInvestment From 493c58440d9d959e52a85c9917b71937f67e794f Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:48:18 -0400 Subject: [PATCH 06/12] CP - Powerplant refactoring functional through Snakemake --- .../osemosys_global/powerplant/GEM_unused.py | 2 +- .../osemosys_global/powerplant/activity.py | 2 +- .../powerplant/availability.py | 2 +- .../osemosys_global/powerplant/constants.py | 2 +- .../osemosys_global/powerplant/costs.py | 4 ++-- .../powerplant/costs_transmission.py | 2 +- .../osemosys_global/powerplant/data.py | 2 +- .../powerplant/data_transmission.py | 2 +- .../powerplant/investment_constraints.py | 2 +- .../osemosys_global/powerplant/main.py | 24 +++++++++++++++---- .../powerplant/operational_life.py | 2 +- .../osemosys_global/powerplant/read.py | 2 -- 12 files changed, 31 insertions(+), 17 deletions(-) diff --git a/workflow/scripts/osemosys_global/powerplant/GEM_unused.py b/workflow/scripts/osemosys_global/powerplant/GEM_unused.py index d80d190e..7199282e 100644 --- a/workflow/scripts/osemosys_global/powerplant/GEM_unused.py +++ b/workflow/scripts/osemosys_global/powerplant/GEM_unused.py @@ -1,4 +1,4 @@ -"""Function to calculate residual capacity for powerplant technologies.""" +"""Function that calculates existing, planned and to be retired capacities based on GEM for coal and gas. CURRENTLY UNUSED""" import pandas as pd import os diff --git a/workflow/scripts/osemosys_global/powerplant/activity.py b/workflow/scripts/osemosys_global/powerplant/activity.py index 82289aa4..b0d1ebf6 100644 --- a/workflow/scripts/osemosys_global/powerplant/activity.py +++ b/workflow/scripts/osemosys_global/powerplant/activity.py @@ -1,4 +1,4 @@ -"""Function to calculate residual capacity for powerplant technologies.""" +"""Function to calculate activity for powerplant technologies.""" import pandas as pd import itertools diff --git a/workflow/scripts/osemosys_global/powerplant/availability.py b/workflow/scripts/osemosys_global/powerplant/availability.py index 3caba9b1..5ec66a93 100644 --- a/workflow/scripts/osemosys_global/powerplant/availability.py +++ b/workflow/scripts/osemosys_global/powerplant/availability.py @@ -1,4 +1,4 @@ -"""Function to calculate residual capacity for powerplant technologies.""" +"""Function to set availability factors.""" import pandas as pd import itertools diff --git a/workflow/scripts/osemosys_global/powerplant/constants.py b/workflow/scripts/osemosys_global/powerplant/constants.py index 0c96618c..f8ae5eb7 100644 --- a/workflow/scripts/osemosys_global/powerplant/constants.py +++ b/workflow/scripts/osemosys_global/powerplant/constants.py @@ -163,7 +163,7 @@ file_line_data = 'resources/data/Costs Line expansion.xlsx' file_default_av_factors = 'resources/data/availability_factors.csv' file_custom_res_cap = 'resources/data/custom_nodes/residual_capacity.csv' - start_year = 2020 + start_year = 2021 end_year = 2050 region_name = 'GLOBAL' custom_nodes = ["INDWE", "INDEA", "INDNE", "INDNO", "INDSO"] diff --git a/workflow/scripts/osemosys_global/powerplant/costs.py b/workflow/scripts/osemosys_global/powerplant/costs.py index 7b22d23c..284a083d 100644 --- a/workflow/scripts/osemosys_global/powerplant/costs.py +++ b/workflow/scripts/osemosys_global/powerplant/costs.py @@ -1,4 +1,4 @@ -"""Function to calculate residual capacity for powerplant technologies.""" +"""Function to calculate powerplant technology costs.""" import pandas as pd import numpy as np @@ -6,7 +6,7 @@ def costs_pwr(df_weo_data, costs_dict): - # ### Costs: Capital, fixed, and variable + # ### Costs: Capital, fixed df_costs = pd.melt(df_weo_data, id_vars = ['technology', 'weo_region', 'parameter'], diff --git a/workflow/scripts/osemosys_global/powerplant/costs_transmission.py b/workflow/scripts/osemosys_global/powerplant/costs_transmission.py index 506c19f3..1c2b7e67 100644 --- a/workflow/scripts/osemosys_global/powerplant/costs_transmission.py +++ b/workflow/scripts/osemosys_global/powerplant/costs_transmission.py @@ -1,4 +1,4 @@ -"""Function to calculate residual capacity for powerplant technologies.""" +"""Function to calculate transmission costs.""" from data_transmission import format_transmission_name diff --git a/workflow/scripts/osemosys_global/powerplant/data.py b/workflow/scripts/osemosys_global/powerplant/data.py index 18d2fe0b..6012c680 100644 --- a/workflow/scripts/osemosys_global/powerplant/data.py +++ b/workflow/scripts/osemosys_global/powerplant/data.py @@ -1,4 +1,4 @@ -"""Functions to extract relevent data""" +"""Functions to extract and format relevent data.""" import pandas as pd from datetime import datetime diff --git a/workflow/scripts/osemosys_global/powerplant/data_transmission.py b/workflow/scripts/osemosys_global/powerplant/data_transmission.py index e801904a..06e6c22f 100644 --- a/workflow/scripts/osemosys_global/powerplant/data_transmission.py +++ b/workflow/scripts/osemosys_global/powerplant/data_transmission.py @@ -1,4 +1,4 @@ -"""Functions to extract and fromat relevent data for tranmissions""" +"""Functions to extract and format relevent data for tranmission.""" def format_transmission_name(df): '''Formats PLEXOS transmission names into OSeMOSYS Global names. diff --git a/workflow/scripts/osemosys_global/powerplant/investment_constraints.py b/workflow/scripts/osemosys_global/powerplant/investment_constraints.py index 790bd2ba..36e3601d 100644 --- a/workflow/scripts/osemosys_global/powerplant/investment_constraints.py +++ b/workflow/scripts/osemosys_global/powerplant/investment_constraints.py @@ -1,4 +1,4 @@ -"""Function to calculate residual capacity for powerplant technologies.""" +"""Function to set min and max capacity investment constraints.""" import pandas as pd diff --git a/workflow/scripts/osemosys_global/powerplant/main.py b/workflow/scripts/osemosys_global/powerplant/main.py index 843d7ca3..1ce305e9 100644 --- a/workflow/scripts/osemosys_global/powerplant/main.py +++ b/workflow/scripts/osemosys_global/powerplant/main.py @@ -1,5 +1,3 @@ -"""Creates demand projections""" - import pandas as pd import os @@ -113,41 +111,55 @@ def main( df_res_cap, custom_techs = res_capacity(gen_table, tech_list, tech_code, custom_res_cap, duplicate_techs) + # Create master table for activity ratios. df_ratios = activity_master_start(gen_table, nodes_extra_list, duplicate_techs, mode_list) + # Set OutputActivitiyRatio for powerplants. df_oar, df_oar_base = activity_output_pwr(df_ratios, thermal_fuel_list_oar) + # Set InputActivityRatio for powerplants. df_iar_base = activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, df_eff_node, df_eff_tech) + # Set OutputActivitiyRatio for upstream and international technologies/fuels. df_oar_upstream, df_oar_int = activity_upstream(df_iar_base, renewables_list, thermal_fuel_list_mining) + # Set activity ratios for transmission. df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar = activity_transmission(df_oar_base, plexos_prop, interface_data) + # Combine and format activity ratios to output as csv. df_oar_final, df_iar_final = activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, df_int_trn_oar, df_iar_base, df_iar_trn, df_int_trn_iar, duplicate_techs) + # Set capital and fixed powerplant costs. df_costs = costs_pwr(weo_costs, costs_dict) + # Set capital and fixed transmission costs. df_trans_capex, df_trans_fix = get_transmission_costs(trn_line, df_oar_final) - + + # Combine and format costs data to output as csv. df_cap_cost_final, df_fix_cost_final = costs_end(weo_regions, df_costs, df_oar_final, df_trans_capex, df_trans_fix) + # Set CapacityToActivityUnit. df_capact_final = capact(df_oar_final) + # Adjust activity limits if cross border trade is not allowed following user config. df_crossborder_final = activity_transmission_limit(cross_border_trade, df_oar_final) + # Set operational life for powerplant technologies. df_op_life_trn, df_op_life_out = set_op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict) + # Set operational life for transmission. df_op_life = set_op_life_transmission(df_op_life_trn, df_op_life_out, op_life_dict) + # Set annual capacity investment constraints. df_max_cap_invest, df_min_cap_invest = cap_investment_constraints(df_iar_final, no_investment_techs) @@ -162,6 +174,7 @@ def main( # Create sets for YEAR, MODE_OF_OPERATION and REGION years_set, mode_list_set, regions_set = output_sets(mode_list) + # Alter output csv's based on user defined capacities following user config. if not tech_capacity is None: (tech_set, fuel_set, @@ -190,6 +203,7 @@ def main( df_oar_custom_val ) + # Set availability factors. df_af_final = availability_factor(availability, tech_set) # OUTPUT CSV's @@ -234,6 +248,7 @@ def main( if __name__ == "__main__": + # SET INPUT DATA plexos_prop = import_plexos_2015(file_plexos, "prop") plexos_memb = import_plexos_2015(file_plexos, "memb") op_life = import_op_life(file_default_op_life) @@ -270,5 +285,6 @@ def main( "custom_res_cap" : custom_res_cap, "default_av_factors": availability, } - + + # CALL MAIN main(**input_data) \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/operational_life.py b/workflow/scripts/osemosys_global/powerplant/operational_life.py index d2075007..165255fa 100644 --- a/workflow/scripts/osemosys_global/powerplant/operational_life.py +++ b/workflow/scripts/osemosys_global/powerplant/operational_life.py @@ -1,4 +1,4 @@ -"""Function to calculate residual capacity for powerplant technologies.""" +"""Functions to set operational life.""" import pandas as pd diff --git a/workflow/scripts/osemosys_global/powerplant/read.py b/workflow/scripts/osemosys_global/powerplant/read.py index deea5542..9f9876a5 100644 --- a/workflow/scripts/osemosys_global/powerplant/read.py +++ b/workflow/scripts/osemosys_global/powerplant/read.py @@ -2,8 +2,6 @@ import pandas as pd -from constants import custom_nodes - def import_plexos_2015(f: str, metric: str) -> dict[str, pd.DataFrame]: """Imports PLEXOS-World 2015 model file. From 58e2241dd7de0c400321e2af6639524552213126 Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:48:42 -0400 Subject: [PATCH 07/12] CP: Transmission functional as standalone set of scripts, snakemake integration needs work --- config/config.yaml | 11 +- workflow/rules/preprocess.smk | 69 ++++- .../osemosys_global/powerplant/activity.py | 195 +++++++------- .../powerplant/availability.py | 10 +- .../osemosys_global/powerplant/constants.py | 81 ++---- .../osemosys_global/powerplant/costs.py | 21 +- .../osemosys_global/powerplant/data.py | 78 ++---- .../powerplant/investment_constraints.py | 12 +- .../osemosys_global/powerplant/main.py | 247 +++++++++--------- .../powerplant/operational_life.py | 20 +- .../osemosys_global/powerplant/read.py | 28 -- .../powerplant/residual_capacity.py | 75 ++++-- .../osemosys_global/powerplant/sets.py | 12 +- .../powerplant/user_defined_capacity.py | 136 +++++----- .../activity.py} | 81 +++++- .../osemosys_global/transmission/constants.py | 19 ++ .../costs.py} | 21 +- .../data.py} | 3 + .../transmission/investment_constraints.py | 34 +++ .../osemosys_global/transmission/main.py | 247 ++++++++++++++++++ .../transmission/operational_life.py | 30 +++ .../osemosys_global/transmission/read.py | 108 ++++++++ .../osemosys_global/transmission/sets.py | 11 + .../transmission/user_defined_capacity.py | 244 +++++++++++++++++ .../osemosys_global/transmission/utils.py | 21 ++ 25 files changed, 1261 insertions(+), 553 deletions(-) rename workflow/scripts/osemosys_global/{powerplant/activity_transmission.py => transmission/activity.py} (71%) create mode 100644 workflow/scripts/osemosys_global/transmission/constants.py rename workflow/scripts/osemosys_global/{powerplant/costs_transmission.py => transmission/costs.py} (78%) rename workflow/scripts/osemosys_global/{powerplant/data_transmission.py => transmission/data.py} (95%) create mode 100644 workflow/scripts/osemosys_global/transmission/investment_constraints.py create mode 100644 workflow/scripts/osemosys_global/transmission/main.py create mode 100644 workflow/scripts/osemosys_global/transmission/operational_life.py create mode 100644 workflow/scripts/osemosys_global/transmission/read.py create mode 100644 workflow/scripts/osemosys_global/transmission/sets.py create mode 100644 workflow/scripts/osemosys_global/transmission/user_defined_capacity.py create mode 100644 workflow/scripts/osemosys_global/transmission/utils.py diff --git a/config/config.yaml b/config/config.yaml index a77007bb..fbf8fe92 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -55,7 +55,16 @@ user_defined_capacity: # first_year_of_expansion, # build_rate_per_year, # cost] - TRNINDEAINDNE: [0, 2030, "open", 2030, 10, 861] + PWRCOAINDWE01: [8, 2000, "open", 2025, 5, 1100] + +user_defined_capacity_transmission: + # technology: [capacity, + # first_year, + # "fixed/open", + # first_year_of_expansion, + # build_rate_per_year, + # cost] + TRNINDEAINDNE: [5, 1975, "open", 2030, 10, 861] nodes_to_add: #- "AAAXX" where AAA is a 3-letter country code, diff --git a/workflow/rules/preprocess.smk b/workflow/rules/preprocess.smk index a064bf78..65b2ec60 100644 --- a/workflow/rules/preprocess.smk +++ b/workflow/rules/preprocess.smk @@ -20,6 +20,24 @@ demand_figures = [ # output script files power_plant_files = [ + 'powerplant/CapitalCost.csv', + 'powerplant/FixedCost.csv', + 'powerplant/CapacityToActivityUnit.csv', + 'powerplant/OperationalLife.csv', + 'powerplant/TotalAnnualMaxCapacityInvestment.csv', + 'powerplant/TotalAnnualMinCapacityInvestment.csv', + 'FUEL.csv', + 'powerplant/InputActivityRatio.csv', + 'powerplant/OutputActivityRatio.csv', + 'MODE_OF_OPERATION.csv', + 'REGION.csv', + 'powerplant/ResidualCapacity.csv', + 'powerplant/TECHNOLOGY.csv', + 'YEAR.csv', + 'AvailabilityFactor.csv' + ] + +transmission_files = [ 'CapitalCost.csv', 'FixedCost.csv', 'CapacityToActivityUnit.csv', @@ -27,15 +45,10 @@ power_plant_files = [ 'TotalAnnualMaxCapacityInvestment.csv', 'TotalAnnualMinCapacityInvestment.csv', 'TotalTechnologyModelPeriodActivityUpperLimit.csv', - 'FUEL.csv', 'InputActivityRatio.csv', 'OutputActivityRatio.csv', - 'MODE_OF_OPERATION.csv', - 'REGION.csv', 'ResidualCapacity.csv', 'TECHNOLOGY.csv', - 'YEAR.csv', - 'AvailabilityFactor.csv' ] timeslice_files = [ @@ -88,7 +101,8 @@ user_capacity_files = [ check_files = os.listdir('resources/otoole/data') generated_files = [ - power_plant_files, + power_plant_files, + transmission_files, timeslice_files, variable_cost_files, demand_files, @@ -96,7 +110,7 @@ generated_files = [ max_capacity_files] for file_list in generated_files: [check_files.remove(csv) for csv in file_list] - + # rules rule make_data_dir: @@ -123,10 +137,16 @@ rule powerplant: default_av_factors = 'resources/data/availability_factors.csv', custom_res_cap = powerplant_cap_custom_csv() params: - trade = config['crossborderTrade'], start_year = config['startYear'], end_year = config['endYear'], - invest_techs = config['no_invest_technologies'] + region_name = 'GLOBAL', + custom_nodes = config['nodes_to_add'], + user_defined_capacity = config['user_defined_capacity'], + no_investment_techs = config['no_invest_technologies'], + output_data_dir = 'results/data', + input_data_dir = 'resources/data', + powerplant_data_dir = 'results/data/powerplant', + output: csv_files = expand('results/data/{output_file}', output_file = power_plant_files) log: @@ -134,6 +154,32 @@ rule powerplant: script: "../scripts/osemosys_global/powerplant/main.py" +rule transmission: + message: + "Generating transmission data..." + input: + rules.powerplant.output.csv_files, + plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx', + default_op_life = 'resources/data/operational_life.csv', + line_data = 'resources/data/Costs Line expansion.xlsx', + params: + trade = config['crossborderTrade'], + start_year = config['startYear'], + end_year = config['endYear'], + region_name = 'GLOBAL', + custom_nodes = config['nodes_to_add'], + user_defined_capacity_transmission = config['user_defined_capacity_transmission'], + no_investment_techs = config['no_invest_technologies'], + output_data_dir = 'results/data', + input_data_dir = 'resources/data', + powerplant_data_dir = 'results/data/powerplant', + output: + csv_files = expand('results/data/{output_file}', output_file = transmission_files) + log: + log = 'results/logs/transmission.log' + script: + "../scripts/osemosys_global/transmission/main.py" + rule timeslice: message: 'Generating timeslice data...' @@ -260,6 +306,7 @@ rule file_check: 'Generating missing files...' input: rules.powerplant.output.csv_files, + rules.transmission.output.csv_files, rules.timeslice.output.csv_files, rules.variable_costs.output.csv_files, rules.demand_projections.output.csv_files, @@ -271,6 +318,4 @@ rule file_check: log: log = 'results/logs/file_check.log' shell: - 'python workflow/scripts/osemosys_global/file_check.py 2> {log}' - - + 'python workflow/scripts/osemosys_global/file_check.py 2> {log}' \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/activity.py b/workflow/scripts/osemosys_global/powerplant/activity.py index b0d1ebf6..b57a210e 100644 --- a/workflow/scripts/osemosys_global/powerplant/activity.py +++ b/workflow/scripts/osemosys_global/powerplant/activity.py @@ -4,24 +4,27 @@ import itertools from constants import ( - region_name, - custom_nodes, - years, - new_iar_ccg, - new_iar_ocg, - new_iar_coa, - new_iar_default, + NODES_EXTRA_LIST, + NEW_IAR_CCG, + NEW_IAR_OCG, + NEW_IAR_COA, + NEW_IAR_DEFAULT, + THERMAL_FUEL_LIST_OAR, + THERMAL_FUEL_LIST_IAR, + THERMAL_FUEL_LIST_MINING, ) from data import( - createPwrTechs, - duplicatePlexosTechs, - newIar + create_pwr_techs, + duplicate_plexos_techs, + new_iar, + get_years ) from utils import apply_dtypes -def activity_master_start(df_gen_base, nodes_extra_list, duplicate_techs, mode_list): +def activity_master_start(df_gen_base, duplicate_techs, mode_list, + custom_nodes, start_year, end_year): # ### Add input and output activity ratios @@ -31,7 +34,7 @@ def activity_master_start(df_gen_base, nodes_extra_list, duplicate_techs, mode_l else: node_list = list(df_gen_base['node_code'].unique()) # Add extra nodes which are not present in 2015 but will be by 2050 - for each_node in nodes_extra_list: + for each_node in NODES_EXTRA_LIST: if len(each_node) <= 6: node_list.append("".join(each_node.split('-')[1:]) + 'XX') else: @@ -43,141 +46,141 @@ def activity_master_start(df_gen_base, nodes_extra_list, duplicate_techs, mode_l df_ratios = pd.DataFrame(list(itertools.product(node_list, master_fuel_list, mode_list, - years) + get_years(start_year, end_year)) ), columns = ['node_code', 'tech_code', 'MODE_OF_OPERATION', 'YEAR'] ) - df_ratios = createPwrTechs(df_ratios, duplicate_techs) + df_ratios = create_pwr_techs(df_ratios, duplicate_techs) return df_ratios -def activity_output_pwr(df_ratios, thermal_fuel_list_oar): +def activity_output_pwr(df_ratios, region_name): # #### OutputActivityRatio - Power Generation Technologies - df_oar = df_ratios.copy() - mask = df_oar['TECHNOLOGY'].apply(lambda x: x[3:6] in thermal_fuel_list_oar) - df_oar['FUEL'] = 0 - df_oar.loc[mask, "FUEL"] = 1 + df_pwr_oar_base = df_ratios.copy() + mask = df_pwr_oar_base['TECHNOLOGY'].apply(lambda x: x[3:6] in THERMAL_FUEL_LIST_OAR) + df_pwr_oar_base['FUEL'] = 0 + df_pwr_oar_base.loc[mask, "FUEL"] = 1 - df_oar = df_oar.loc[~((df_oar['MODE_OF_OPERATION'] > 1) & - (df_oar['FUEL'] == 0))] - df_oar['FUEL'] = ('ELC' + - df_oar['TECHNOLOGY'].str[6:11] + + df_pwr_oar_base = df_pwr_oar_base.loc[~((df_pwr_oar_base['MODE_OF_OPERATION'] > 1) & + (df_pwr_oar_base['FUEL'] == 0))] + df_pwr_oar_base['FUEL'] = ('ELC' + + df_pwr_oar_base['TECHNOLOGY'].str[6:11] + '01' ) - df_oar['VALUE'] = 1 + df_pwr_oar_base['VALUE'] = 1 # Add 'REGION' column and fill 'GLOBAL' throughout - df_oar['REGION'] = region_name + df_pwr_oar_base['REGION'] = region_name # Select columns for final output table - df_oar_base = df_oar[['REGION', + df_pwr_oar_final = df_pwr_oar_base[['REGION', 'TECHNOLOGY', 'FUEL', 'MODE_OF_OPERATION', 'YEAR', 'VALUE',]] - return df_oar, df_oar_base + # Copy OAR table with all columns to IAR + df_pwr_iar_base = df_pwr_oar_base.copy() + + return df_pwr_oar_final, df_pwr_iar_base -def activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, - df_eff_node, df_eff_tech): +def activity_input_pwr(df_pwr_iar_base, renewables_list, df_eff_node, + df_eff_tech, region_name): # #### InputActivityRatio - Power Generation Technologies - # Copy OAR table with all columns to IAR - df_iar = df_oar.copy() - - df_iar['FUEL'] = 0 - df_iar['FUEL'] = df_iar['FUEL'].astype(str) + df_pwr_iar_base['FUEL'] = 0 + df_pwr_iar_base['FUEL'] = df_pwr_iar_base['FUEL'].astype(str) # Deal with GAS techs first... OCG and CCG # OCG Mode 1: Domestic GAS - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['OCG'])), - 'FUEL'] = 'GAS'+df_iar['TECHNOLOGY'].str[6:9] + df_pwr_iar_base.loc[(df_pwr_iar_base['MODE_OF_OPERATION'] == 1) & + (df_pwr_iar_base['TECHNOLOGY'].str[3:6].isin(['OCG'])), + 'FUEL'] = 'GAS'+df_pwr_iar_base['TECHNOLOGY'].str[6:9] # OCG Mode 2: International GAS - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['OCG'])), + df_pwr_iar_base.loc[(df_pwr_iar_base['MODE_OF_OPERATION'] == 2) & + (df_pwr_iar_base['TECHNOLOGY'].str[3:6].isin(['OCG'])), 'FUEL'] = 'GASINT' # CCG Mode 1: Domestic GAS - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['CCG'])), - 'FUEL'] = 'GAS'+df_iar['TECHNOLOGY'].str[6:9] + df_pwr_iar_base.loc[(df_pwr_iar_base['MODE_OF_OPERATION'] == 1) & + (df_pwr_iar_base['TECHNOLOGY'].str[3:6].isin(['CCG'])), + 'FUEL'] = 'GAS'+df_pwr_iar_base['TECHNOLOGY'].str[6:9] # CCG Mode 2: International GAS - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['CCG'])), + df_pwr_iar_base.loc[(df_pwr_iar_base['MODE_OF_OPERATION'] == 2) & + (df_pwr_iar_base['TECHNOLOGY'].str[3:6].isin(['CCG'])), 'FUEL'] = 'GASINT' # CCS Mode 1: Domestic COA - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['CCS'])), - 'FUEL'] = 'COA'+df_iar['TECHNOLOGY'].str[6:9] + df_pwr_iar_base.loc[(df_pwr_iar_base['MODE_OF_OPERATION'] == 1) & + (df_pwr_iar_base['TECHNOLOGY'].str[3:6].isin(['CCS'])), + 'FUEL'] = 'COA'+df_pwr_iar_base['TECHNOLOGY'].str[6:9] # CCS Mode 2: International COA - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & - (df_iar['TECHNOLOGY'].str[3:6].isin(['CCS'])), + df_pwr_iar_base.loc[(df_pwr_iar_base['MODE_OF_OPERATION'] == 2) & + (df_pwr_iar_base['TECHNOLOGY'].str[3:6].isin(['CCS'])), 'FUEL'] = 'COAINT' # For non-GAS thermal fuels, domestic fuel input by country in mode 1 and # 'international' fuel input in mode 2 - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(thermal_fuel_list_iar)), - 'FUEL'] = df_iar['TECHNOLOGY'].str[3:9] + df_pwr_iar_base.loc[(df_pwr_iar_base['MODE_OF_OPERATION'] == 1) & + (df_pwr_iar_base['TECHNOLOGY'].str[3:6].isin(THERMAL_FUEL_LIST_IAR)), + 'FUEL'] = df_pwr_iar_base['TECHNOLOGY'].str[3:9] - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 2) & - (df_iar['TECHNOLOGY'].str[3:6].isin(thermal_fuel_list_iar)), - 'FUEL'] = df_iar['TECHNOLOGY'].str[3:6] + 'INT' + df_pwr_iar_base.loc[(df_pwr_iar_base['MODE_OF_OPERATION'] == 2) & + (df_pwr_iar_base['TECHNOLOGY'].str[3:6].isin(THERMAL_FUEL_LIST_IAR)), + 'FUEL'] = df_pwr_iar_base['TECHNOLOGY'].str[3:6] + 'INT' # For renewable fuels, input by node in mode 1 - df_iar.loc[(df_iar['MODE_OF_OPERATION'] == 1) & - (df_iar['TECHNOLOGY'].str[3:6].isin(renewables_list)), - 'FUEL'] = df_iar['TECHNOLOGY'].str[3:11] + df_pwr_iar_base.loc[(df_pwr_iar_base['MODE_OF_OPERATION'] == 1) & + (df_pwr_iar_base['TECHNOLOGY'].str[3:6].isin(renewables_list)), + 'FUEL'] = df_pwr_iar_base['TECHNOLOGY'].str[3:11] # Remove mode 2 when not used - df_iar = df_iar.loc[df_iar['FUEL'] != 0] + df_pwr_iar_base = df_pwr_iar_base.loc[df_pwr_iar_base['FUEL'] != 0] # Join efficiency columns: one with node and technology average, and the # other with technology average - df_iar = df_iar.join(df_eff_node.set_index(['tech_code', 'node_code']), + df_pwr_iar_base = df_pwr_iar_base.join(df_eff_node.set_index(['tech_code', 'node_code']), on=['tech_code', 'node_code']) - df_iar = df_iar.join(df_eff_tech.set_index('tech_code'), + df_pwr_iar_base = df_pwr_iar_base.join(df_eff_tech.set_index('tech_code'), on='tech_code') # When available, choose node and technology average. Else, # choose technology average - df_iar['VALUE'] = df_iar['node_average_iar'] + df_pwr_iar_base['VALUE'] = df_pwr_iar_base['node_average_iar'] - df_iar.loc[df_iar['TECHNOLOGY'].str.startswith('PWRCCS'), + df_pwr_iar_base.loc[df_pwr_iar_base['TECHNOLOGY'].str.startswith('PWRCCS'), 'tech_average_iar'] = 3 - df_iar.loc[df_iar['VALUE'].isna(), - 'VALUE'] = df_iar['tech_average_iar'] - df_iar.drop_duplicates(inplace=True) + df_pwr_iar_base.loc[df_pwr_iar_base['VALUE'].isna(), + 'VALUE'] = df_pwr_iar_base['tech_average_iar'] + df_pwr_iar_base.drop_duplicates(inplace=True) # Add 'REGION' column and fill 'GLOBAL' throughout - df_iar['REGION'] = region_name + df_pwr_iar_base['REGION'] = region_name # Select columns for final output table - df_iar_base = df_iar[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] + df_pwr_iar_final = df_pwr_iar_base[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] - return df_iar_base + return df_pwr_iar_final -def activity_upstream(df_iar_base, renewables_list, thermal_fuels_list_mining): +def activity_upstream(df_pwr_iar_final, renewables_list): # #### OutputActivityRatios - Upstream # We have to create a technology to produce every fuel that is input into any of the power technologies: - df_oar_upstream = df_iar_base.copy() + df_oar_upstream = df_pwr_iar_final.copy() # All mining and resource technologies have an OAR of 1... df_oar_upstream['VALUE'] = 1 @@ -187,7 +190,7 @@ def activity_upstream(df_iar_base, renewables_list, thermal_fuels_list_mining): 'TECHNOLOGY'] = 'RNW'+df_oar_upstream['FUEL'] # If the fuel is a thermal fuel, we need to create the OAR for the mining technology... BUT NOT FOR THE INT FUELS... - df_oar_upstream.loc[df_oar_upstream['FUEL'].str[0:3].isin(thermal_fuels_list_mining) & + df_oar_upstream.loc[df_oar_upstream['FUEL'].str[0:3].isin(THERMAL_FUEL_LIST_MINING) & ~(df_oar_upstream['FUEL'].str[3:6] == "INT"), 'TECHNOLOGY'] = 'MIN'+df_oar_upstream['FUEL'] @@ -220,9 +223,8 @@ def activity_upstream(df_iar_base, renewables_list, thermal_fuels_list_mining): return df_oar_upstream, df_oar_int -def activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, - df_int_trn_oar, df_iar_final, df_iar_trn, df_int_trn_iar, - duplicate_techs): +def activity_master_end(df_pwr_oar_final, df_oar_upstream, df_oar_int, + df_pwr_iar_final, duplicate_techs): # #### Output IAR and OAR @@ -230,11 +232,9 @@ def activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, df_oar_final = pd.concat( [ - df_oar_base, # If we want to split transmission to different script might have to not used df_oar_final as input to this function. + df_pwr_oar_final, df_oar_upstream, df_oar_int, - df_oar_trn, - df_int_trn_oar, ] ).dropna() @@ -246,16 +246,8 @@ def activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, 'YEAR', 'VALUE',]] - df_iar_final = pd.concat( - [ - df_iar_final, - df_iar_trn, - df_int_trn_iar, - ] - ).dropna() - # Select columns for final output table - df_iar_final = df_iar_final[['REGION', + df_iar_final = df_pwr_iar_final[['REGION', 'TECHNOLOGY', 'FUEL', 'MODE_OF_OPERATION', @@ -263,14 +255,14 @@ def activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, df_oar_trn, 'VALUE']] # Add iar for techs not using PLEXOS values - df_iar_newTechs = duplicatePlexosTechs(df_iar_final, duplicate_techs) + df_iar_newTechs = duplicate_plexos_techs(df_iar_final, duplicate_techs) for duplicate_tech in duplicate_techs: - df_new_iar = newIar(df_iar_newTechs, duplicate_tech,new_iar_ccg, - new_iar_ocg, new_iar_coa, new_iar_default) + df_new_iar = new_iar(df_iar_newTechs, duplicate_tech,NEW_IAR_CCG, + NEW_IAR_OCG, NEW_IAR_COA, NEW_IAR_DEFAULT) df_iar_final = pd.concat([df_iar_final, df_new_iar]) # Add oar for techs not using PLEXOS values - df_oar_newTechs = duplicatePlexosTechs(df_oar_final, duplicate_techs) + df_oar_newTechs = duplicate_plexos_techs(df_oar_final, duplicate_techs) df_oar_final = pd.concat([df_oar_final, df_oar_newTechs]) df_oar_final.drop_duplicates(subset=['REGION', @@ -292,15 +284,8 @@ def capact(df_oar_final): 'TECHNOLOGY' ]] df_capact_final = df_capact_final.drop_duplicates() - df_capact_final = (df_capact_final - .loc[(df_capact_final['TECHNOLOGY'] - .str.startswith('PWR') - ) | - (df_capact_final['TECHNOLOGY'] - .str.contains('TRN') - ) - ] - ) + df_capact_final = df_capact_final.loc[df_capact_final['TECHNOLOGY' + ].str.startswith('PWR')] df_capact_final['VALUE'] = 31.536 df_capact_final.drop_duplicates(inplace=True) diff --git a/workflow/scripts/osemosys_global/powerplant/availability.py b/workflow/scripts/osemosys_global/powerplant/availability.py index 5ec66a93..887a7dc1 100644 --- a/workflow/scripts/osemosys_global/powerplant/availability.py +++ b/workflow/scripts/osemosys_global/powerplant/availability.py @@ -3,12 +3,10 @@ import pandas as pd import itertools -from constants import( - years, - region_name - ) +from data import get_years -def availability_factor(availability, tech_set_base): +def availability_factor(availability, tech_set_base, + start_year, end_year, region_name): af_dict = dict(zip(list(availability['technology']), list(availability['value']))) @@ -16,7 +14,7 @@ def availability_factor(availability, tech_set_base): tech_list = [x for x in tech_set_base['VALUE'] if x.startswith('PWR')] df_af_final = pd.DataFrame(list(itertools.product(tech_list, - years) + get_years(start_year, end_year)) ), columns = ['TECHNOLOGY', 'YEAR'] ) diff --git a/workflow/scripts/osemosys_global/powerplant/constants.py b/workflow/scripts/osemosys_global/powerplant/constants.py index f8ae5eb7..82769a23 100644 --- a/workflow/scripts/osemosys_global/powerplant/constants.py +++ b/workflow/scripts/osemosys_global/powerplant/constants.py @@ -2,21 +2,21 @@ """Change IAR for CSP value taken from PLEXOS to 1.0. Relevant for the 'average_efficiency' function.""" -avg_csp_eff = 1 +AVG_CSP_EFF = 1 """Change IAR for URN value taken from PLEXOS to 2.2 (45%). Relevant for the 'average_efficiency' function.""" -avg_urn_eff = 0.45 +AVG_URN_EFF = 0.45 """Technologies that will have 00 and 01 suffixes to represent PLEXOS historical values and future values. Relevant for the 'residual_capacity' and activity functions.""" -duplicate_techs = ['CCG', 'OCG'] +DUPLICATE_TECHS = ['CCG', 'OCG'] """Add extra nodes which exist in 2050 but are not in the 2015 PLEXOS-World data to enable their addition to the workflow. Relevant for the 'generator_table' and activity functions""" -nodes_extra_list = ['AF-SOM', +NODES_EXTRA_LIST = ['AF-SOM', 'AF-TCD', 'AS-TLS', 'EU-MLT', @@ -29,11 +29,11 @@ """Sets the mode of operations for technologies. Relevant for the activity_master_start function.""" -mode_list = [1,2] +MODE_LIST = [1,2] """List of technologies (thermal) for which output activity ratios need to be developed. Relevant for the 'activity_output_pwr' function.""" -thermal_fuel_list_oar = ['COA', +THERMAL_FUEL_LIST_OAR = ['COA', 'COG', 'OCG', 'CCG', @@ -46,7 +46,7 @@ """List of technologies (thermal) for which input activity ratios need to be developed. Relevant for the 'activity_input_pwr' function.""" -thermal_fuel_list_iar = ['COA', +THERMAL_FUEL_LIST_IAR = ['COA', 'COG', 'PET', 'URN', @@ -56,7 +56,7 @@ """List of upstream fuels for which output activity ratios need to be developed. Relevant for the 'activity_upstream' function.""" -thermal_fuel_list_mining = ['COA', +THERMAL_FUEL_LIST_MINING = ['COA', 'COG', 'GAS', 'PET', @@ -68,7 +68,7 @@ """List of technologies (renewable) for which activity ratios need to be developed. Relevant for the 'activity_input_pwr' and 'activity_upstream' functions.""" -renewables_list = ['BIO', +RENEWABLES_LIST = ['BIO', 'GEO', 'HYD', 'SPV', @@ -80,7 +80,7 @@ """Technology input for the mapping of WEO cost data to OG technologies. Relevant for the 'costs_pwr' function.""" -costs_dict = {'Biomass - waste incineration - CHP':'WAS', +COSTS_DICT = {'Biomass - waste incineration - CHP':'WAS', 'Biomass Power plant':'BIO', 'CCGT':'CCG', 'CCGT - CHP':'COG', @@ -104,79 +104,36 @@ """setings for custom iar values deviating from derived values from the PLEXOS-World dataset. Relevant for the 'newIar' function.""" -new_iar_ccg = 0.5 -new_iar_ocg = 0.35 -new_iar_coa = 0.33 -new_iar_default = 1 +NEW_IAR_CCG = 0.5 +NEW_IAR_OCG = 0.35 +NEW_IAR_COA = 0.33 +NEW_IAR_DEFAULT = 1 """Set iar and oar values for custom transmission entries. I.e. oar of 0.9 assumes 10% losses. Relevant for the user_defined_capacity function.""" -df_iar_custom_val = 1 -df_oar_custom_val = 0.9 +DF_IAR_CUSTOM_VAL = 1 +DF_OAR_CUSTOM_VAL = 0.9 """Set column name dictionaries for different Global Energy Monitor (gem) input datasets""" -gem_coal_col = {'Country' : 'Country', 'Capacity (MW)' : 'VALUE', +GEM_COAL_COL = {'Country' : 'Country', 'Capacity (MW)' : 'VALUE', 'Status' : 'Status', 'Year' : 'Year_built', 'RETIRED' : 'Year_retired', 'Planned Retire' : 'Year_retired_planned', 'Latitude' : 'Latitude', 'Longitude' : 'Longitude'} -gem_gas_col = {'Country' : 'Country', 'Capacity elec. (MW)' : 'VALUE', +GEM_GAS_COL = {'Country' : 'Country', 'Capacity elec. (MW)' : 'VALUE', 'Status' : 'Status', 'Start year' : 'Year_built', 'Retired year' : 'Year_retired', 'Planned retire' : 'Year_retired_planned', 'Latitude' : 'Latitude', 'Longitude' : 'Longitude', 'Technology' : 'Technology'} """Set technology dictionary to match with OSeMOSYS global technologies""" -gem_gas_dict = {'CC' : 'CCG', +GEM_GAS_DICT = {'CC' : 'CCG', 'GT' : 'OCG', 'ICCC' : 'CCG', 'ISCC' : 'CCG', 'ST' : 'OCG', 'AFC' : 'CCG'} -if "snakemake" in globals(): - file_plexos = snakemake.input.plexos - file_default_op_life = snakemake.input.default_op_life - file_naming_convention_tech = snakemake.input.naming_convention_tech - file_weo_costs = snakemake.input.weo_costs - file_weo_regions = snakemake.input.weo_regions - file_line_data = snakemake.input.line_data - file_default_av_factors = snakemake.input.default_av_factors - file_custom_res_cap = snakemake.input.custom_res_cap - start_year = snakemake.params.start_year - end_year = snakemake.params.end_year - region_name = snakemake.params.region_name - custom_nodes = snakemake.params.custom_nodes - custom_nodes_data = snakemake.input.custom_nodes - tech_capacity = snakemake.params.user_defined_capacity - cross_border_trade = snakemake.params.crossborderTrade - no_investment_techs = snakemake.params.no_invest_technologies - output_data_dir = snakemake.params.output_data_dir - input_data_dir = snakemake.params.output_data_dir - -else: - file_plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx' - file_default_op_life = 'resources/data/operational_life.csv' - file_naming_convention_tech = 'resources/data/naming_convention_tech.csv' - file_weo_costs = 'resources/data/weo_2020_powerplant_costs.csv' - file_weo_regions = 'resources/data/weo_region_mapping.csv' - file_line_data = 'resources/data/Costs Line expansion.xlsx' - file_default_av_factors = 'resources/data/availability_factors.csv' - file_custom_res_cap = 'resources/data/custom_nodes/residual_capacity.csv' - start_year = 2021 - end_year = 2050 - region_name = 'GLOBAL' - custom_nodes = ["INDWE", "INDEA", "INDNE", "INDNO", "INDSO"] - custom_nodes_data = 'resources/data/custom_nodes/specified_annual_demand.csv' - tech_capacity = {'TRNINDEAINDNE': [0, 2030, "open", 2030, 10, 861]} - cross_border_trade = True - no_investment_techs = ["CSP", "WAV", "URN", "OTH", "WAS", - "COG", "GEO", "BIO", "PET"] - output_data_dir = 'results/data' - input_data_dir = 'resources/data' - -years = list(range(start_year, end_year + 1)) - SET_DTYPES = { "DAILYTIMEBRACKET": int, "EMISSION":str, diff --git a/workflow/scripts/osemosys_global/powerplant/costs.py b/workflow/scripts/osemosys_global/powerplant/costs.py index 284a083d..98121cd4 100644 --- a/workflow/scripts/osemosys_global/powerplant/costs.py +++ b/workflow/scripts/osemosys_global/powerplant/costs.py @@ -2,9 +2,13 @@ import pandas as pd import numpy as np + +from constants import COSTS_DICT + from utils import apply_dtypes -def costs_pwr(df_weo_data, costs_dict): + +def costs_pwr(df_weo_data): # ### Costs: Capital, fixed @@ -45,13 +49,12 @@ def costs_pwr(df_weo_data, costs_dict): ) df_costs['YEAR'] = df_costs['YEAR'].astype(int) - df_costs = df_costs.loc[df_costs['technology'].isin(costs_dict.keys())] - df_costs['technology_code'] = df_costs['technology'].replace(costs_dict) + df_costs = df_costs.loc[df_costs['technology'].isin(COSTS_DICT.keys())] + df_costs['technology_code'] = df_costs['technology'].replace(COSTS_DICT) return df_costs -def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, - df_trans_fix): +def costs_end(df_weo_regions, df_costs, df_oar_final): weo_regions_dict = dict([(k, v) for k, v @@ -124,12 +127,12 @@ def costs_end(df_weo_regions, df_costs, df_oar_final, df_trans_capex, df_costs_final = df_costs_final[~df_costs_final['VALUE'].isnull()] if each_cost in ['Capital']: - df_costs_final_capital = df_costs_final.merge(df_trans_capex, how='outer') - df_costs_final_capital = apply_dtypes(df_costs_final_capital, "CapitalCost") + #df_costs_final_capital = df_costs_final.merge(df_trans_capex, how='outer') + df_costs_final_capital = apply_dtypes(df_costs_final, "CapitalCost") if each_cost in ['O&M']: - df_costs_final_fixed = df_costs_final.merge(df_trans_fix, how='outer') - df_costs_final_fixed = apply_dtypes(df_costs_final_fixed, "FixedCost") + #df_costs_final_fixed = df_costs_final.merge(df_trans_fix, how='outer') + df_costs_final_fixed = apply_dtypes(df_costs_final, "FixedCost") return df_costs_final_capital, df_costs_final_fixed \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/data.py b/workflow/scripts/osemosys_global/powerplant/data.py index 6012c680..aefe158a 100644 --- a/workflow/scripts/osemosys_global/powerplant/data.py +++ b/workflow/scripts/osemosys_global/powerplant/data.py @@ -2,19 +2,20 @@ import pandas as pd from datetime import datetime -import itertools import logging from constants import ( - nodes_extra_list, - start_year, - region_name, - custom_nodes, - years + NODES_EXTRA_LIST, + AVG_CSP_EFF, + AVG_URN_EFF ) +def get_years(start: int, end: int) -> range: + return range(start, end + 1) + def set_generator_table(plexos_prop: pd.DataFrame, plexos_memb: pd.DataFrame, - op_life_dict: dict[str, int], tech_code_dict: dict[str, str]) -> pd.DataFrame: + op_life_dict: dict[str, int], tech_code_dict: dict[str, str], + start_year: int, end_year: int) -> pd.DataFrame: """Sets the main generator table derived from the PLEXOS-World model. """ @@ -105,7 +106,7 @@ def set_generator_table(plexos_prop: pd.DataFrame, plexos_memb: pd.DataFrame, nodes_extra_df = pd.DataFrame(columns=['node']) - nodes_extra_df['node'] = nodes_extra_list + nodes_extra_df['node'] = NODES_EXTRA_LIST df_gen_agg_node = pd.concat( [df_gen_agg_node,nodes_extra_df], @@ -155,7 +156,7 @@ def set_generator_table(plexos_prop: pd.DataFrame, plexos_memb: pd.DataFrame, return df_gen_base -def average_efficiency(df_gen_base, avg_csp_eff, avg_urn_eff): +def average_efficiency(df_gen_base): # ### Calculate average InputActivityRatio by node+technology and only by technology df_eff = df_gen_base[['node_code', @@ -163,10 +164,10 @@ def average_efficiency(df_gen_base, avg_csp_eff, avg_urn_eff): 'tech_code']] # Change IAR for CSP value taken from PLEXOS - df_eff.loc[df_eff['tech_code']=='CSP', 'efficiency'] = avg_csp_eff + df_eff.loc[df_eff['tech_code']=='CSP', 'efficiency'] = AVG_CSP_EFF # Change IAR for URN value taken from PLEXOS - df_eff.loc[df_eff['tech_code']=='URN', 'efficiency'] = avg_urn_eff + df_eff.loc[df_eff['tech_code']=='URN', 'efficiency'] = AVG_URN_EFF # Average efficiency by node and technology df_eff_node = df_eff.groupby(['tech_code', @@ -192,7 +193,7 @@ def average_efficiency(df_gen_base, avg_csp_eff, avg_urn_eff): return df_eff_node, df_eff_tech -def createPwrTechs(df_in, techs): +def create_pwr_techs(df_in, techs): """Formats power generation technology name Adds a 'TECHNOLOGY' column to a dataframe with formatted power @@ -226,56 +227,7 @@ def createPwrTechs(df_in, techs): df_out = df_out.drop('tech_suffix', axis = 1) return df_out -def custom_nodes_csv(df_custom, tech_list): - '''Add custom nodes to the model for each relevant input parameter data csv. - - Args: - df : Pandas DataFrame with columns 'From' and 'To' describing the - transmission from and to contries. ie. - - Returns: - df_out : - - ''' - df_param = pd.DataFrame(list(itertools.product(custom_nodes, - tech_list, - years) - ), - columns = ['CUSTOM_NODE', - 'FUEL_TYPE', - 'YEAR'] - ) - df_param['REGION'] = region_name - df_custom = df_custom.groupby(['CUSTOM_NODE', - 'FUEL_TYPE', - 'START_YEAR', - 'END_YEAR'], - as_index=False)['CAPACITY'].sum() - df_param = pd.merge(df_param, - df_custom, - how='left', - on=['CUSTOM_NODE', - 'FUEL_TYPE']) - df_param['TECHNOLOGY'] = ('PWR' + - df_param['FUEL_TYPE'] + - df_param['CUSTOM_NODE'] + - '01') - technologies = df_param['TECHNOLOGY'].unique() - df_param.dropna(inplace=True) - df_param.drop_duplicates(inplace=True) - df_param = df_param.loc[df_param['YEAR'] >= df_param['START_YEAR']] - df_param = df_param.loc[df_param['YEAR'] <= df_param['END_YEAR']] - df_param['VALUE'] = df_param['CAPACITY'].div(1000) - df_param['REGION'] = region_name - df_param = df_param[['REGION','TECHNOLOGY','YEAR','VALUE']] - df_param = df_param.groupby(['REGION', - 'TECHNOLOGY', - 'YEAR'], - as_index=False)['VALUE'].sum() - - return df_param, technologies - -def duplicatePlexosTechs(df_in, techs): +def duplicate_plexos_techs(df_in, techs): """Creates new technologies to replace PLEXOS technolgoies. New technologies will end in '01', while historical ones end in '00' @@ -302,7 +254,7 @@ def duplicatePlexosTechs(df_in, techs): repl='01') return df_out -def newIar(df_in, tech, new_iar_ccg, +def new_iar(df_in, tech, new_iar_ccg, new_iar_ocg, new_iar_coa, new_iar_default): """Replaces the input activity ratio value with a hardcoded value diff --git a/workflow/scripts/osemosys_global/powerplant/investment_constraints.py b/workflow/scripts/osemosys_global/powerplant/investment_constraints.py index 36e3601d..d91885e5 100644 --- a/workflow/scripts/osemosys_global/powerplant/investment_constraints.py +++ b/workflow/scripts/osemosys_global/powerplant/investment_constraints.py @@ -2,14 +2,12 @@ import pandas as pd -from constants import ( - region_name, - years -) +from data import get_years from utils import apply_dtypes -def cap_investment_constraints(df_iar_final, no_investment_techs): +def cap_investment_constraints(df_iar_final, no_investment_techs, + start_year, end_year, region_name): # Create totalAnnualMaxCapacityInvestment data @@ -18,7 +16,7 @@ def cap_investment_constraints(df_iar_final, no_investment_techs): df_iar_final.loc[df_iar_final['TECHNOLOGY'].str.endswith('00')]['TECHNOLOGY'].tolist())) max_cap_invest_data = [] for tech in max_cap_invest_techs: - for year in years: + for year in get_years(start_year, end_year): max_cap_invest_data.append([region_name, tech, year, 0]) # Do not allow investment for all xxxABCxxxxxxx technologies @@ -29,7 +27,7 @@ def cap_investment_constraints(df_iar_final, no_investment_techs): df_iar_final['TECHNOLOGY'].str[3:6].isin(no_investment_techs)][ 'TECHNOLOGY'].tolist())) for tech in max_cap_invest_techs: - for year in years: + for year in get_years(start_year, end_year): max_cap_invest_data.append([region_name, tech, year, 0]) # Save totalAnnualMaxCapacityInvestment diff --git a/workflow/scripts/osemosys_global/powerplant/main.py b/workflow/scripts/osemosys_global/powerplant/main.py index 1ce305e9..5ae82a39 100644 --- a/workflow/scripts/osemosys_global/powerplant/main.py +++ b/workflow/scripts/osemosys_global/powerplant/main.py @@ -7,37 +7,16 @@ import_weo_costs, import_op_life, import_naming_convention_tech, - import_line_data, import_afs, import_custom_res_cap, ) from constants import( - file_plexos, - file_default_op_life, - file_naming_convention_tech, - file_weo_costs, - file_weo_regions, - file_line_data, - file_custom_res_cap, - file_default_av_factors, - avg_csp_eff, - avg_urn_eff, - duplicate_techs, - nodes_extra_list, - mode_list, - thermal_fuel_list_oar, - thermal_fuel_list_iar, - renewables_list, - thermal_fuel_list_mining, - costs_dict, - cross_border_trade, - no_investment_techs, - tech_capacity, - df_iar_custom_val, - df_oar_custom_val, - custom_nodes, - output_data_dir, + DUPLICATE_TECHS, + MODE_LIST, + RENEWABLES_LIST, + DF_IAR_CUSTOM_VAL, + DF_OAR_CUSTOM_VAL ) from data import( @@ -45,7 +24,10 @@ average_efficiency, ) -from residual_capacity import res_capacity +from residual_capacity import( + res_capacity, + add_custom_res_cap + ) from activity import( activity_master_start, @@ -56,22 +38,12 @@ capact ) -from activity_transmission import( - activity_transmission, - activity_transmission_limit - ) - from costs import( costs_pwr, costs_end ) -from costs_transmission import get_transmission_costs - -from operational_life import( - set_op_life, - set_op_life_transmission - ) +from operational_life import set_op_life from investment_constraints import cap_investment_constraints @@ -91,8 +63,6 @@ def main( weo_regions: pd.DataFrame, default_op_life: pd.DataFrame, naming_convention_tech: pd.DataFrame, - line_data: pd.DataFrame, - interface_data: pd.DataFrame, custom_res_cap: pd.DataFrame, default_av_factors: pd.DataFrame, ): @@ -100,89 +70,87 @@ def main( # CALL FUNCTIONS # return generator_table - gen_table = set_generator_table(plexos_prop, plexos_memb, - op_life_dict, tech_code_dict) + gen_table = set_generator_table(plexos_prop, plexos_memb, op_life_dict, + tech_code_dict, start_year, end_year) # Calculate average technology efficiencies. - df_eff_node, df_eff_tech = average_efficiency(gen_table, avg_csp_eff, - avg_urn_eff) - - # Calculate residual capacity. - df_res_cap, custom_techs = res_capacity(gen_table, tech_list, tech_code, - custom_res_cap, duplicate_techs) + df_eff_node, df_eff_tech = average_efficiency(gen_table) # Create master table for activity ratios. - df_ratios = activity_master_start(gen_table, nodes_extra_list, - duplicate_techs, mode_list) + df_ratios = activity_master_start(gen_table, DUPLICATE_TECHS, MODE_LIST, + custom_nodes, start_year, end_year) - # Set OutputActivitiyRatio for powerplants. - df_oar, df_oar_base = activity_output_pwr(df_ratios, thermal_fuel_list_oar) + # Set OutputActivitiyRatio for powerplants and set df structure for InputActivityRatio. + df_pwr_oar_final, df_pwr_iar_base = activity_output_pwr(df_ratios, region_name) # Set InputActivityRatio for powerplants. - df_iar_base = activity_input_pwr(df_oar, thermal_fuel_list_iar, renewables_list, - df_eff_node, df_eff_tech) + df_pwr_iar_final = activity_input_pwr(df_pwr_iar_base, RENEWABLES_LIST, df_eff_node, + df_eff_tech, region_name) # Set OutputActivitiyRatio for upstream and international technologies/fuels. - df_oar_upstream, df_oar_int = activity_upstream(df_iar_base, renewables_list, - thermal_fuel_list_mining) - - # Set activity ratios for transmission. - df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar = activity_transmission(df_oar_base, - plexos_prop, - interface_data) + df_oar_upstream, df_oar_int = activity_upstream(df_pwr_iar_final, RENEWABLES_LIST) # Combine and format activity ratios to output as csv. - df_oar_final, df_iar_final = activity_master_end(df_oar_base, df_oar_upstream, df_oar_int, - df_oar_trn, df_int_trn_oar, df_iar_base, - df_iar_trn, df_int_trn_iar, duplicate_techs) + df_oar_final, df_iar_final = activity_master_end(df_pwr_oar_final, df_oar_upstream, + df_oar_int, df_pwr_iar_final, + DUPLICATE_TECHS) # Set capital and fixed powerplant costs. - df_costs = costs_pwr(weo_costs, costs_dict) - - # Set capital and fixed transmission costs. - df_trans_capex, df_trans_fix = get_transmission_costs(trn_line, df_oar_final) + df_costs = costs_pwr(weo_costs) # Combine and format costs data to output as csv. - df_cap_cost_final, df_fix_cost_final = costs_end(weo_regions, df_costs, df_oar_final, - df_trans_capex, df_trans_fix) + df_cap_cost_final, df_fix_cost_final = costs_end(weo_regions, df_costs, df_oar_final) # Set CapacityToActivityUnit. df_capact_final = capact(df_oar_final) - - # Adjust activity limits if cross border trade is not allowed following user config. - df_crossborder_final = activity_transmission_limit(cross_border_trade, df_oar_final) # Set operational life for powerplant technologies. - df_op_life_trn, df_op_life_out = set_op_life(tech_code_dict, df_iar_final, - df_oar_final, op_life_dict) - - # Set operational life for transmission. - df_op_life = set_op_life_transmission(df_op_life_trn, df_op_life_out, op_life_dict) - + df_op_life = set_op_life(tech_code_dict, df_iar_final, + df_oar_final, op_life_dict, region_name) + # Set annual capacity investment constraints. df_max_cap_invest, df_min_cap_invest = cap_investment_constraints(df_iar_final, - no_investment_techs) + no_investment_techs, + start_year, + end_year, + region_name) + # Calculate residual capacity. + df_res_cap = res_capacity(gen_table, tech_list, tech_code, + DUPLICATE_TECHS, start_year, + end_year, region_name) - # Creates sets for TECHNOLOGIES and FUELS. if custom_nodes: - tech_set = create_sets('TECHNOLOGY', df_oar_final, output_data_dir, custom_techs) + + # Adds residual capacity for custom entries. + df_res_cap, custom_techs = add_custom_res_cap(df_res_cap, custom_res_cap, + tech_list, custom_nodes, + start_year, end_year, + region_name) + + # Creates sets for TECHNOLOGIES including custom entries. + tech_set = create_sets('TECHNOLOGY', df_oar_final, powerplant_data_dir, custom_techs) else: - tech_set = create_sets('TECHNOLOGY', df_oar_final, output_data_dir, []) + custom_techs = [] - fuel_set = create_sets('FUEL', df_oar_final, output_data_dir, []) + # Creates sets for TECHNOLOGIES absent custom entries. + tech_set = create_sets('TECHNOLOGY', df_oar_final, powerplant_data_dir, []) + + # Creates sets for FUEL. + fuel_set = create_sets('FUEL', df_oar_final, powerplant_data_dir, []) - # Create sets for YEAR, MODE_OF_OPERATION and REGION - years_set, mode_list_set, regions_set = output_sets(mode_list) + # Creates sets for YEAR, MODE_OF_OPERATION and REGION + years_set, mode_list_set, regions_set = output_sets(MODE_LIST, start_year, + end_year, region_name) # Alter output csv's based on user defined capacities following user config. if not tech_capacity is None: (tech_set, - fuel_set, df_max_cap_invest, df_min_cap_invest, df_res_cap, - df_iar_final, - df_oar_final, + df_iar_final, + df_oar_final, + fuel_set, df_op_life, df_capact_final, df_cap_cost_final @@ -193,50 +161,52 @@ def main( df_min_cap_invest, df_max_cap_invest, df_res_cap, + df_op_life, + df_capact_final, + df_cap_cost_final, df_iar_final, df_oar_final, fuel_set, - df_op_life, - df_capact_final, - df_cap_cost_final, - df_iar_custom_val, - df_oar_custom_val + start_year, + end_year, + region_name, + DF_IAR_CUSTOM_VAL, + DF_OAR_CUSTOM_VAL ) - # Set availability factors. - df_af_final = availability_factor(availability, tech_set) + # Set availability factors. Occurs after set_user_defined_capacity as tech_set gets updated. + df_af_final = availability_factor(availability, tech_set, + start_year, end_year, region_name) - # OUTPUT CSV's + # OUTPUT CSV's USED AS INPUT FOR TRANSMISSION RULE - df_res_cap.to_csv(os.path.join(output_data_dir, "ResidualCapacity.csv"), index=None) + df_res_cap.to_csv(os.path.join(powerplant_data_dir, "ResidualCapacity.csv"), index=None) - df_oar_final.to_csv(os.path.join(output_data_dir, "OutputActivityRatio.csv"), index=None) + df_oar_final.to_csv(os.path.join(powerplant_data_dir, "OutputActivityRatio.csv"), index=None) - df_iar_final.to_csv(os.path.join(output_data_dir, "InputActivityRatio.csv"), index=None) + df_iar_final.to_csv(os.path.join(powerplant_data_dir, "InputActivityRatio.csv"), index=None) - df_cap_cost_final.to_csv(os.path.join(output_data_dir, "CapitalCost.csv"), index = None) - - df_fix_cost_final.to_csv(os.path.join(output_data_dir, "FixedCost.csv"), index = None) + df_cap_cost_final.to_csv(os.path.join(powerplant_data_dir, "CapitalCost.csv"), index = None) - df_capact_final.to_csv(os.path.join(output_data_dir, "CapacityToActivityUnit.csv"), index = None) + df_fix_cost_final.to_csv(os.path.join(powerplant_data_dir, "FixedCost.csv"), index = None) - df_crossborder_final.to_csv(os.path.join(output_data_dir, - "TotalTechnologyModelPeriodActivityUpperLimit.csv"), - index = None) + df_capact_final.to_csv(os.path.join(powerplant_data_dir, "CapacityToActivityUnit.csv"), index = None) - df_op_life.to_csv(os.path.join(output_data_dir, "OperationalLife.csv"), index = None) + df_op_life.to_csv(os.path.join(powerplant_data_dir, "OperationalLife.csv"), index = None) - df_max_cap_invest.to_csv(os.path.join(output_data_dir, + df_max_cap_invest.to_csv(os.path.join(powerplant_data_dir, 'TotalAnnualMaxCapacityInvestment.csv'), index = None) - df_min_cap_invest.to_csv(os.path.join(output_data_dir, + df_min_cap_invest.to_csv(os.path.join(powerplant_data_dir, 'TotalAnnualMinCapacityInvestment.csv'), index = None) - df_af_final.to_csv(os.path.join(output_data_dir, 'AvailabilityFactor.csv'), index=None) + tech_set.to_csv(os.path.join(powerplant_data_dir, "TECHNOLOGY.csv"), index = None) + + # OUTPUT CSV's NOT USED AS INPUT FOR TRANMISSION RULE - tech_set.to_csv(os.path.join(output_data_dir, "TECHNOLOGY.csv"), index = None) + df_af_final.to_csv(os.path.join(output_data_dir, 'AvailabilityFactor.csv'), index=None) fuel_set.to_csv(os.path.join(output_data_dir, "FUEL.csv"), index = None) @@ -247,6 +217,47 @@ def main( regions_set.to_csv(os.path.join(output_data_dir, "REGION.csv"), index = None) if __name__ == "__main__": + + if "snakemake" in globals(): + file_plexos = snakemake.input.plexos + file_default_op_life = snakemake.input.default_op_life + file_naming_convention_tech = snakemake.input.naming_convention_tech + file_weo_costs = snakemake.input.weo_costs + file_weo_regions = snakemake.input.weo_regions + file_default_av_factors = snakemake.input.default_av_factors + start_year = snakemake.params.start_year + end_year = snakemake.params.end_year + region_name = snakemake.params.region_name + custom_nodes = snakemake.params.custom_nodes + tech_capacity = snakemake.params.user_defined_capacity + no_investment_techs = snakemake.params.no_investment_techs + output_data_dir = snakemake.params.output_data_dir + input_data_dir = snakemake.params.input_data_dir + powerplant_data_dir = snakemake.params.powerplant_data_dir + + if custom_nodes: + file_custom_res_cap = snakemake.input.custom_res_cap + + else: + file_plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx' + file_default_op_life = 'resources/data/operational_life.csv' + file_naming_convention_tech = 'resources/data/naming_convention_tech.csv' + file_weo_costs = 'resources/data/weo_2020_powerplant_costs.csv' + file_weo_regions = 'resources/data/weo_region_mapping.csv' + file_default_av_factors = 'resources/data/availability_factors.csv' + start_year = 2021 + end_year = 2050 + region_name = 'GLOBAL' + custom_nodes = ["INDWE", "INDEA", "INDNE", "INDNO", "INDSO"] + tech_capacity = {'PWRCOAINDWE01': [8, 2000, "open", 2025, 5, 1100]} + no_investment_techs = ["CSP", "WAV", "URN", "OTH", "WAS", + "COG", "GEO", "BIO", "PET"] + output_data_dir = 'results/data' + input_data_dir = 'resources/data' + powerplant_data_dir = 'results/data/powerplant' + + if custom_nodes: + file_custom_res_cap = 'resources/data/custom_nodes/residual_capacity.csv' # SET INPUT DATA plexos_prop = import_plexos_2015(file_plexos, "prop") @@ -263,16 +274,16 @@ def main( tech_code_dict = dict(zip(list(tech_code['tech']), list(tech_code['code']))) - trn_line = import_line_data(file_line_data, "Lines") - trn_interface = import_line_data(file_line_data, "Interface") - weo_costs = import_weo_costs(file_weo_costs) weo_regions = import_weo_regions(file_weo_regions) - custom_res_cap = import_custom_res_cap(file_custom_res_cap) - availability = import_afs(file_default_av_factors) + if custom_nodes: + custom_res_cap = import_custom_res_cap(file_custom_res_cap) + else: + custom_res_cap = [] + input_data = { "plexos_prop": plexos_prop, "plexos_memb": plexos_memb, @@ -280,8 +291,6 @@ def main( "weo_regions": weo_regions, "default_op_life": op_life, "naming_convention_tech": tech_code, - "line_data": trn_line, - "interface_data": trn_interface, "custom_res_cap" : custom_res_cap, "default_av_factors": availability, } diff --git a/workflow/scripts/osemosys_global/powerplant/operational_life.py b/workflow/scripts/osemosys_global/powerplant/operational_life.py index 165255fa..d1a9aa98 100644 --- a/workflow/scripts/osemosys_global/powerplant/operational_life.py +++ b/workflow/scripts/osemosys_global/powerplant/operational_life.py @@ -1,20 +1,16 @@ -"""Functions to set operational life.""" +"""Function to set operational life.""" import pandas as pd -from constants import ( - region_name, -) - from utils import apply_dtypes -def set_op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict): +def set_op_life(tech_code_dict, df_iar_final, df_oar_final, + op_life_dict, region_name): # Create Operational Life data tech_code_dict_reverse = dict((v,k) for k,v in tech_code_dict.items()) op_life_techs = list(set(list(df_iar_final['TECHNOLOGY'].unique()) + list(df_oar_final['TECHNOLOGY'].unique()))) op_life_pwr = [tech for tech in op_life_techs if (tech[0:3] == 'PWR') and (len(tech) == 13)] - op_life_trn = [tech for tech in op_life_techs if tech[0:3] == 'TRN'] op_life_out = [] @@ -26,16 +22,8 @@ def set_op_life(tech_code_dict, df_iar_final, df_oar_final, op_life_dict): op_life_tech, op_life_dict[op_life_tech_name]]) - return op_life_trn, op_life_out - -def set_op_life_transmission(op_life_trn, op_life_out, op_life_dict): - - # transmission technologies - for op_life_tech in op_life_trn: - op_life_out.append([region_name, op_life_tech, op_life_dict.get('TRN')]) - op_life_base = pd.DataFrame(op_life_out, columns = ['REGION', 'TECHNOLOGY', 'VALUE']) op_life_base = apply_dtypes(op_life_base, "OperationalLife") - + return op_life_base \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/read.py b/workflow/scripts/osemosys_global/powerplant/read.py index 9f9876a5..1d31d34f 100644 --- a/workflow/scripts/osemosys_global/powerplant/read.py +++ b/workflow/scripts/osemosys_global/powerplant/read.py @@ -44,20 +44,6 @@ def import_naming_convention_tech(f: str) -> pd.DataFrame: """ return pd.read_csv(f) -def import_line_data(f: str, metric: str) -> dict[str, pd.DataFrame]: - """Imports transmission data from PLEXOS-World. - - Costs Line expansion.xlsx - """ - if metric.lower() == "interface": - sheet_name = "Interface" - elif metric.lower() == "lines": - sheet_name = "Lines" - else: - raise NotImplementedError - - return pd.read_excel(f, sheet_name=sheet_name) - def import_afs(f: str) -> pd.DataFrame: """Imports default availability factors. @@ -70,18 +56,4 @@ def import_custom_res_cap(f: str) -> pd.DataFrame: custom_nodes\residual_capacity.csv """ - return pd.read_csv(f) - -def import_tech_set(f: str) -> pd.DataFrame: - """Imports list of technologies. - - TECHNOLOGY.csv - """ - return pd.read_csv(f) - -def import_fuel_set(f: str) -> pd.DataFrame: - """Imports list of fuels. - - FUEL.csv - """ return pd.read_csv(f) \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/residual_capacity.py b/workflow/scripts/osemosys_global/powerplant/residual_capacity.py index 32671c1f..8c2bf49a 100644 --- a/workflow/scripts/osemosys_global/powerplant/residual_capacity.py +++ b/workflow/scripts/osemosys_global/powerplant/residual_capacity.py @@ -1,21 +1,15 @@ """Function to calculate residual capacity for powerplant technologies.""" import pandas as pd - -from constants import ( - start_year, - end_year, - region_name, - custom_nodes, -) +import itertools from data import( - createPwrTechs, - custom_nodes_csv + create_pwr_techs, + get_years ) - -def res_capacity(df_gen_base, tech_list, df_tech_code, df_custom_res_cap, duplicate_techs): +def res_capacity(df_gen_base, tech_list, df_tech_code, duplicate_techs, + start_year, end_year, region_name): # ### Calculate residual capacity res_cap_cols = [ @@ -28,7 +22,10 @@ def res_capacity(df_gen_base, tech_list, df_tech_code, df_custom_res_cap, duplic df_res_cap = df_gen_base[res_cap_cols] - df_res_cap = df_res_cap.reindex(columns=[*df_res_cap.columns.tolist(), *list(range(start_year, end_year+1))], fill_value=0) + df_res_cap = df_res_cap.reindex(columns= + [*df_res_cap.columns.tolist(), + *list(range(start_year, end_year+1))], + fill_value=0) df_res_cap = pd.melt( df_res_cap, @@ -52,7 +49,7 @@ def res_capacity(df_gen_base, tech_list, df_tech_code, df_custom_res_cap, duplic )["value"].sum() # Add column with naming convention - df_res_cap = createPwrTechs(df_res_cap, duplicate_techs) + df_res_cap = create_pwr_techs(df_res_cap, duplicate_techs) # Convert total capacity from MW to GW df_res_cap['value'] = df_res_cap['value'].div(1000) @@ -73,15 +70,55 @@ def res_capacity(df_gen_base, tech_list, df_tech_code, df_custom_res_cap, duplic # Reorder columns df_res_cap = df_res_cap[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - if custom_nodes: - df_res_cap_custom, custom_techs = custom_nodes_csv(df_custom_res_cap, - tech_list) - df_res_cap = pd.concat([df_res_cap, df_res_cap_custom]) - + return df_res_cap + +def add_custom_res_cap(df_res_cap, df_custom, tech_list, custom_nodes, + start_year, end_year, region_name): + + df_custom_res_cap = pd.DataFrame(list(itertools.product(custom_nodes, + tech_list, + get_years(start_year, end_year)) + ), + columns = ['CUSTOM_NODE', + 'FUEL_TYPE', + 'YEAR'] + ) + df_custom_res_cap['REGION'] = region_name + df_custom = df_custom.groupby(['CUSTOM_NODE', + 'FUEL_TYPE', + 'START_YEAR', + 'END_YEAR'], + as_index=False)['CAPACITY'].sum() + df_custom_res_cap = pd.merge(df_custom_res_cap, + df_custom, + how='left', + on=['CUSTOM_NODE', + 'FUEL_TYPE']) + df_custom_res_cap['TECHNOLOGY'] = ('PWR' + + df_custom_res_cap['FUEL_TYPE'] + + df_custom_res_cap['CUSTOM_NODE'] + + '01') + technologies = df_custom_res_cap['TECHNOLOGY'].unique() + df_custom_res_cap.dropna(inplace=True) + df_custom_res_cap.drop_duplicates(inplace=True) + df_custom_res_cap = df_custom_res_cap.loc[df_custom_res_cap['YEAR'] >= + df_custom_res_cap['START_YEAR']] + df_custom_res_cap = df_custom_res_cap.loc[df_custom_res_cap['YEAR'] <= + df_custom_res_cap['END_YEAR']] + df_custom_res_cap['VALUE'] = df_custom_res_cap['CAPACITY'].div(1000) + df_custom_res_cap['REGION'] = region_name + df_custom_res_cap = df_custom_res_cap[['REGION','TECHNOLOGY','YEAR','VALUE']] + df_custom_res_cap = df_custom_res_cap.groupby(['REGION', + 'TECHNOLOGY', + 'YEAR'], + as_index=False)['VALUE'].sum() + + df_res_cap = pd.concat([df_res_cap, df_custom_res_cap]) + df_res_cap.drop_duplicates(subset=['REGION','TECHNOLOGY','YEAR'], keep='last', inplace=True) df_res_cap = df_res_cap.loc[(df_res_cap['TECHNOLOGY'].str.startswith('PWR')) & (~df_res_cap['TECHNOLOGY'].str.endswith('00'))] - return df_res_cap, custom_techs if custom_nodes else df_res_cap \ No newline at end of file + return df_custom_res_cap, technologies \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/sets.py b/workflow/scripts/osemosys_global/powerplant/sets.py index 1b7729ec..cd980c01 100644 --- a/workflow/scripts/osemosys_global/powerplant/sets.py +++ b/workflow/scripts/osemosys_global/powerplant/sets.py @@ -2,11 +2,9 @@ import pandas as pd -from constants import ( - region_name, - years, - SET_DTYPES -) +from data import get_years + +from constants import SET_DTYPES def create_sets(x, df, output_dir, custom_node_elements): """Creates a formatted otoole set csv @@ -29,10 +27,10 @@ def create_sets(x, df, output_dir, custom_node_elements): set_elements_df = pd.DataFrame(set_elements, columns = ['VALUE']) return set_elements_df -def output_sets(mode_list): +def output_sets(mode_list, start_year, end_year, region_name): # ## Create set for YEAR, REGION, MODE_OF_OPERATION - years_df = pd.DataFrame(years, columns = ['VALUE']).astype(SET_DTYPES["YEAR"]) + years_df = pd.DataFrame(get_years(start_year, end_year), columns = ['VALUE']).astype(SET_DTYPES["YEAR"]) mode_list_df = pd.DataFrame(mode_list, columns = ['VALUE']).astype(SET_DTYPES["MODE_OF_OPERATION"]) diff --git a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py index b2eb8a48..eca62469 100644 --- a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py +++ b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py @@ -3,27 +3,15 @@ import pandas as pd import itertools -from constants import ( - region_name, - years -) +from data import get_years def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, df_min_cap_invest, df_max_cap_invest, df_res_cap, - df_iar_final, df_oar_final, fuel_set, op_life_base, cap_act_base, - cap_cost_base, df_iar_custom_val, df_oar_custom_val): - """User-defined capacities are used when a specific technology must be - invested, for a given year and capacity. This is applied through the - parameter 'TotalAnnualMinCapacityInvestment'. - - Args: - region: From config file (e.g. 'GLOBAL') - tech_capacity: User-defined capacity in config file - (e.g. TRNAGOXXCODXX: [5, 2030]) - - Returns - None - """ + op_life_base, cap_act_base, cap_cost_base, df_iar_final, + df_oar_final, fuel_set, start_year, end_year, region_name, + DF_IAR_CUSTOM_VAL, DF_OAR_CUSTOM_VAL + ): + techCapacity = [] tech_capacity_dict = {} first_year_dict = {} @@ -35,11 +23,11 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, for tech, tech_params in tech_capacity.items(): techCapacity.append([tech, tech_params[0], tech_params[1]]) - tech_capacity_dict[tech] = tech_params[2] + tech_capacity_dict[tech] = tech_params[2] #UNUSED ENTRY build_year_dict[tech] = tech_params[1] first_year_dict[tech] = tech_params[3] build_rate_dict[tech] = tech_params[4] - capex_dict[tech] = tech_params[5] # + capex_dict[tech] = tech_params[5] tech_capacity_df = pd.DataFrame(techCapacity, columns=['TECHNOLOGY', 'VALUE', 'YEAR']) tech_capacity_df['REGION'] = region_name @@ -54,50 +42,48 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_df]) df_min_cap_inv.drop_duplicates(inplace=True) - df = pd.DataFrame(list(itertools.product(list(tech_capacity_df['TECHNOLOGY'].unique()), - years) + max_cap_techs_df = pd.DataFrame(list(itertools.product(list(tech_capacity_df['TECHNOLOGY'].unique()), + get_years(start_year, end_year)) ), columns = ['TECHNOLOGY', 'YEAR'] ) - df['REGION'] = region_name - df = pd.merge(df, df_min_cap_inv, + max_cap_techs_df['REGION'] = region_name + max_cap_techs_df = pd.merge(max_cap_techs_df, df_min_cap_inv, how='left', on=['REGION', 'TECHNOLOGY', 'YEAR']) - df['FIRST_YEAR'] = df['TECHNOLOGY'].map(first_year_dict) - df['BUILD_YEAR'] = df['TECHNOLOGY'].map(build_year_dict) - df['MAX_BUILD'] = df['TECHNOLOGY'].map(build_rate_dict) + max_cap_techs_df['FIRST_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(first_year_dict) + max_cap_techs_df['BUILD_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(build_year_dict) + max_cap_techs_df['MAX_BUILD'] = max_cap_techs_df['TECHNOLOGY'].map(build_rate_dict) # Fill VALUE with MAX_BUILD for YEAR >= FIRST_YEAR - df.loc[(df['YEAR']>=df['FIRST_YEAR']) & - (df['YEAR']>df['BUILD_YEAR']), - 'VALUE'] = df['MAX_BUILD'] - df.infer_objects().fillna(0, + max_cap_techs_df.loc[(max_cap_techs_df['YEAR']>=max_cap_techs_df['FIRST_YEAR']) & + (max_cap_techs_df['YEAR']>max_cap_techs_df['BUILD_YEAR']), + 'VALUE'] = max_cap_techs_df['MAX_BUILD'] + max_cap_techs_df.infer_objects().fillna(0, inplace=True) - max_cap_techs_df = df[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] - - #Replace VALUE with CAPEX - df['VALUE'] = df['TECHNOLOGY'].map(capex_dict) - # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD for TRN + max_cap_techs_df = max_cap_techs_df[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + + # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD df_max_cap_inv = pd.concat([df_max_cap_invest, max_cap_techs_df]).drop_duplicates() - # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD for TRN + # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD df_max_cap_inv.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], keep='last', inplace=True) - + # For technologies with start year before model start year, add to ResidualCapacity - df_res_cap_ud = df_min_cap_inv.copy().loc[df_min_cap_inv.copy()['YEAR'] < min(years)] - df_res_cap_ud.rename(columns={'YEAR':'START_YEAR'}, + df_res_cap_ud = df_min_cap_inv.copy().loc[df_min_cap_inv.copy()['YEAR'] < min(get_years(start_year, end_year))] + df_res_cap_ud.rename(columns={'YEAR':'start_year'}, inplace=True) df_res_cap_ud_final = pd.DataFrame(list(itertools.product(df_res_cap_ud['TECHNOLOGY'].unique(), - years) + get_years(start_year, end_year)) ), columns = ['TECHNOLOGY', 'YEAR'] @@ -107,39 +93,36 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, how='left', on=['TECHNOLOGY']) df_res_cap_ud_final['TECH'] = df_res_cap_ud_final['TECHNOLOGY'].str[3:6] - df_res_cap_ud_final.loc[df_res_cap_ud_final['TECHNOLOGY'].str.contains('TRN'), - 'TECH'] = 'TRN' df_res_cap_ud_final['OP_LIFE'] = df_res_cap_ud_final['TECH'].map(op_life_dict) - df_res_cap_ud_final['END_YEAR'] = (df_res_cap_ud_final['OP_LIFE'] - + df_res_cap_ud_final['START_YEAR']) + df_res_cap_ud_final['end_year'] = (df_res_cap_ud_final['OP_LIFE'] + + df_res_cap_ud_final['start_year']) df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] - >= df_res_cap_ud_final['START_YEAR']] + >= df_res_cap_ud_final['start_year']] df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] - <= df_res_cap_ud_final['END_YEAR']] + <= df_res_cap_ud_final['end_year']] df_res_cap_ud_final['REGION'] = region_name df_res_cap_ud_final = df_res_cap_ud_final[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final if not df_res_cap_ud_final.empty else None]) # For technologies with start year at or after model start year, add to # TotalAnnualMinCapacityInvestment - df_min_cap_inv = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] >= min(years)] + df_min_cap_inv = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] >= min(get_years(start_year, end_year))] df_min_cap_inv.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], keep='last', inplace=True) - # Add IAR and OAR for custom technologies tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) + df_iar_custom = pd.DataFrame(list(itertools.product(tech_list, [1, 2], - years) + get_years(start_year, end_year)) ), columns = ['TECHNOLOGY', 'MODE_OF_OPERATION', @@ -147,7 +130,7 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, ) df_oar_custom = pd.DataFrame(list(itertools.product(tech_list, [1, 2], - years) + get_years(start_year, end_year)) ), columns = ['TECHNOLOGY', 'MODE_OF_OPERATION', @@ -172,8 +155,8 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, 'FUEL'] = ('ELC' + df_oar_custom['TECHNOLOGY'].str[3:8] + '01') - df_iar_custom['VALUE'] = df_iar_custom_val - df_oar_custom['VALUE'] = df_oar_custom_val + df_iar_custom['VALUE'] = DF_IAR_CUSTOM_VAL + df_oar_custom['VALUE'] = DF_OAR_CUSTOM_VAL df_iar_custom['REGION'] = region_name df_oar_custom['REGION'] = region_name @@ -208,7 +191,7 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, 'YEAR'], keep='last', inplace=True) - + # Add new fuels to FUEL set, if not already present fuel_list = [] fuel_list = list(df_iar_custom['FUEL'].unique()) + list(df_oar_custom['FUEL'].unique()) @@ -217,11 +200,13 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, for each_fuel in fuel_list: if each_fuel not in list(fuel_set['VALUE']): fuel_set = pd.concat([fuel_set, pd.DataFrame({'VALUE':[each_fuel]})]) - + op_life_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) + + for each_tech in op_life_custom['TECHNOLOGY']: + op_life_custom.loc[op_life_custom['TECHNOLOGY'] == each_tech, + 'VALUE'] = op_life_dict.get(each_tech[3:6]) - op_life_custom.loc[op_life_custom['TECHNOLOGY'].str.contains('TRN'), - 'VALUE'] = op_life_dict.get('TRN') op_life_custom['REGION'] = region_name op_life_custom = op_life_custom[['REGION', 'TECHNOLOGY', @@ -233,9 +218,8 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, keep='last', inplace=True) + # Add CapacityToActivityUnit for custom technologies cap_act_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) - cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('TRN'), - 'VALUE'] = 31.536 cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('PWR'), 'VALUE'] = 31.536 cap_act_custom['REGION'] = region_name @@ -245,29 +229,27 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, cap_act = pd.concat([cap_act_base, cap_act_custom]) cap_act.drop_duplicates(inplace=True) + # Update CapitalCost with user-defined costs tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) - cap_cost_trn = pd.DataFrame(list(itertools.product(tech_list, - years)), + cap_cost = pd.DataFrame(list(itertools.product(tech_list, + get_years(start_year, end_year))), columns = ['TECHNOLOGY', 'YEAR']) - # Update CapitalCost with user-defined costs by transmission line - for each_trn in tech_list: - cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.startswith(each_trn), - 'VALUE'] = capex_dict[each_trn] + for each_tech in tech_list: + cap_cost.loc[cap_cost['TECHNOLOGY'].str.startswith(each_tech), + 'VALUE'] = capex_dict[each_tech] - cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.contains('PWRTRN'), - 'VALUE'] = 300 - cap_cost_trn['REGION'] = region_name - cap_cost_trn = cap_cost_trn[['REGION', + cap_cost['REGION'] = region_name + cap_cost = cap_cost[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - cap_cost = pd.concat([cap_cost_base, cap_cost_trn]) + cap_cost = pd.concat([cap_cost_base, cap_cost]) cap_cost.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], keep="last", inplace=True) - return(df_tech_set, fuel_set, df_max_cap_inv, df_min_cap_inv, - df_res_cap, df_iar, df_oar, op_life, cap_act, cap_cost) - \ No newline at end of file + return(df_tech_set, df_max_cap_inv, df_min_cap_inv, + df_res_cap, df_iar, df_oar, fuel_set, op_life, + cap_act, cap_cost) \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/activity_transmission.py b/workflow/scripts/osemosys_global/transmission/activity.py similarity index 71% rename from workflow/scripts/osemosys_global/powerplant/activity_transmission.py rename to workflow/scripts/osemosys_global/transmission/activity.py index 8dbcfad7..8cd47547 100644 --- a/workflow/scripts/osemosys_global/powerplant/activity_transmission.py +++ b/workflow/scripts/osemosys_global/transmission/activity.py @@ -1,18 +1,12 @@ """Function to calaculate activity related to transmission.""" - import pandas as pd -from constants import ( - region_name, - start_year, - end_year -) - -from data_transmission import format_transmission_name +from data import format_transmission_name from utils import apply_dtypes -def activity_transmission(df_oar_base, df_pw_prop, df_trn_efficiencies): +def activity_transmission(df_iar_base, df_oar_base, df_pw_prop, df_trn_efficiencies, + start_year, end_year, region_name): # #### Downstream Activity Ratios @@ -27,6 +21,9 @@ def activity_transmission(df_oar_base, df_pw_prop, df_trn_efficiencies): # And remove all the duplicate entries df_iar_trn.drop_duplicates(keep="first", inplace=True) + # Only keep technologies linked to the correct fuel + df_iar_trn = df_iar_trn.loc[df_iar_trn['FUEL'].str.contains('ELC')] + # OAR for transmission technologies is IAR, but the fuel is 02 instead of 01: df_oar_trn = df_iar_trn.copy() df_oar_trn["FUEL"] = df_oar_trn["FUEL"].str[0:8] + "02" @@ -154,13 +151,54 @@ def activity_transmission(df_oar_base, df_pw_prop, df_trn_efficiencies): df_int_trn_oar, df_trn_efficiencies, how="outer", on="TECHNOLOGY" ) - return df_iar_trn, df_oar_trn, df_int_trn_oar, df_int_trn_iar + df_oar_trn_final = pd.concat( + [ + df_oar_base, + df_oar_trn, + df_int_trn_oar, + ] + ).dropna() + + # Select columns for final output table + df_oar_trn_final = df_oar_trn_final[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + + df_iar_trn_final = pd.concat( + [ + df_iar_base, + df_iar_trn, + df_int_trn_iar, + ] + ).dropna() + + + # Select columns for final output table + df_iar_trn_final = df_iar_trn_final[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] -def activity_transmission_limit(cross_border_trade, df_oar_final): + df_oar_trn_final.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', + inplace=True) + + return df_iar_trn_final, df_oar_trn_final + +def activity_transmission_limit(cross_border_trade, df_oar_trn_final): # Set cross-border trade to 0 if False if not cross_border_trade: - df_crossborder_final = df_oar_final[['REGION', + df_crossborder_final = df_oar_trn_final[['REGION', 'TECHNOLOGY' ]] df_crossborder_final = df_crossborder_final.drop_duplicates() @@ -178,4 +216,21 @@ def activity_transmission_limit(cross_border_trade, df_oar_final): df_crossborder_final = apply_dtypes(df_crossborder_final, "TotalTechnologyModelPeriodActivityUpperLimit") - return df_crossborder_final \ No newline at end of file + return df_crossborder_final + +def capact_transmission(df_capact_base, df_oar_trn_final): + + # Create CapacityToActivityUnit csv + df_capact_trn_final = df_oar_trn_final[['REGION', + 'TECHNOLOGY' + ]] + df_capact_trn_final = df_capact_trn_final.drop_duplicates() + df_capact_trn_final = df_capact_trn_final.loc[df_capact_trn_final['TECHNOLOGY' + ].str.startswith('TRN')] + + df_capact_trn_final['VALUE'] = 31.536 + df_capact_trn_final.drop_duplicates(inplace=True) + + df_capact_trn_final = pd.concat([df_capact_base, df_capact_trn_final]) + + return df_capact_trn_final \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/constants.py b/workflow/scripts/osemosys_global/transmission/constants.py new file mode 100644 index 00000000..a38ca4bf --- /dev/null +++ b/workflow/scripts/osemosys_global/transmission/constants.py @@ -0,0 +1,19 @@ +"""Constants for the transmission module""" + +"""Set iar and oar values for custom transmission entries. I.e. oar of 0.9 +assumes 10% losses. Relevant for the user_defined_capacity function.""" +DF_IAR_CUSTOM_VAL = 1 +DF_OAR_CUSTOM_VAL = 0.9 + +SET_DTYPES = { + "DAILYTIMEBRACKET": int, + "EMISSION":str, + "FUEL":str, + "MODE_OF_OPERATION":int, + "REGION":str, + "SEASON":str, + "STORAGE":str, + "TECHNOLOGY":str, + "TIMESLICE":str, + "YEAR":int, +} \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/costs_transmission.py b/workflow/scripts/osemosys_global/transmission/costs.py similarity index 78% rename from workflow/scripts/osemosys_global/powerplant/costs_transmission.py rename to workflow/scripts/osemosys_global/transmission/costs.py index 1c2b7e67..58d52bae 100644 --- a/workflow/scripts/osemosys_global/powerplant/costs_transmission.py +++ b/workflow/scripts/osemosys_global/transmission/costs.py @@ -1,13 +1,13 @@ """Function to calculate transmission costs.""" -from data_transmission import format_transmission_name +import pandas as pd -from constants import( - region_name, - years - ) +from data import( + format_transmission_name, + get_years) -def get_transmission_costs(df_trn_lines, df_oar_final): +def get_transmission_costs(df_trn_lines, df_oar_final, cap_cost_base, fix_cost_base, + start_year, end_year, region_name): '''Gets electrical transmission capital and fixed cost per technology. Both the capital costs and fixed cost are written out to avoid having @@ -56,11 +56,11 @@ def get_transmission_costs(df_trn_lines, df_oar_final): df_trans_fix = df_trans_fix.rename(columns={'O&M':'VALUE'}) df_trans_capex['REGION'] = region_name - df_trans_capex['YEAR'] = [years] * len(df_trans_capex) + df_trans_capex['YEAR'] = [get_years(start_year, end_year)] * len(df_trans_capex) df_trans_capex = df_trans_capex.explode('YEAR') df_trans_fix['REGION'] = region_name - df_trans_fix['YEAR'] = [years] * len(df_trans_fix) + df_trans_fix['YEAR'] = [get_years(start_year, end_year)] * len(df_trans_fix) df_trans_fix = df_trans_fix.explode('YEAR') # Filter out techs that don't have activity ratios @@ -69,4 +69,7 @@ def get_transmission_costs(df_trn_lines, df_oar_final): df_trans_fix = df_trans_fix.loc[ df_trans_fix['TECHNOLOGY'].isin(df_oar_final['TECHNOLOGY'])] - return df_trans_capex, df_trans_fix \ No newline at end of file + cap_cost_trn_final = pd.concat([cap_cost_base, df_trans_capex]) + fix_cost_trn_final = pd.concat([fix_cost_base, df_trans_fix]) + + return cap_cost_trn_final, fix_cost_trn_final \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/powerplant/data_transmission.py b/workflow/scripts/osemosys_global/transmission/data.py similarity index 95% rename from workflow/scripts/osemosys_global/powerplant/data_transmission.py rename to workflow/scripts/osemosys_global/transmission/data.py index 06e6c22f..6a29022e 100644 --- a/workflow/scripts/osemosys_global/powerplant/data_transmission.py +++ b/workflow/scripts/osemosys_global/transmission/data.py @@ -1,5 +1,8 @@ """Functions to extract and format relevent data for tranmission.""" +def get_years(start: int, end: int) -> range: + return range(start, end + 1) + def format_transmission_name(df): '''Formats PLEXOS transmission names into OSeMOSYS Global names. diff --git a/workflow/scripts/osemosys_global/transmission/investment_constraints.py b/workflow/scripts/osemosys_global/transmission/investment_constraints.py new file mode 100644 index 00000000..0cc27eee --- /dev/null +++ b/workflow/scripts/osemosys_global/transmission/investment_constraints.py @@ -0,0 +1,34 @@ +"""Function to set min and max capacity investment constraints.""" + +import pandas as pd + +from data import get_years + +from utils import apply_dtypes + +def cap_investment_constraints_trn(df_iar_trn_final, df_max_cap_invest_base, + no_investment_techs, start_year, + end_year, region_name): + + if 'TRN' in no_investment_techs: + max_cap_invest_data = [] + + max_cap_invest_techs = list(set(df_iar_trn_final.loc[ + df_iar_trn_final['TECHNOLOGY'].str.startswith('TRN')][ + 'TECHNOLOGY'].tolist())) + for tech in max_cap_invest_techs: + for year in get_years(start_year, end_year): + max_cap_invest_data.append([region_name, tech, year, 0]) + + # Save totalAnnualMaxCapacityInvestment + df_max_cap_invest_trn = pd.DataFrame(max_cap_invest_data, + columns = ['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE'] + ) + df_max_cap_invest_trn = apply_dtypes(df_max_cap_invest_trn, "TotalAnnualMaxCapacityInvestment") + + df_max_cap_invest_trn = pd.concat([df_max_cap_invest_base, df_max_cap_invest_trn]) + + else: + df_max_cap_invest_trn = df_max_cap_invest_base + + return df_max_cap_invest_trn \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/main.py b/workflow/scripts/osemosys_global/transmission/main.py new file mode 100644 index 00000000..0410dce2 --- /dev/null +++ b/workflow/scripts/osemosys_global/transmission/main.py @@ -0,0 +1,247 @@ +import pandas as pd +import os + +from read import( + import_plexos_2015, + import_op_life, + import_line_data, + import_iar_base, + import_oar_base, + import_capact_base, + import_cap_cost_base, + import_fix_cost_base, + import_op_life_base, + import_max_cap_invest_base, + import_min_cap_invest_base, + import_res_cap_base, + import_tech_set_base +) + +from constants import( + DF_IAR_CUSTOM_VAL, + DF_OAR_CUSTOM_VAL + ) + +from activity import( + activity_transmission, + activity_transmission_limit, + capact_transmission + ) + +from costs import get_transmission_costs + +from operational_life import set_op_life_transmission + +from investment_constraints import cap_investment_constraints_trn + +from user_defined_capacity import set_user_defined_capacity_trn + +from sets import create_tech_set_trn + +def main( + plexos_prop: pd.DataFrame, + default_op_life: pd.DataFrame, + line_data: pd.DataFrame, + interface_data: pd.DataFrame, + iar_base: pd.DataFrame, + oar_base: pd.DataFrame, + capact_base: pd.DataFrame, + cap_cost_base: pd.DataFrame, + fix_cost_base: pd.DataFrame, + op_life_base: pd.DataFrame, + max_cap_invest_base: pd.DataFrame, + min_cap_invest_base: pd.DataFrame, + res_cap_base: pd.DataFrame, + tech_set_base: pd.DataFrame, +): + + # CALL FUNCTIONS + + # Set activity ratios for transmission. + df_iar_trn_final, df_oar_trn_final = activity_transmission(iar_base, oar_base, + plexos_prop, interface_data, + start_year, end_year, + region_name) + + # Adjust activity limits if cross border trade is not allowed following user config. + df_crossborder_final = activity_transmission_limit(cross_border_trade, df_oar_trn_final) + + # Set CapacityToActivityUnit. + df_capact_trn_final = capact_transmission(capact_base, df_oar_trn_final) + + # Set capital and fixed transmission costs. + df_cap_cost_trn_final, df_fix_cost_trn_final = get_transmission_costs(line_data, df_oar_trn_final, + cap_cost_base, fix_cost_base, + start_year, end_year, + region_name) + + # Set operational life for transmission. + df_op_life_trn_final = set_op_life_transmission(df_iar_trn_final, df_oar_trn_final, + op_life_dict, op_life_base, region_name) + + # Set annual capacity investment constraints. + df_max_cap_invest_trn_final = cap_investment_constraints_trn(df_iar_trn_final, + max_cap_invest_base, + no_investment_techs, + start_year, + end_year, + region_name) + + tech_set_trn_final = pd.concat([tech_set, create_tech_set_trn(interface_data)]) + + # Alter output csv's based on user defined capacities following user config. + if not tech_capacity_trn is None: + (tech_set_trn_final, + df_max_cap_invest_trn_final, + df_min_cap_invest_trn_final, + df_res_cap_trn_final, + df_iar_trn_final, + df_oar_trn_final, + df_op_life_trn_final, + df_capact_trn_final, + df_cap_cost_trn_final + ) = set_user_defined_capacity_trn( + tech_capacity_trn, + op_life_dict, + tech_set_trn_final, + min_cap_invest_base, + df_max_cap_invest_trn_final, + res_cap_base, + df_iar_trn_final, + df_oar_trn_final, + df_op_life_trn_final, + df_capact_trn_final, + df_cap_cost_trn_final, + start_year, + end_year, + region_name, + DF_IAR_CUSTOM_VAL, + DF_OAR_CUSTOM_VAL + ) + + # OUTPUT CSV's + + df_oar_trn_final.to_csv(os.path.join(output_data_dir, "OutputActivityRatio.csv"), index=None) + + df_iar_trn_final.to_csv(os.path.join(output_data_dir, "InputActivityRatio.csv"), index=None) + + df_crossborder_final.to_csv(os.path.join(output_data_dir, + "TotalTechnologyModelPeriodActivityUpperLimit.csv"), + index = None) + + df_capact_trn_final.to_csv(os.path.join(output_data_dir, + "CapacityToActivityUnit.csv"), index = None) + + df_cap_cost_trn_final.to_csv(os.path.join(output_data_dir, "CapitalCost.csv"), index = None) + + df_fix_cost_trn_final.to_csv(os.path.join(output_data_dir, "FixedCost.csv"), index = None) + + df_op_life_trn_final.to_csv(os.path.join(output_data_dir, "OperationalLife.csv"), index = None) + + df_max_cap_invest_trn_final.to_csv(os.path.join(output_data_dir, + 'TotalAnnualMaxCapacityInvestment.csv'), + index = None) + + tech_set_trn_final.to_csv(os.path.join(output_data_dir, "TECHNOLOGY.csv"), index = None) + + if not tech_capacity_trn is None: + + df_res_cap_trn_final.to_csv(os.path.join(output_data_dir, + 'ResidualCapacity.csv'), + index = None) + + df_min_cap_invest_trn_final.to_csv(os.path.join(output_data_dir, + 'TotalAnnualMinCapacityInvestment.csv'), + index = None) + +if __name__ == "__main__": + + if "snakemake" in globals(): + file_plexos = snakemake.input.plexos + file_default_op_life = snakemake.input.default_op_life + file_line_data = snakemake.input.line_data + start_year = snakemake.params.start_year + end_year = snakemake.params.end_year + region_name = snakemake.params.region_name + custom_nodes = snakemake.params.custom_nodes + tech_capacity_trn = snakemake.params.user_defined_capacity_transmission + no_investment_techs = snakemake.params.no_investment_techs + cross_border_trade = snakemake.params.trade + output_data_dir = snakemake.params.output_data_dir + input_data_dir = snakemake.params.input_data_dir + powerplant_data_dir = snakemake.params.powerplant_data_dir + file_iar_base = f'{powerplant_data_dir}/InputActivityRatio.csv' + file_oar_base = f'{powerplant_data_dir}/OutputActivityRatio.csv' + file_capact_base = f'{powerplant_data_dir}/CapacityToActivityUnit.csv' + file_cap_cost_base = f'{powerplant_data_dir}/CapitalCost.csv' + file_fix_cost_base = f'{powerplant_data_dir}/FixedCost.csv' + file_op_life_base = f'{powerplant_data_dir}/OperationalLife.csv' + file_max_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMaxCapacityInvestment.csv' + file_min_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMinCapacityInvestment.csv' + file_res_cap_base = f'{powerplant_data_dir}/ResidualCapacity.csv' + file_tech_set = f'{powerplant_data_dir}/TECHNOLOGY.csv' + + else: + file_plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx' + file_default_op_life = 'resources/data/operational_life.csv' + file_line_data = 'resources/data/Costs Line expansion.xlsx' + start_year = 2021 + end_year = 2050 + region_name = 'GLOBAL' + custom_nodes = ["INDWE", "INDEA", "INDNE", "INDNO", "INDSO"] + tech_capacity_trn = {'TRNINDEAINDNE': [5, 1975, "open", 2030, 10, 861]} + no_investment_techs = ["CSP", "WAV", "URN", "OTH", "WAS", + "COG", "GEO", "BIO", "PET"] + cross_border_trade = True + output_data_dir = 'results/data' + input_data_dir = 'resources/data' + powerplant_data_dir = 'results/data/powerplant' + file_iar_base = f'{powerplant_data_dir}/InputActivityRatio' + file_oar_base = f'{powerplant_data_dir}/OutputActivityRatio' + file_capact_base = f'{powerplant_data_dir}/CapacityToActivityUnit' + file_cap_cost_base = f'{powerplant_data_dir}/CapitalCost' + file_fix_cost_base = f'{powerplant_data_dir}/FixedCost' + file_op_life_base = f'{powerplant_data_dir}/OperationalLife' + file_max_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMaxCapacityInvestment.csv' + file_min_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMinCapacityInvestment.csv' + file_res_cap_base = f'{powerplant_data_dir}/ResidualCapacity.csv' + file_tech_set = f'{powerplant_data_dir}/TECHNOLOGY.csv' + + # SET INPUT DATA + plexos_prop = import_plexos_2015(file_plexos, "prop") + op_life = import_op_life(file_default_op_life) + op_life_dict = dict(zip(list(op_life['tech']), + list(op_life['years']))) + + trn_line = import_line_data(file_line_data, "Lines") + trn_interface = import_line_data(file_line_data, "Interface") + iar_base = import_iar_base(file_iar_base) + oar_base = import_oar_base(file_oar_base) + capact_base = import_capact_base(file_capact_base) + cap_cost_base = import_cap_cost_base(file_cap_cost_base) + fix_cost_base = import_fix_cost_base(file_fix_cost_base) + op_life_base = import_op_life_base(file_op_life_base) + max_cap_invest_base = import_max_cap_invest_base(file_max_cap_invest_base) + min_cap_invest_base = import_min_cap_invest_base(file_min_cap_invest_base) + res_cap_base = import_res_cap_base(file_res_cap_base) + tech_set = import_tech_set_base(file_tech_set) + + input_data = { + "plexos_prop": plexos_prop, + "default_op_life": op_life, + "line_data": trn_line, + "interface_data": trn_interface, + "iar_base" : iar_base, + "oar_base" : oar_base, + "capact_base" : capact_base, + "cap_cost_base" : cap_cost_base, + "fix_cost_base" : fix_cost_base, + "op_life_base" : op_life_base, + "max_cap_invest_base" : max_cap_invest_base, + "min_cap_invest_base" : min_cap_invest_base, + "res_cap_base" : res_cap_base, + "tech_set_base" : tech_set, + } + + # CALL MAIN + main(**input_data) \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/operational_life.py b/workflow/scripts/osemosys_global/transmission/operational_life.py new file mode 100644 index 00000000..5c89fd72 --- /dev/null +++ b/workflow/scripts/osemosys_global/transmission/operational_life.py @@ -0,0 +1,30 @@ +"""Functions to set operational life.""" + +import pandas as pd + +from utils import apply_dtypes + +def set_op_life_transmission(df_iar_final, df_oar_final, + op_life_dict, op_life_base, + region_name): + + # Create Operational Life data + op_life_techs = list(set(list(df_iar_final['TECHNOLOGY'].unique() + ) + list(df_oar_final['TECHNOLOGY'].unique()))) + + op_life_trn = [tech for tech in op_life_techs if tech[0:3] == 'TRN'] + + op_life_out = [] + + # transmission technologies + for op_life_tech in op_life_trn: + op_life_out.append([region_name, op_life_tech, op_life_dict.get('TRN')]) + + op_life_trn_final = pd.DataFrame(op_life_out, columns = ['REGION', + 'TECHNOLOGY', 'VALUE']) + + op_life_trn_final = apply_dtypes(op_life_trn_final, "OperationalLife") + + op_life_trn_final = pd.concat([op_life_base, op_life_trn_final]) + + return op_life_trn_final \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/read.py b/workflow/scripts/osemosys_global/transmission/read.py new file mode 100644 index 00000000..6ad7066c --- /dev/null +++ b/workflow/scripts/osemosys_global/transmission/read.py @@ -0,0 +1,108 @@ +"""Module for reading in data sources""" + +import pandas as pd + +def import_plexos_2015(f: str, metric: str) -> dict[str, pd.DataFrame]: + """Imports PLEXOS-World 2015 model file. + + PLEXOS_World_2015_Gold_V1.1.xlsx + """ + if metric.lower() == "memb": + sheet_name = "Memberships" + elif metric.lower() == "prop": + sheet_name = "Properties" + else: + raise NotImplementedError + + return pd.read_excel(f, sheet_name=sheet_name) + +def import_line_data(f: str, metric: str) -> dict[str, pd.DataFrame]: + """Imports transmission data from PLEXOS-World. + + Costs Line expansion.xlsx + """ + if metric.lower() == "interface": + sheet_name = "Interface" + elif metric.lower() == "lines": + sheet_name = "Lines" + else: + raise NotImplementedError + + return pd.read_excel(f, sheet_name=sheet_name) + +def import_op_life(f: str) -> pd.DataFrame: + """Imports default operational life data. + + operational_life.csv + """ + return pd.read_csv(f) + +def import_iar_base(f: str) -> pd.DataFrame: + """Imports InputActivityRatio.csv as output from the Powerplant rule. + + InputActivityRatio.csv + """ + return pd.read_csv(f) + +def import_oar_base(f: str) -> pd.DataFrame: + """Imports OutputActivityRatio.csv as output from the Powerplant rule. + + OutputActivityRatio.csv + """ + return pd.read_csv(f) + +def import_capact_base(f: str) -> pd.DataFrame: + """Imports CapacityToActivityUnit.csv as output from the Powerplant rule. + + CapacityToActivityUnit.csv + """ + return pd.read_csv(f) + +def import_cap_cost_base(f: str) -> pd.DataFrame: + """Imports CapitalCost.csv as output from the Powerplant rule. + + CapitalCost.csv + """ + return pd.read_csv(f) + +def import_fix_cost_base(f: str) -> pd.DataFrame: + """Imports FixedCost.csv as output from the Powerplant rule. + + FixedCost.csv + """ + return pd.read_csv(f) + +def import_op_life_base(f: str) -> pd.DataFrame: + """Imports OperationalLife.csv as output from the Powerplant rule. + + OperationalLife.csv + """ + return pd.read_csv(f) + +def import_max_cap_invest_base(f: str) -> pd.DataFrame: + """Imports TotalAnnualMaxCapacityInvestment.csv as output from the Powerplant rule. + + TotalAnnualMaxCapacityInvestment.csv + """ + return pd.read_csv(f) + +def import_min_cap_invest_base(f: str) -> pd.DataFrame: + """Imports TotalAnnualMinCapacityInvestment.csv as output from the Powerplant rule. + + TotalAnnualMinCapacityInvestment.csv + """ + return pd.read_csv(f) + +def import_res_cap_base(f: str) -> pd.DataFrame: + """Imports ResidualCapacity.csv as output from the Powerplant rule. + + ResidualCapacity.csv + """ + return pd.read_csv(f) + +def import_tech_set_base(f: str) -> pd.DataFrame: + """Imports TECHNOLOGY.csv as output from the Powerplant rule. + + TECHNOLOGY.csv + """ + return pd.read_csv(f) \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/sets.py b/workflow/scripts/osemosys_global/transmission/sets.py new file mode 100644 index 00000000..4067fbd6 --- /dev/null +++ b/workflow/scripts/osemosys_global/transmission/sets.py @@ -0,0 +1,11 @@ +"""Function to create sets.""" + +from data import format_transmission_name + +def create_tech_set_trn(df_lines): + + df = df_lines[['From', 'To']].copy() + df = format_transmission_name(df) + df = df.rename(columns = {'TECHNOLOGY' : 'VALUE'}) + + return df \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py b/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py new file mode 100644 index 00000000..eca9db15 --- /dev/null +++ b/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py @@ -0,0 +1,244 @@ +"""Function to integrate user defined capacities for transmission.""" + +import pandas as pd +import itertools + +from data import get_years + +def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_tech_set, + df_min_cap_invest, df_max_cap_invest, df_res_cap, + df_iar_final, df_oar_final, op_life_base, cap_act_base, + cap_cost_base, start_year, end_year, region_name, + df_iar_custom_val, df_oar_custom_val): + + techCapacity_trn = [] + tech_capacity_trn_dict = {} + first_year_dict = {} + build_rate_dict = {} + capex_dict = {} + build_year_dict = {} + + if not tech_capacity_trn is None: + + for tech, tech_params in tech_capacity_trn.items(): + techCapacity_trn.append([tech, tech_params[0], tech_params[1]]) + tech_capacity_trn_dict[tech] = tech_params[2] #UNUSED ENTRY + build_year_dict[tech] = tech_params[1] + first_year_dict[tech] = tech_params[3] + build_rate_dict[tech] = tech_params[4] + capex_dict[tech] = tech_params[5] + tech_capacity_trn_df = pd.DataFrame(techCapacity_trn, + columns=['TECHNOLOGY', 'VALUE', 'YEAR']) + tech_capacity_trn_df['REGION'] = region_name + tech_capacity_trn_df = tech_capacity_trn_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] + + for each_tech in list(tech_capacity_trn_df['TECHNOLOGY'].unique()): + if each_tech not in list(df_tech_set['VALUE']): + df_tech_set = pd.concat([df_tech_set, pd.DataFrame({'VALUE':[each_tech]})]) + + df_tech_set.drop_duplicates(inplace=True) + + df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_trn_df]) + df_min_cap_inv.drop_duplicates(inplace=True) + + max_cap_techs_df = pd.DataFrame(list(itertools.product(list(tech_capacity_trn_df['TECHNOLOGY'].unique()), + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'YEAR'] + ) + max_cap_techs_df['REGION'] = region_name + max_cap_techs_df = pd.merge(max_cap_techs_df, df_min_cap_inv, + how='left', + on=['REGION', 'TECHNOLOGY', 'YEAR']) + max_cap_techs_df['FIRST_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(first_year_dict) + max_cap_techs_df['BUILD_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(build_year_dict) + max_cap_techs_df['MAX_BUILD'] = max_cap_techs_df['TECHNOLOGY'].map(build_rate_dict) + + # Fill VALUE with MAX_BUILD for YEAR >= FIRST_YEAR + max_cap_techs_df.loc[(max_cap_techs_df['YEAR']>=max_cap_techs_df['FIRST_YEAR']) & + (max_cap_techs_df['YEAR']>max_cap_techs_df['BUILD_YEAR']), + 'VALUE'] = max_cap_techs_df['MAX_BUILD'] + max_cap_techs_df.infer_objects().fillna(0, + inplace=True) + + max_cap_techs_df = max_cap_techs_df[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + + # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD + df_max_cap_inv = pd.concat([df_max_cap_invest, max_cap_techs_df]).drop_duplicates() + + # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD + df_max_cap_inv.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'YEAR'], + keep='last', + inplace=True) + + # For technologies with start year before model start year, add to ResidualCapacity + df_res_cap_ud = df_min_cap_inv.copy().loc[df_min_cap_inv.copy()['YEAR'] < min(get_years(start_year, end_year))] + df_res_cap_ud.rename(columns={'YEAR':'START_YEAR'}, + inplace=True) + df_res_cap_ud_final = pd.DataFrame(list(itertools.product(df_res_cap_ud['TECHNOLOGY'].unique(), + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'YEAR'] + ) + df_res_cap_ud_final = pd.merge(df_res_cap_ud_final, + df_res_cap_ud, + how='left', + on=['TECHNOLOGY']) + df_res_cap_ud_final['TECH'] = df_res_cap_ud_final['TECHNOLOGY'].str[3:6] + df_res_cap_ud_final.loc[df_res_cap_ud_final['TECHNOLOGY'].str.contains('TRN'), + 'TECH'] = 'TRN' + df_res_cap_ud_final['OP_LIFE'] = df_res_cap_ud_final['TECH'].map(op_life_dict) + df_res_cap_ud_final['END_YEAR'] = (df_res_cap_ud_final['OP_LIFE'] + + df_res_cap_ud_final['START_YEAR']) + df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] + >= df_res_cap_ud_final['START_YEAR']] + df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] + <= df_res_cap_ud_final['END_YEAR']] + df_res_cap_ud_final['REGION'] = region_name + df_res_cap_ud_final = df_res_cap_ud_final[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + + df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final + if not df_res_cap_ud_final.empty else None]) + + # For technologies with start year at or after model start year, add to + # TotalAnnualMinCapacityInvestment + df_min_cap_inv = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] >= min(get_years(start_year, end_year))] + df_min_cap_inv.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'YEAR'], + keep='last', + inplace=True) + + # Add IAR and OAR for custom technologies + tech_list = list(tech_capacity_trn_df['TECHNOLOGY'].unique()) + df_iar_custom = pd.DataFrame(list(itertools.product(tech_list, + [1, 2], + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'MODE_OF_OPERATION', + 'YEAR'] + ) + df_oar_custom = pd.DataFrame(list(itertools.product(tech_list, + [1, 2], + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'MODE_OF_OPERATION', + 'YEAR'] + ) + # IAR in modes 1 and 2 are primary electricity commodity ('ELC*01') in + # node_from and node_to, respectively. + # OAR is the inverse of the above + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, + 'FUEL'] = ('ELC' + + df_iar_custom['TECHNOLOGY'].str[3:8] + + '01') + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, + 'FUEL'] = ('ELC' + + df_iar_custom['TECHNOLOGY'].str[8:13] + + '01') + df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, + 'FUEL'] = ('ELC' + + df_oar_custom['TECHNOLOGY'].str[8:13] + + '01') + df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, + 'FUEL'] = ('ELC' + + df_oar_custom['TECHNOLOGY'].str[3:8] + + '01') + df_iar_custom['VALUE'] = df_iar_custom_val + df_oar_custom['VALUE'] = df_oar_custom_val + df_iar_custom['REGION'] = region_name + df_oar_custom['REGION'] = region_name + + df_iar_custom = df_iar_custom[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + df_oar_custom = df_oar_custom[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + + df_iar = pd.concat([df_iar_final, df_iar_custom]) + df_oar = pd.concat([df_oar_final, df_oar_custom]) + + df_iar.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', + inplace=True) + + df_oar.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', + inplace=True) + + op_life_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) + + op_life_custom.loc[op_life_custom['TECHNOLOGY'].str.contains('TRN'), + 'VALUE'] = op_life_dict.get('TRN') + op_life_custom['REGION'] = region_name + op_life_custom = op_life_custom[['REGION', + 'TECHNOLOGY', + 'VALUE']] + + op_life = pd.concat([op_life_base, op_life_custom]) + op_life.drop_duplicates(subset=['REGION', + 'TECHNOLOGY'], + keep='last', + inplace=True) + + cap_act_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) + cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('TRN'), + 'VALUE'] = 31.536 + cap_act_custom['REGION'] = region_name + cap_act_custom = cap_act_custom[['REGION', + 'TECHNOLOGY', + 'VALUE']] + cap_act = pd.concat([cap_act_base, cap_act_custom]) + cap_act.drop_duplicates(inplace=True) + + # Update CapitalCost with user-defined costs by transmission line + tech_list = list(tech_capacity_trn_df['TECHNOLOGY'].unique()) + cap_cost_trn = pd.DataFrame(list(itertools.product(tech_list, + get_years(start_year, end_year))), + columns = ['TECHNOLOGY', + 'YEAR']) + + for each_trn in tech_list: + cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.startswith(each_trn), + 'VALUE'] = capex_dict[each_trn] + + cap_cost_trn['REGION'] = region_name + cap_cost_trn = cap_cost_trn[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + cap_cost = pd.concat([cap_cost_base, cap_cost_trn]) + cap_cost.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], + keep="last", + inplace=True) + + return(df_tech_set, df_max_cap_inv, df_min_cap_inv, + df_res_cap, df_iar, df_oar, op_life, cap_act, cap_cost) + \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/utils.py b/workflow/scripts/osemosys_global/transmission/utils.py new file mode 100644 index 00000000..336101da --- /dev/null +++ b/workflow/scripts/osemosys_global/transmission/utils.py @@ -0,0 +1,21 @@ +"""Utility Functions""" + +import pandas as pd +from typing import Optional +from constants import SET_DTYPES + +import logging +logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) + +def apply_dtypes(df:pd.DataFrame, name: Optional[str]) -> pd.DataFrame: + """Sets datatypes on dataframe""" + + for col in df.columns: + try: + df[col] = df[col].astype(SET_DTYPES[col]) + except KeyError: + if name: + logging.info(f"Can not set dtype for {name} on {col}") + else: + logging.info(f"Can not set dtype on {col}") + return df From 754705ef63c6c555d0f92b1f68a40c6a3eb398d7 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sat, 5 Oct 2024 20:56:53 -0700 Subject: [PATCH 08/12] update file check --- workflow/rules/preprocess.smk | 59 ++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/workflow/rules/preprocess.smk b/workflow/rules/preprocess.smk index 2cec1d5e..1a645709 100644 --- a/workflow/rules/preprocess.smk +++ b/workflow/rules/preprocess.smk @@ -15,35 +15,35 @@ demand_figures = [ # output script files power_plant_files = [ - 'powerplant/CapitalCost.csv', - 'powerplant/FixedCost.csv', - 'powerplant/CapacityToActivityUnit.csv', - 'powerplant/OperationalLife.csv', - 'powerplant/TotalAnnualMaxCapacityInvestment.csv', - 'powerplant/TotalAnnualMinCapacityInvestment.csv', - 'FUEL.csv', - 'powerplant/InputActivityRatio.csv', - 'powerplant/OutputActivityRatio.csv', - 'MODE_OF_OPERATION.csv', - 'REGION.csv', - 'powerplant/ResidualCapacity.csv', - 'powerplant/TECHNOLOGY.csv', - 'YEAR.csv', - 'AvailabilityFactor.csv' + 'powerplant/CapitalCost', + 'powerplant/FixedCost', + 'powerplant/CapacityToActivityUnit', + 'powerplant/OperationalLife', + 'powerplant/TotalAnnualMaxCapacityInvestment', + 'powerplant/TotalAnnualMinCapacityInvestment', + 'FUEL', + 'powerplant/InputActivityRatio', + 'powerplant/OutputActivityRatio', + 'MODE_OF_OPERATION', + 'REGION', + 'powerplant/ResidualCapacity', + 'powerplant/TECHNOLOGY', + 'YEAR', + 'AvailabilityFactor' ] transmission_files = [ - 'CapitalCost.csv', - 'FixedCost.csv', - 'CapacityToActivityUnit.csv', - 'OperationalLife.csv', - 'TotalAnnualMaxCapacityInvestment.csv', - 'TotalAnnualMinCapacityInvestment.csv', - 'TotalTechnologyModelPeriodActivityUpperLimit.csv', - 'InputActivityRatio.csv', - 'OutputActivityRatio.csv', - 'ResidualCapacity.csv', - 'TECHNOLOGY.csv', + 'CapitalCost', + 'FixedCost', + 'CapacityToActivityUnit', + 'OperationalLife', + 'TotalAnnualMaxCapacityInvestment', + 'TotalAnnualMinCapacityInvestment', + 'TotalTechnologyModelPeriodActivityUpperLimit', + 'InputActivityRatio', + 'OutputActivityRatio', + 'ResidualCapacity', + 'TECHNOLOGY', ] timeslice_files = [ @@ -95,9 +95,10 @@ user_capacity_files = [ ] GENERATED_CSVS = ( - power_plant_files + timeslice_files + variable_cost_files + demand_files \ - + emission_files + max_capacity_files + power_plant_files + transmission_files + timeslice_files + variable_cost_files \ + + demand_files + emission_files + max_capacity_files ) +GENERATED_CSVS = [Path(x).stem for x in GENERATED_CSVS] EMPTY_CSVS = [x for x in OTOOLE_PARAMS if x not in GENERATED_CSVS] # rules @@ -163,7 +164,7 @@ rule transmission: input_data_dir = 'resources/data', powerplant_data_dir = 'results/data/powerplant', output: - csv_files = expand('results/data/{output_file}', output_file = transmission_files) + csv_files = expand('results/data/{output_file}.csv', output_file = transmission_files) log: log = 'results/logs/transmission.log' script: From ea6f89aea8a4bfbe53f87e403fc205f03a732a1b Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 6 Oct 2024 16:19:32 -0700 Subject: [PATCH 09/12] correct residual capacity concat --- .../powerplant/user_defined_capacity.py | 5 +++-- .../scripts/osemosys_global/transmission/main.py | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py index eca62469..27b3b50d 100644 --- a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py +++ b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py @@ -106,8 +106,9 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, 'YEAR', 'VALUE']] - df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final - if not df_res_cap_ud_final.empty else None]) + if not df_res_cap_ud_final.empty: + df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final]) + df_res_cap = df_res_cap.groupby(by=["REGION", "TECHNOLOGY", "YEAR"], as_index=False).sum() # For technologies with start year at or after model start year, add to # TotalAnnualMinCapacityInvestment diff --git a/workflow/scripts/osemosys_global/transmission/main.py b/workflow/scripts/osemosys_global/transmission/main.py index 0410dce2..e6b01548 100644 --- a/workflow/scripts/osemosys_global/transmission/main.py +++ b/workflow/scripts/osemosys_global/transmission/main.py @@ -196,12 +196,12 @@ def main( output_data_dir = 'results/data' input_data_dir = 'resources/data' powerplant_data_dir = 'results/data/powerplant' - file_iar_base = f'{powerplant_data_dir}/InputActivityRatio' - file_oar_base = f'{powerplant_data_dir}/OutputActivityRatio' - file_capact_base = f'{powerplant_data_dir}/CapacityToActivityUnit' - file_cap_cost_base = f'{powerplant_data_dir}/CapitalCost' - file_fix_cost_base = f'{powerplant_data_dir}/FixedCost' - file_op_life_base = f'{powerplant_data_dir}/OperationalLife' + file_iar_base = f'{powerplant_data_dir}/InputActivityRatio.csv' + file_oar_base = f'{powerplant_data_dir}/OutputActivityRatio.csv' + file_capact_base = f'{powerplant_data_dir}/CapacityToActivityUnit.csv' + file_cap_cost_base = f'{powerplant_data_dir}/CapitalCost.csv' + file_fix_cost_base = f'{powerplant_data_dir}/FixedCost.csv' + file_op_life_base = f'{powerplant_data_dir}/OperationalLife.csv' file_max_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMaxCapacityInvestment.csv' file_min_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMinCapacityInvestment.csv' file_res_cap_base = f'{powerplant_data_dir}/ResidualCapacity.csv' From a85e95bd32bbfbed1d62b832815ad99d1ec4e01d Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 6 Oct 2024 20:13:45 -0700 Subject: [PATCH 10/12] fix udc powerplant fuels --- .../powerplant/user_defined_capacity.py | 29 +++++++------------ workflow/snakefile | 2 +- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py index 27b3b50d..ca5f133c 100644 --- a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py +++ b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py @@ -137,25 +137,16 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, 'MODE_OF_OPERATION', 'YEAR'] ) - # IAR in modes 1 and 2 are primary electricity commodity ('ELC*01') in - # node_from and node_to, respectively. - # OAR is the inverse of the above - df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, - 'FUEL'] = ('ELC' + - df_iar_custom['TECHNOLOGY'].str[3:8] + - '01') - df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, - 'FUEL'] = ('ELC' + - df_iar_custom['TECHNOLOGY'].str[8:13] + - '01') - df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, - 'FUEL'] = ('ELC' + - df_oar_custom['TECHNOLOGY'].str[8:13] + - '01') - df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, - 'FUEL'] = ('ELC' + - df_oar_custom['TECHNOLOGY'].str[3:8] + - '01') + + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1,'FUEL'] = ( + df_iar_custom['TECHNOLOGY'].str[3:9]) + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2,'FUEL'] = ( + df_iar_custom['TECHNOLOGY'].str[3:6] + "INT") + df_oar_custom.loc[df_oar_custom['MODE_OF_OPERATION']==1,'FUEL'] = ( + 'ELC' + df_oar_custom['TECHNOLOGY'].str[6:11] + '01') + df_oar_custom.loc[df_oar_custom['MODE_OF_OPERATION']==2,'FUEL'] = ( + 'ELC' + df_oar_custom['TECHNOLOGY'].str[6:11] + '01') + df_iar_custom['VALUE'] = DF_IAR_CUSTOM_VAL df_oar_custom['VALUE'] = DF_OAR_CUSTOM_VAL df_iar_custom['REGION'] = region_name diff --git a/workflow/snakefile b/workflow/snakefile index aed6d03e..ab2c20bb 100644 --- a/workflow/snakefile +++ b/workflow/snakefile @@ -85,7 +85,7 @@ rule generate_input_data: message: "Generating input CSV data..." input: - csv_files = expand('results/{scenario}/data/{csv}', scenario=config['scenario'], csv=OTOOLE_PARAMS), + csv_files = expand('results/{scenario}/data/{csv}.csv', scenario=config['scenario'], csv=OTOOLE_PARAMS), rule make_dag: message: From 25528c7333e2c9035e29d2d3092219259dfb84cd Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sun, 6 Oct 2024 22:58:35 -0700 Subject: [PATCH 11/12] out of domain error fixed --- workflow/rules/preprocess.smk | 3 +- .../osemosys_global/powerplant/main.py | 4 +- .../osemosys_global/transmission/activity.py | 31 ++--- .../osemosys_global/transmission/main.py | 116 ++++++++++-------- .../osemosys_global/transmission/read.py | 7 +- .../osemosys_global/transmission/sets.py | 24 +++- .../transmission/user_defined_capacity.py | 24 +--- 7 files changed, 108 insertions(+), 101 deletions(-) diff --git a/workflow/rules/preprocess.smk b/workflow/rules/preprocess.smk index 1a645709..b53bc5c1 100644 --- a/workflow/rules/preprocess.smk +++ b/workflow/rules/preprocess.smk @@ -21,7 +21,7 @@ power_plant_files = [ 'powerplant/OperationalLife', 'powerplant/TotalAnnualMaxCapacityInvestment', 'powerplant/TotalAnnualMinCapacityInvestment', - 'FUEL', + 'powerplant/FUEL', 'powerplant/InputActivityRatio', 'powerplant/OutputActivityRatio', 'MODE_OF_OPERATION', @@ -44,6 +44,7 @@ transmission_files = [ 'OutputActivityRatio', 'ResidualCapacity', 'TECHNOLOGY', + 'FUEL' ] timeslice_files = [ diff --git a/workflow/scripts/osemosys_global/powerplant/main.py b/workflow/scripts/osemosys_global/powerplant/main.py index 5ae82a39..80ed15a4 100644 --- a/workflow/scripts/osemosys_global/powerplant/main.py +++ b/workflow/scripts/osemosys_global/powerplant/main.py @@ -204,12 +204,12 @@ def main( tech_set.to_csv(os.path.join(powerplant_data_dir, "TECHNOLOGY.csv"), index = None) + fuel_set.to_csv(os.path.join(powerplant_data_dir, "FUEL.csv"), index = None) + # OUTPUT CSV's NOT USED AS INPUT FOR TRANMISSION RULE df_af_final.to_csv(os.path.join(output_data_dir, 'AvailabilityFactor.csv'), index=None) - fuel_set.to_csv(os.path.join(output_data_dir, "FUEL.csv"), index = None) - years_set.to_csv(os.path.join(output_data_dir, "YEAR.csv"), index = None) mode_list_set.to_csv(os.path.join(output_data_dir, "MODE_OF_OPERATION.csv"), index = None) diff --git a/workflow/scripts/osemosys_global/transmission/activity.py b/workflow/scripts/osemosys_global/transmission/activity.py index 8cd47547..97535f93 100644 --- a/workflow/scripts/osemosys_global/transmission/activity.py +++ b/workflow/scripts/osemosys_global/transmission/activity.py @@ -1,8 +1,8 @@ """Function to calaculate activity related to transmission.""" -import pandas as pd +from typing import Optional +import pandas as pd from data import format_transmission_name - from utils import apply_dtypes def activity_transmission(df_iar_base, df_oar_base, df_pw_prop, df_trn_efficiencies, @@ -218,19 +218,20 @@ def activity_transmission_limit(cross_border_trade, df_oar_trn_final): return df_crossborder_final -def capact_transmission(df_capact_base, df_oar_trn_final): - - # Create CapacityToActivityUnit csv - df_capact_trn_final = df_oar_trn_final[['REGION', - 'TECHNOLOGY' - ]] - df_capact_trn_final = df_capact_trn_final.drop_duplicates() - df_capact_trn_final = df_capact_trn_final.loc[df_capact_trn_final['TECHNOLOGY' - ].str.startswith('TRN')] - df_capact_trn_final['VALUE'] = 31.536 - df_capact_trn_final.drop_duplicates(inplace=True) +def create_trn_dist_capacity_activity(*dfs: pd.DataFrame, value: Optional[float] = 31.536, region: Optional[str] = "GLOBAL") -> pd.DataFrame: + """Creates tranmission and distribution capacity to activity unit data - df_capact_trn_final = pd.concat([df_capact_base, df_capact_trn_final]) + Inputs are any number of dataframes with a TECHNOLOGY colums. + """ - return df_capact_trn_final \ No newline at end of file + techs = [] + for df in dfs: + temp = df[(df.TECHNOLOGY.str.startswith("TRN")) | (df.TECHNOLOGY.str.startswith("PWRTRN"))] + techs += temp.TECHNOLOGY.to_list() + + data = [] + for tech in set(techs): + data.append([region, tech, value]) + + return pd.DataFrame(data, columns=["REGION", "TECHNOLOGY", "VALUE"]) diff --git a/workflow/scripts/osemosys_global/transmission/main.py b/workflow/scripts/osemosys_global/transmission/main.py index e6b01548..f59219ab 100644 --- a/workflow/scripts/osemosys_global/transmission/main.py +++ b/workflow/scripts/osemosys_global/transmission/main.py @@ -14,7 +14,7 @@ import_max_cap_invest_base, import_min_cap_invest_base, import_res_cap_base, - import_tech_set_base + import_set_base ) from constants import( @@ -25,7 +25,7 @@ from activity import( activity_transmission, activity_transmission_limit, - capact_transmission + create_trn_dist_capacity_activity ) from costs import get_transmission_costs @@ -36,7 +36,7 @@ from user_defined_capacity import set_user_defined_capacity_trn -from sets import create_tech_set_trn +from sets import create_set_from_iterators, get_unique_fuels, get_unique_technologies def main( plexos_prop: pd.DataFrame, @@ -53,84 +53,91 @@ def main( min_cap_invest_base: pd.DataFrame, res_cap_base: pd.DataFrame, tech_set_base: pd.DataFrame, + fuel_set_base: pd.DataFrame ): # CALL FUNCTIONS # Set activity ratios for transmission. - df_iar_trn_final, df_oar_trn_final = activity_transmission(iar_base, oar_base, + iar_trn, oar_trn = activity_transmission(iar_base, oar_base, plexos_prop, interface_data, start_year, end_year, region_name) + # Adjust activity limits if cross border trade is not allowed following user config. - df_crossborder_final = activity_transmission_limit(cross_border_trade, df_oar_trn_final) - - # Set CapacityToActivityUnit. - df_capact_trn_final = capact_transmission(capact_base, df_oar_trn_final) + df_crossborder_final = activity_transmission_limit(cross_border_trade, oar_trn) # Set capital and fixed transmission costs. - df_cap_cost_trn_final, df_fix_cost_trn_final = get_transmission_costs(line_data, df_oar_trn_final, + df_cap_cost_trn_final, df_fix_cost_trn_final = get_transmission_costs(line_data, oar_trn, cap_cost_base, fix_cost_base, start_year, end_year, region_name) # Set operational life for transmission. - df_op_life_trn_final = set_op_life_transmission(df_iar_trn_final, df_oar_trn_final, + df_op_life_trn_final = set_op_life_transmission(iar_trn, oar_trn, op_life_dict, op_life_base, region_name) # Set annual capacity investment constraints. - df_max_cap_invest_trn_final = cap_investment_constraints_trn(df_iar_trn_final, + df_max_cap_invest_trn_final = cap_investment_constraints_trn(iar_trn, max_cap_invest_base, no_investment_techs, start_year, end_year, region_name) - - tech_set_trn_final = pd.concat([tech_set, create_tech_set_trn(interface_data)]) # Alter output csv's based on user defined capacities following user config. if not tech_capacity_trn is None: - (tech_set_trn_final, - df_max_cap_invest_trn_final, + (df_max_cap_invest_trn_final, df_min_cap_invest_trn_final, df_res_cap_trn_final, - df_iar_trn_final, - df_oar_trn_final, + iar_trn, + oar_trn, df_op_life_trn_final, - df_capact_trn_final, df_cap_cost_trn_final ) = set_user_defined_capacity_trn( - tech_capacity_trn, - op_life_dict, - tech_set_trn_final, - min_cap_invest_base, - df_max_cap_invest_trn_final, - res_cap_base, - df_iar_trn_final, - df_oar_trn_final, - df_op_life_trn_final, - df_capact_trn_final, - df_cap_cost_trn_final, - start_year, - end_year, - region_name, - DF_IAR_CUSTOM_VAL, - DF_OAR_CUSTOM_VAL - ) + tech_capacity_trn, + op_life_dict, + min_cap_invest_base, + df_max_cap_invest_trn_final, + res_cap_base, + iar_trn, + oar_trn, + df_op_life_trn_final, + df_cap_cost_trn_final, + start_year, + end_year, + region_name, + DF_IAR_CUSTOM_VAL, + DF_OAR_CUSTOM_VAL + ) + # get new additions to fuel and technology sets + exising_techs = tech_set_base.VALUE.to_list() + iar_techs = get_unique_technologies(iar_trn) + oar_techs = get_unique_technologies(oar_trn) + tech_set = create_set_from_iterators(exising_techs, iar_techs, oar_techs) + + exising_fuels = fuel_set_base.VALUE.to_list() + iar_fuels = get_unique_fuels(iar_trn) + oar_fuels = get_unique_fuels(oar_trn) + fuel_set = create_set_from_iterators(exising_fuels, iar_fuels, oar_fuels) + + # assign capacity to activity unit to transmission + distribution techs + cap_activity_trn = create_trn_dist_capacity_activity(iar_trn, oar_trn) + cap_activity = pd.concat([capact_base, cap_activity_trn]).drop_duplicates(subset=["REGION", "TECHNOLOGY"], keep="last") + # OUTPUT CSV's - df_oar_trn_final.to_csv(os.path.join(output_data_dir, "OutputActivityRatio.csv"), index=None) + oar_trn.to_csv(os.path.join(output_data_dir, "OutputActivityRatio.csv"), index=None) - df_iar_trn_final.to_csv(os.path.join(output_data_dir, "InputActivityRatio.csv"), index=None) + iar_trn.to_csv(os.path.join(output_data_dir, "InputActivityRatio.csv"), index=None) df_crossborder_final.to_csv(os.path.join(output_data_dir, "TotalTechnologyModelPeriodActivityUpperLimit.csv"), index = None) - df_capact_trn_final.to_csv(os.path.join(output_data_dir, - "CapacityToActivityUnit.csv"), index = None) + cap_activity.to_csv(os.path.join(output_data_dir, "CapacityToActivityUnit.csv"), index = None) df_cap_cost_trn_final.to_csv(os.path.join(output_data_dir, "CapitalCost.csv"), index = None) @@ -142,7 +149,8 @@ def main( 'TotalAnnualMaxCapacityInvestment.csv'), index = None) - tech_set_trn_final.to_csv(os.path.join(output_data_dir, "TECHNOLOGY.csv"), index = None) + tech_set.to_csv(os.path.join(output_data_dir, "TECHNOLOGY.csv"), index = None) + fuel_set.to_csv(os.path.join(output_data_dir, "FUEL.csv"), index = None) if not tech_capacity_trn is None: @@ -180,6 +188,7 @@ def main( file_min_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMinCapacityInvestment.csv' file_res_cap_base = f'{powerplant_data_dir}/ResidualCapacity.csv' file_tech_set = f'{powerplant_data_dir}/TECHNOLOGY.csv' + file_fuel_set = f'{powerplant_data_dir}/FUEL.csv' else: file_plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx' @@ -205,7 +214,8 @@ def main( file_max_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMaxCapacityInvestment.csv' file_min_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMinCapacityInvestment.csv' file_res_cap_base = f'{powerplant_data_dir}/ResidualCapacity.csv' - file_tech_set = f'{powerplant_data_dir}/TECHNOLOGY.csv' + file_tech_set = f'{powerplant_data_dir}/TECHNOLOGY.csv' + file_fuel_set = f'{powerplant_data_dir}/FUEL.csv' # SET INPUT DATA plexos_prop = import_plexos_2015(file_plexos, "prop") @@ -224,23 +234,25 @@ def main( max_cap_invest_base = import_max_cap_invest_base(file_max_cap_invest_base) min_cap_invest_base = import_min_cap_invest_base(file_min_cap_invest_base) res_cap_base = import_res_cap_base(file_res_cap_base) - tech_set = import_tech_set_base(file_tech_set) + tech_set_base = import_set_base(file_tech_set) + fuel_set_base = import_set_base(file_tech_set) input_data = { "plexos_prop": plexos_prop, "default_op_life": op_life, "line_data": trn_line, "interface_data": trn_interface, - "iar_base" : iar_base, - "oar_base" : oar_base, - "capact_base" : capact_base, - "cap_cost_base" : cap_cost_base, - "fix_cost_base" : fix_cost_base, - "op_life_base" : op_life_base, - "max_cap_invest_base" : max_cap_invest_base, - "min_cap_invest_base" : min_cap_invest_base, - "res_cap_base" : res_cap_base, - "tech_set_base" : tech_set, + "iar_base" : iar_base, + "oar_base" : oar_base, + "capact_base" : capact_base, + "cap_cost_base" : cap_cost_base, + "fix_cost_base" : fix_cost_base, + "op_life_base" : op_life_base, + "max_cap_invest_base" : max_cap_invest_base, + "min_cap_invest_base" : min_cap_invest_base, + "res_cap_base" : res_cap_base, + "tech_set_base" : tech_set_base, + "fuel_set_base" : fuel_set_base, } # CALL MAIN diff --git a/workflow/scripts/osemosys_global/transmission/read.py b/workflow/scripts/osemosys_global/transmission/read.py index 6ad7066c..f3bfc32f 100644 --- a/workflow/scripts/osemosys_global/transmission/read.py +++ b/workflow/scripts/osemosys_global/transmission/read.py @@ -100,9 +100,6 @@ def import_res_cap_base(f: str) -> pd.DataFrame: """ return pd.read_csv(f) -def import_tech_set_base(f: str) -> pd.DataFrame: - """Imports TECHNOLOGY.csv as output from the Powerplant rule. - - TECHNOLOGY.csv - """ +def import_set_base(f: str) -> pd.DataFrame: + """Imports a set csv""" return pd.read_csv(f) \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/sets.py b/workflow/scripts/osemosys_global/transmission/sets.py index 4067fbd6..805f007d 100644 --- a/workflow/scripts/osemosys_global/transmission/sets.py +++ b/workflow/scripts/osemosys_global/transmission/sets.py @@ -1,11 +1,23 @@ """Function to create sets.""" -from data import format_transmission_name +import pandas as pd -def create_tech_set_trn(df_lines): +def get_unique_technologies(ar: pd.DataFrame) -> list[str]: + """Gets unique technologies from activity ratio dataframe""" + return ar.TECHNOLOGY.unique().tolist() - df = df_lines[['From', 'To']].copy() - df = format_transmission_name(df) - df = df.rename(columns = {'TECHNOLOGY' : 'VALUE'}) +def get_unique_fuels(ar: pd.DataFrame) -> list[str]: + """Gets unique fuels from activity ratio dataframe""" + return ar.FUEL.unique().tolist() + +def create_set_from_iterators(*args: list[str] | set[str]) -> pd.DataFrame: + """Creates a set formatted dataframe from an arbitrary number of arguments""" + + data = [] + + for arg in args: + data += arg + + data = list(set(data)) - return df \ No newline at end of file + return pd.Series(data, name="VALUE").to_frame() \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py b/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py index eca9db15..88f34f50 100644 --- a/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py +++ b/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py @@ -5,9 +5,9 @@ from data import get_years -def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_tech_set, +def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_min_cap_invest, df_max_cap_invest, df_res_cap, - df_iar_final, df_oar_final, op_life_base, cap_act_base, + df_iar_final, df_oar_final, op_life_base, cap_cost_base, start_year, end_year, region_name, df_iar_custom_val, df_oar_custom_val): @@ -32,12 +32,6 @@ def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_tech_set, tech_capacity_trn_df['REGION'] = region_name tech_capacity_trn_df = tech_capacity_trn_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - for each_tech in list(tech_capacity_trn_df['TECHNOLOGY'].unique()): - if each_tech not in list(df_tech_set['VALUE']): - df_tech_set = pd.concat([df_tech_set, pd.DataFrame({'VALUE':[each_tech]})]) - - df_tech_set.drop_duplicates(inplace=True) - df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_trn_df]) df_min_cap_inv.drop_duplicates(inplace=True) @@ -207,16 +201,6 @@ def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_tech_set, 'TECHNOLOGY'], keep='last', inplace=True) - - cap_act_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) - cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('TRN'), - 'VALUE'] = 31.536 - cap_act_custom['REGION'] = region_name - cap_act_custom = cap_act_custom[['REGION', - 'TECHNOLOGY', - 'VALUE']] - cap_act = pd.concat([cap_act_base, cap_act_custom]) - cap_act.drop_duplicates(inplace=True) # Update CapitalCost with user-defined costs by transmission line tech_list = list(tech_capacity_trn_df['TECHNOLOGY'].unique()) @@ -239,6 +223,6 @@ def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_tech_set, keep="last", inplace=True) - return(df_tech_set, df_max_cap_inv, df_min_cap_inv, - df_res_cap, df_iar, df_oar, op_life, cap_act, cap_cost) + return(df_max_cap_inv, df_min_cap_inv, + df_res_cap, df_iar, df_oar, op_life, cap_cost) \ No newline at end of file From dcf3edb09dbfa9a337b4ea6f39677277d62a5767 Mon Sep 17 00:00:00 2001 From: maartenbrinkerink <65602545+maartenbrinkerink@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:10:33 -0400 Subject: [PATCH 12/12] PR review updates -Updates following @trevorb1 PR review comments -Fixed user defined IAR bug following user defined capacities for transmission -Added comment in main scripts that config variables defined locally are for testing purposes only -Compared results to master which are in line with small differences possibly as a result of the changes in the config file (user defined capacities separated for transmission and powerplants) --- .../osemosys_global/powerplant/main.py | 26 +- .../powerplant/residual_capacity.py | 3 +- .../powerplant/user_defined_capacity.py | 436 +++++++++--------- .../osemosys_global/transmission/main.py | 30 +- .../transmission/user_defined_capacity.py | 390 ++++++++-------- 5 files changed, 444 insertions(+), 441 deletions(-) diff --git a/workflow/scripts/osemosys_global/powerplant/main.py b/workflow/scripts/osemosys_global/powerplant/main.py index 80ed15a4..3be04f23 100644 --- a/workflow/scripts/osemosys_global/powerplant/main.py +++ b/workflow/scripts/osemosys_global/powerplant/main.py @@ -62,7 +62,9 @@ def main( weo_costs: pd.DataFrame, weo_regions: pd.DataFrame, default_op_life: pd.DataFrame, - naming_convention_tech: pd.DataFrame, + tech_code: pd.DataFrame, + tech_list: pd.DataFrame, + tech_code_dict: pd.DataFrame, custom_res_cap: pd.DataFrame, default_av_factors: pd.DataFrame, ): @@ -70,7 +72,7 @@ def main( # CALL FUNCTIONS # return generator_table - gen_table = set_generator_table(plexos_prop, plexos_memb, op_life_dict, + gen_table = set_generator_table(plexos_prop, plexos_memb, default_op_life, tech_code_dict, start_year, end_year) # Calculate average technology efficiencies. @@ -106,7 +108,7 @@ def main( # Set operational life for powerplant technologies. df_op_life = set_op_life(tech_code_dict, df_iar_final, - df_oar_final, op_life_dict, region_name) + df_oar_final, default_op_life, region_name) # Set annual capacity investment constraints. df_max_cap_invest, df_min_cap_invest = cap_investment_constraints(df_iar_final, @@ -115,9 +117,7 @@ def main( end_year, region_name) # Calculate residual capacity. - df_res_cap = res_capacity(gen_table, tech_list, tech_code, - DUPLICATE_TECHS, start_year, - end_year, region_name) + df_res_cap = res_capacity(gen_table, DUPLICATE_TECHS, start_year, end_year, region_name) if custom_nodes: @@ -156,7 +156,7 @@ def main( df_cap_cost_final ) = set_user_defined_capacity( tech_capacity, - op_life_dict, + default_op_life, tech_set, df_min_cap_invest, df_max_cap_invest, @@ -175,7 +175,7 @@ def main( ) # Set availability factors. Occurs after set_user_defined_capacity as tech_set gets updated. - df_af_final = availability_factor(availability, tech_set, + df_af_final = availability_factor(default_av_factors, tech_set, start_year, end_year, region_name) # OUTPUT CSV's USED AS INPUT FOR TRANSMISSION RULE @@ -238,6 +238,10 @@ def main( if custom_nodes: file_custom_res_cap = snakemake.input.custom_res_cap + # The below else statement defines variables if the 'powerplant/main' script is to be run locally + # outside the snakemake workflow. This is relevant for testing purposes only! User inputs when running + # the full workflow need to be defined in the config file. + else: file_plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx' file_default_op_life = 'resources/data/operational_life.csv' @@ -289,8 +293,10 @@ def main( "plexos_memb": plexos_memb, "weo_costs": weo_costs, "weo_regions": weo_regions, - "default_op_life": op_life, - "naming_convention_tech": tech_code, + "default_op_life": op_life_dict, + "tech_code": tech_code, + "tech_list": tech_list, + "tech_code_dict": tech_code_dict, "custom_res_cap" : custom_res_cap, "default_av_factors": availability, } diff --git a/workflow/scripts/osemosys_global/powerplant/residual_capacity.py b/workflow/scripts/osemosys_global/powerplant/residual_capacity.py index 8c2bf49a..cb17198f 100644 --- a/workflow/scripts/osemosys_global/powerplant/residual_capacity.py +++ b/workflow/scripts/osemosys_global/powerplant/residual_capacity.py @@ -8,8 +8,7 @@ get_years ) -def res_capacity(df_gen_base, tech_list, df_tech_code, duplicate_techs, - start_year, end_year, region_name): +def res_capacity(df_gen_base, duplicate_techs, start_year, end_year, region_name): # ### Calculate residual capacity res_cap_cols = [ diff --git a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py index ca5f133c..9a1f2b48 100644 --- a/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py +++ b/workflow/scripts/osemosys_global/powerplant/user_defined_capacity.py @@ -18,230 +18,228 @@ def set_user_defined_capacity(tech_capacity, op_life_dict, df_tech_set, build_rate_dict = {} capex_dict = {} build_year_dict = {} + + for tech, tech_params in tech_capacity.items(): + techCapacity.append([tech, tech_params[0], tech_params[1]]) + tech_capacity_dict[tech] = tech_params[2] #UNUSED ENTRY + build_year_dict[tech] = tech_params[1] + first_year_dict[tech] = tech_params[3] + build_rate_dict[tech] = tech_params[4] + capex_dict[tech] = tech_params[5] + tech_capacity_df = pd.DataFrame(techCapacity, + columns=['TECHNOLOGY', 'VALUE', 'YEAR']) + tech_capacity_df['REGION'] = region_name + tech_capacity_df = tech_capacity_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] + + for each_tech in list(tech_capacity_df['TECHNOLOGY'].unique()): + if each_tech not in list(df_tech_set['VALUE']): + df_tech_set = pd.concat([df_tech_set, pd.DataFrame({'VALUE':[each_tech]})]) + + df_tech_set.drop_duplicates(inplace=True) + + df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_df]) + df_min_cap_inv.drop_duplicates(inplace=True) + + max_cap_techs_df = pd.DataFrame(list(itertools.product(list(tech_capacity_df['TECHNOLOGY'].unique()), + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'YEAR'] + ) + max_cap_techs_df['REGION'] = region_name + max_cap_techs_df = pd.merge(max_cap_techs_df, df_min_cap_inv, + how='left', + on=['REGION', 'TECHNOLOGY', 'YEAR']) + max_cap_techs_df['FIRST_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(first_year_dict) + max_cap_techs_df['BUILD_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(build_year_dict) + max_cap_techs_df['MAX_BUILD'] = max_cap_techs_df['TECHNOLOGY'].map(build_rate_dict) + + # Fill VALUE with MAX_BUILD for YEAR >= FIRST_YEAR + max_cap_techs_df.loc[(max_cap_techs_df['YEAR']>=max_cap_techs_df['FIRST_YEAR']) & + (max_cap_techs_df['YEAR']>max_cap_techs_df['BUILD_YEAR']), + 'VALUE'] = max_cap_techs_df['MAX_BUILD'] + max_cap_techs_df.infer_objects().fillna(0, + inplace=True) - if not tech_capacity is None: - - for tech, tech_params in tech_capacity.items(): - techCapacity.append([tech, tech_params[0], tech_params[1]]) - tech_capacity_dict[tech] = tech_params[2] #UNUSED ENTRY - build_year_dict[tech] = tech_params[1] - first_year_dict[tech] = tech_params[3] - build_rate_dict[tech] = tech_params[4] - capex_dict[tech] = tech_params[5] - tech_capacity_df = pd.DataFrame(techCapacity, - columns=['TECHNOLOGY', 'VALUE', 'YEAR']) - tech_capacity_df['REGION'] = region_name - tech_capacity_df = tech_capacity_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - - for each_tech in list(tech_capacity_df['TECHNOLOGY'].unique()): - if each_tech not in list(df_tech_set['VALUE']): - df_tech_set = pd.concat([df_tech_set, pd.DataFrame({'VALUE':[each_tech]})]) - - df_tech_set.drop_duplicates(inplace=True) - - df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_df]) - df_min_cap_inv.drop_duplicates(inplace=True) - - max_cap_techs_df = pd.DataFrame(list(itertools.product(list(tech_capacity_df['TECHNOLOGY'].unique()), - get_years(start_year, end_year)) - ), - columns = ['TECHNOLOGY', - 'YEAR'] - ) - max_cap_techs_df['REGION'] = region_name - max_cap_techs_df = pd.merge(max_cap_techs_df, df_min_cap_inv, - how='left', - on=['REGION', 'TECHNOLOGY', 'YEAR']) - max_cap_techs_df['FIRST_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(first_year_dict) - max_cap_techs_df['BUILD_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(build_year_dict) - max_cap_techs_df['MAX_BUILD'] = max_cap_techs_df['TECHNOLOGY'].map(build_rate_dict) - - # Fill VALUE with MAX_BUILD for YEAR >= FIRST_YEAR - max_cap_techs_df.loc[(max_cap_techs_df['YEAR']>=max_cap_techs_df['FIRST_YEAR']) & - (max_cap_techs_df['YEAR']>max_cap_techs_df['BUILD_YEAR']), - 'VALUE'] = max_cap_techs_df['MAX_BUILD'] - max_cap_techs_df.infer_objects().fillna(0, - inplace=True) - - max_cap_techs_df = max_cap_techs_df[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] - - # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD - df_max_cap_inv = pd.concat([df_max_cap_invest, max_cap_techs_df]).drop_duplicates() - - # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD - df_max_cap_inv.drop_duplicates(subset=['REGION', + max_cap_techs_df = max_cap_techs_df[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + + # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD + df_max_cap_inv = pd.concat([df_max_cap_invest, max_cap_techs_df]).drop_duplicates() + + # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD + df_max_cap_inv.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'YEAR'], + keep='last', + inplace=True) + + # For technologies with start year before model start year, add to ResidualCapacity + df_res_cap_ud = df_min_cap_inv.copy().loc[df_min_cap_inv.copy()['YEAR'] < min(get_years(start_year, end_year))] + df_res_cap_ud.rename(columns={'YEAR':'start_year'}, + inplace=True) + df_res_cap_ud_final = pd.DataFrame(list(itertools.product(df_res_cap_ud['TECHNOLOGY'].unique(), + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'YEAR'] + ) + df_res_cap_ud_final = pd.merge(df_res_cap_ud_final, + df_res_cap_ud, + how='left', + on=['TECHNOLOGY']) + df_res_cap_ud_final['TECH'] = df_res_cap_ud_final['TECHNOLOGY'].str[3:6] + df_res_cap_ud_final['OP_LIFE'] = df_res_cap_ud_final['TECH'].map(op_life_dict) + df_res_cap_ud_final['end_year'] = (df_res_cap_ud_final['OP_LIFE'] + + df_res_cap_ud_final['start_year']) + df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] + >= df_res_cap_ud_final['start_year']] + df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] + <= df_res_cap_ud_final['end_year']] + df_res_cap_ud_final['REGION'] = region_name + df_res_cap_ud_final = df_res_cap_ud_final[['REGION', 'TECHNOLOGY', - 'YEAR'], - keep='last', - inplace=True) + 'YEAR', + 'VALUE']] + + if not df_res_cap_ud_final.empty: + df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final]) + df_res_cap = df_res_cap.groupby(by=["REGION", "TECHNOLOGY", "YEAR"], as_index=False).sum() + + # For technologies with start year at or after model start year, add to + # TotalAnnualMinCapacityInvestment + df_min_cap_inv = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] >= min(get_years(start_year, end_year))] + df_min_cap_inv.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'YEAR'], + keep='last', + inplace=True) + + tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) + + df_iar_custom = pd.DataFrame(list(itertools.product(tech_list, + [1, 2], + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'MODE_OF_OPERATION', + 'YEAR'] + ) + df_oar_custom = pd.DataFrame(list(itertools.product(tech_list, + [1, 2], + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'MODE_OF_OPERATION', + 'YEAR'] + ) + + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1,'FUEL'] = ( + df_iar_custom['TECHNOLOGY'].str[3:9]) + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2,'FUEL'] = ( + df_iar_custom['TECHNOLOGY'].str[3:6] + "INT") + df_oar_custom.loc[df_oar_custom['MODE_OF_OPERATION']==1,'FUEL'] = ( + 'ELC' + df_oar_custom['TECHNOLOGY'].str[6:11] + '01') + df_oar_custom.loc[df_oar_custom['MODE_OF_OPERATION']==2,'FUEL'] = ( + 'ELC' + df_oar_custom['TECHNOLOGY'].str[6:11] + '01') + + df_iar_custom['VALUE'] = DF_IAR_CUSTOM_VAL + df_oar_custom['VALUE'] = DF_OAR_CUSTOM_VAL + df_iar_custom['REGION'] = region_name + df_oar_custom['REGION'] = region_name + + df_iar_custom = df_iar_custom[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + df_oar_custom = df_oar_custom[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + + df_iar = pd.concat([df_iar_final, df_iar_custom]) + df_oar = pd.concat([df_oar_final, df_oar_custom]) + + df_iar.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', + inplace=True) - # For technologies with start year before model start year, add to ResidualCapacity - df_res_cap_ud = df_min_cap_inv.copy().loc[df_min_cap_inv.copy()['YEAR'] < min(get_years(start_year, end_year))] - df_res_cap_ud.rename(columns={'YEAR':'start_year'}, + df_oar.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', inplace=True) - df_res_cap_ud_final = pd.DataFrame(list(itertools.product(df_res_cap_ud['TECHNOLOGY'].unique(), - get_years(start_year, end_year)) - ), - columns = ['TECHNOLOGY', - 'YEAR'] - ) - df_res_cap_ud_final = pd.merge(df_res_cap_ud_final, - df_res_cap_ud, - how='left', - on=['TECHNOLOGY']) - df_res_cap_ud_final['TECH'] = df_res_cap_ud_final['TECHNOLOGY'].str[3:6] - df_res_cap_ud_final['OP_LIFE'] = df_res_cap_ud_final['TECH'].map(op_life_dict) - df_res_cap_ud_final['end_year'] = (df_res_cap_ud_final['OP_LIFE'] - + df_res_cap_ud_final['start_year']) - df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] - >= df_res_cap_ud_final['start_year']] - df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] - <= df_res_cap_ud_final['end_year']] - df_res_cap_ud_final['REGION'] = region_name - df_res_cap_ud_final = df_res_cap_ud_final[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] - - if not df_res_cap_ud_final.empty: - df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final]) - df_res_cap = df_res_cap.groupby(by=["REGION", "TECHNOLOGY", "YEAR"], as_index=False).sum() - - # For technologies with start year at or after model start year, add to - # TotalAnnualMinCapacityInvestment - df_min_cap_inv = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] >= min(get_years(start_year, end_year))] - df_min_cap_inv.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'YEAR'], - keep='last', - inplace=True) - - tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) - - df_iar_custom = pd.DataFrame(list(itertools.product(tech_list, - [1, 2], - get_years(start_year, end_year)) - ), - columns = ['TECHNOLOGY', - 'MODE_OF_OPERATION', - 'YEAR'] - ) - df_oar_custom = pd.DataFrame(list(itertools.product(tech_list, - [1, 2], - get_years(start_year, end_year)) - ), - columns = ['TECHNOLOGY', - 'MODE_OF_OPERATION', - 'YEAR'] - ) - - df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1,'FUEL'] = ( - df_iar_custom['TECHNOLOGY'].str[3:9]) - df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2,'FUEL'] = ( - df_iar_custom['TECHNOLOGY'].str[3:6] + "INT") - df_oar_custom.loc[df_oar_custom['MODE_OF_OPERATION']==1,'FUEL'] = ( - 'ELC' + df_oar_custom['TECHNOLOGY'].str[6:11] + '01') - df_oar_custom.loc[df_oar_custom['MODE_OF_OPERATION']==2,'FUEL'] = ( - 'ELC' + df_oar_custom['TECHNOLOGY'].str[6:11] + '01') - - df_iar_custom['VALUE'] = DF_IAR_CUSTOM_VAL - df_oar_custom['VALUE'] = DF_OAR_CUSTOM_VAL - df_iar_custom['REGION'] = region_name - df_oar_custom['REGION'] = region_name - - df_iar_custom = df_iar_custom[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] - df_oar_custom = df_oar_custom[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] - - df_iar = pd.concat([df_iar_final, df_iar_custom]) - df_oar = pd.concat([df_oar_final, df_oar_custom]) - - df_iar.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR'], - keep='last', - inplace=True) - df_oar.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR'], - keep='last', - inplace=True) - - # Add new fuels to FUEL set, if not already present - fuel_list = [] - fuel_list = list(df_iar_custom['FUEL'].unique()) + list(df_oar_custom['FUEL'].unique()) - fuel_list = list(set(fuel_list)) - - for each_fuel in fuel_list: - if each_fuel not in list(fuel_set['VALUE']): - fuel_set = pd.concat([fuel_set, pd.DataFrame({'VALUE':[each_fuel]})]) - - op_life_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) - - for each_tech in op_life_custom['TECHNOLOGY']: - op_life_custom.loc[op_life_custom['TECHNOLOGY'] == each_tech, - 'VALUE'] = op_life_dict.get(each_tech[3:6]) - - op_life_custom['REGION'] = region_name - op_life_custom = op_life_custom[['REGION', - 'TECHNOLOGY', - 'VALUE']] + # Add new fuels to FUEL set, if not already present + fuel_list = [] + fuel_list = list(df_iar_custom['FUEL'].unique()) + list(df_oar_custom['FUEL'].unique()) + fuel_list = list(set(fuel_list)) + + for each_fuel in fuel_list: + if each_fuel not in list(fuel_set['VALUE']): + fuel_set = pd.concat([fuel_set, pd.DataFrame({'VALUE':[each_fuel]})]) + + op_life_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) + + for each_tech in op_life_custom['TECHNOLOGY']: + op_life_custom.loc[op_life_custom['TECHNOLOGY'] == each_tech, + 'VALUE'] = op_life_dict.get(each_tech[3:6]) - op_life = pd.concat([op_life_base, op_life_custom]) - op_life.drop_duplicates(subset=['REGION', - 'TECHNOLOGY'], - keep='last', - inplace=True) - - # Add CapacityToActivityUnit for custom technologies - cap_act_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) - cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('PWR'), - 'VALUE'] = 31.536 - cap_act_custom['REGION'] = region_name - cap_act_custom = cap_act_custom[['REGION', - 'TECHNOLOGY', - 'VALUE']] - cap_act = pd.concat([cap_act_base, cap_act_custom]) - cap_act.drop_duplicates(inplace=True) - - # Update CapitalCost with user-defined costs - tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) - cap_cost = pd.DataFrame(list(itertools.product(tech_list, - get_years(start_year, end_year))), - columns = ['TECHNOLOGY', - 'YEAR']) - - for each_tech in tech_list: - cap_cost.loc[cap_cost['TECHNOLOGY'].str.startswith(each_tech), - 'VALUE'] = capex_dict[each_tech] - - cap_cost['REGION'] = region_name - cap_cost = cap_cost[['REGION', + op_life_custom['REGION'] = region_name + op_life_custom = op_life_custom[['REGION', 'TECHNOLOGY', - 'YEAR', 'VALUE']] - cap_cost = pd.concat([cap_cost_base, cap_cost]) - cap_cost.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], - keep="last", - inplace=True) - - return(df_tech_set, df_max_cap_inv, df_min_cap_inv, - df_res_cap, df_iar, df_oar, fuel_set, op_life, - cap_act, cap_cost) \ No newline at end of file + + op_life = pd.concat([op_life_base, op_life_custom]) + op_life.drop_duplicates(subset=['REGION', + 'TECHNOLOGY'], + keep='last', + inplace=True) + + # Add CapacityToActivityUnit for custom technologies + cap_act_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) + cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('PWR'), + 'VALUE'] = 31.536 + cap_act_custom['REGION'] = region_name + cap_act_custom = cap_act_custom[['REGION', + 'TECHNOLOGY', + 'VALUE']] + cap_act = pd.concat([cap_act_base, cap_act_custom]) + cap_act.drop_duplicates(inplace=True) + + # Update CapitalCost with user-defined costs + tech_list = list(tech_capacity_df['TECHNOLOGY'].unique()) + cap_cost = pd.DataFrame(list(itertools.product(tech_list, + get_years(start_year, end_year))), + columns = ['TECHNOLOGY', + 'YEAR']) + + for each_tech in tech_list: + cap_cost.loc[cap_cost['TECHNOLOGY'].str.startswith(each_tech), + 'VALUE'] = capex_dict[each_tech] + + cap_cost['REGION'] = region_name + cap_cost = cap_cost[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + cap_cost = pd.concat([cap_cost_base, cap_cost]) + cap_cost.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], + keep="last", + inplace=True) + + return(df_tech_set, df_max_cap_inv, df_min_cap_inv, + df_res_cap, df_iar, df_oar, fuel_set, op_life, + cap_act, cap_cost) \ No newline at end of file diff --git a/workflow/scripts/osemosys_global/transmission/main.py b/workflow/scripts/osemosys_global/transmission/main.py index f59219ab..c423c776 100644 --- a/workflow/scripts/osemosys_global/transmission/main.py +++ b/workflow/scripts/osemosys_global/transmission/main.py @@ -76,7 +76,7 @@ def main( # Set operational life for transmission. df_op_life_trn_final = set_op_life_transmission(iar_trn, oar_trn, - op_life_dict, op_life_base, region_name) + default_op_life, op_life_base, region_name) # Set annual capacity investment constraints. df_max_cap_invest_trn_final = cap_investment_constraints_trn(iar_trn, @@ -87,7 +87,7 @@ def main( region_name) # Alter output csv's based on user defined capacities following user config. - if not tech_capacity_trn is None: + if tech_capacity_trn is not None: (df_max_cap_invest_trn_final, df_min_cap_invest_trn_final, df_res_cap_trn_final, @@ -97,7 +97,7 @@ def main( df_cap_cost_trn_final ) = set_user_defined_capacity_trn( tech_capacity_trn, - op_life_dict, + default_op_life, min_cap_invest_base, df_max_cap_invest_trn_final, res_cap_base, @@ -151,16 +151,14 @@ def main( tech_set.to_csv(os.path.join(output_data_dir, "TECHNOLOGY.csv"), index = None) fuel_set.to_csv(os.path.join(output_data_dir, "FUEL.csv"), index = None) - - if not tech_capacity_trn is None: - - df_res_cap_trn_final.to_csv(os.path.join(output_data_dir, - 'ResidualCapacity.csv'), - index = None) - df_min_cap_invest_trn_final.to_csv(os.path.join(output_data_dir, - 'TotalAnnualMinCapacityInvestment.csv'), - index = None) + df_res_cap_trn_final.to_csv(os.path.join(output_data_dir, + 'ResidualCapacity.csv'), + index = None) + + df_min_cap_invest_trn_final.to_csv(os.path.join(output_data_dir, + 'TotalAnnualMinCapacityInvestment.csv'), + index = None) if __name__ == "__main__": @@ -188,7 +186,11 @@ def main( file_min_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMinCapacityInvestment.csv' file_res_cap_base = f'{powerplant_data_dir}/ResidualCapacity.csv' file_tech_set = f'{powerplant_data_dir}/TECHNOLOGY.csv' - file_fuel_set = f'{powerplant_data_dir}/FUEL.csv' + file_fuel_set = f'{powerplant_data_dir}/FUEL.csv' + + # The below else statement defines variables if the 'transmission/main' script is to be run locally + # outside the snakemake workflow. This is relevant for testing purposes only! User inputs when running + # the full workflow need to be defined in the config file. else: file_plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx' @@ -239,7 +241,7 @@ def main( input_data = { "plexos_prop": plexos_prop, - "default_op_life": op_life, + "default_op_life": op_life_dict, "line_data": trn_line, "interface_data": trn_interface, "iar_base" : iar_base, diff --git a/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py b/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py index 88f34f50..cd7f9a0a 100644 --- a/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py +++ b/workflow/scripts/osemosys_global/transmission/user_defined_capacity.py @@ -17,212 +17,210 @@ def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, build_rate_dict = {} capex_dict = {} build_year_dict = {} - - if not tech_capacity_trn is None: - for tech, tech_params in tech_capacity_trn.items(): - techCapacity_trn.append([tech, tech_params[0], tech_params[1]]) - tech_capacity_trn_dict[tech] = tech_params[2] #UNUSED ENTRY - build_year_dict[tech] = tech_params[1] - first_year_dict[tech] = tech_params[3] - build_rate_dict[tech] = tech_params[4] - capex_dict[tech] = tech_params[5] - tech_capacity_trn_df = pd.DataFrame(techCapacity_trn, - columns=['TECHNOLOGY', 'VALUE', 'YEAR']) - tech_capacity_trn_df['REGION'] = region_name - tech_capacity_trn_df = tech_capacity_trn_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] + for tech, tech_params in tech_capacity_trn.items(): + techCapacity_trn.append([tech, tech_params[0], tech_params[1]]) + tech_capacity_trn_dict[tech] = tech_params[2] #UNUSED ENTRY + build_year_dict[tech] = tech_params[1] + first_year_dict[tech] = tech_params[3] + build_rate_dict[tech] = tech_params[4] + capex_dict[tech] = tech_params[5] + tech_capacity_trn_df = pd.DataFrame(techCapacity_trn, + columns=['TECHNOLOGY', 'VALUE', 'YEAR']) + tech_capacity_trn_df['REGION'] = region_name + tech_capacity_trn_df = tech_capacity_trn_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']] - df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_trn_df]) - df_min_cap_inv.drop_duplicates(inplace=True) - - max_cap_techs_df = pd.DataFrame(list(itertools.product(list(tech_capacity_trn_df['TECHNOLOGY'].unique()), - get_years(start_year, end_year)) - ), - columns = ['TECHNOLOGY', - 'YEAR'] - ) - max_cap_techs_df['REGION'] = region_name - max_cap_techs_df = pd.merge(max_cap_techs_df, df_min_cap_inv, - how='left', - on=['REGION', 'TECHNOLOGY', 'YEAR']) - max_cap_techs_df['FIRST_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(first_year_dict) - max_cap_techs_df['BUILD_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(build_year_dict) - max_cap_techs_df['MAX_BUILD'] = max_cap_techs_df['TECHNOLOGY'].map(build_rate_dict) + df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_trn_df]) + df_min_cap_inv.drop_duplicates(inplace=True) + + max_cap_techs_df = pd.DataFrame(list(itertools.product(list(tech_capacity_trn_df['TECHNOLOGY'].unique()), + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'YEAR'] + ) + max_cap_techs_df['REGION'] = region_name + max_cap_techs_df = pd.merge(max_cap_techs_df, df_min_cap_inv, + how='left', + on=['REGION', 'TECHNOLOGY', 'YEAR']) + max_cap_techs_df['FIRST_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(first_year_dict) + max_cap_techs_df['BUILD_YEAR'] = max_cap_techs_df['TECHNOLOGY'].map(build_year_dict) + max_cap_techs_df['MAX_BUILD'] = max_cap_techs_df['TECHNOLOGY'].map(build_rate_dict) - # Fill VALUE with MAX_BUILD for YEAR >= FIRST_YEAR - max_cap_techs_df.loc[(max_cap_techs_df['YEAR']>=max_cap_techs_df['FIRST_YEAR']) & - (max_cap_techs_df['YEAR']>max_cap_techs_df['BUILD_YEAR']), - 'VALUE'] = max_cap_techs_df['MAX_BUILD'] - max_cap_techs_df.infer_objects().fillna(0, - inplace=True) - - max_cap_techs_df = max_cap_techs_df[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] + # Fill VALUE with MAX_BUILD for YEAR >= FIRST_YEAR + max_cap_techs_df.loc[(max_cap_techs_df['YEAR']>=max_cap_techs_df['FIRST_YEAR']) & + (max_cap_techs_df['YEAR']>max_cap_techs_df['BUILD_YEAR']), + 'VALUE'] = max_cap_techs_df['MAX_BUILD'] + max_cap_techs_df.infer_objects().fillna(0, + inplace=True) + + max_cap_techs_df = max_cap_techs_df[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] - # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD - df_max_cap_inv = pd.concat([df_max_cap_invest, max_cap_techs_df]).drop_duplicates() + # Append existing TotalAnnualMaxCapacityInvestment data with MAX_BUILD + df_max_cap_inv = pd.concat([df_max_cap_invest, max_cap_techs_df]).drop_duplicates() - # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD - df_max_cap_inv.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'YEAR'], - keep='last', - inplace=True) - - # For technologies with start year before model start year, add to ResidualCapacity - df_res_cap_ud = df_min_cap_inv.copy().loc[df_min_cap_inv.copy()['YEAR'] < min(get_years(start_year, end_year))] - df_res_cap_ud.rename(columns={'YEAR':'START_YEAR'}, - inplace=True) - df_res_cap_ud_final = pd.DataFrame(list(itertools.product(df_res_cap_ud['TECHNOLOGY'].unique(), - get_years(start_year, end_year)) - ), - columns = ['TECHNOLOGY', - 'YEAR'] - ) - df_res_cap_ud_final = pd.merge(df_res_cap_ud_final, - df_res_cap_ud, - how='left', - on=['TECHNOLOGY']) - df_res_cap_ud_final['TECH'] = df_res_cap_ud_final['TECHNOLOGY'].str[3:6] - df_res_cap_ud_final.loc[df_res_cap_ud_final['TECHNOLOGY'].str.contains('TRN'), - 'TECH'] = 'TRN' - df_res_cap_ud_final['OP_LIFE'] = df_res_cap_ud_final['TECH'].map(op_life_dict) - df_res_cap_ud_final['END_YEAR'] = (df_res_cap_ud_final['OP_LIFE'] - + df_res_cap_ud_final['START_YEAR']) - df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] - >= df_res_cap_ud_final['START_YEAR']] - df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] - <= df_res_cap_ud_final['END_YEAR']] - df_res_cap_ud_final['REGION'] = region_name - df_res_cap_ud_final = df_res_cap_ud_final[['REGION', - 'TECHNOLOGY', - 'YEAR', - 'VALUE']] - - df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final - if not df_res_cap_ud_final.empty else None]) - - # For technologies with start year at or after model start year, add to - # TotalAnnualMinCapacityInvestment - df_min_cap_inv = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] >= min(get_years(start_year, end_year))] - df_min_cap_inv.drop_duplicates(subset=['REGION', + # Print TotalAnnualMaxCapacityInvestment.csv with MAX_BUILD + df_max_cap_inv.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'YEAR'], + keep='last', + inplace=True) + + # For technologies with start year before model start year, add to ResidualCapacity + df_res_cap_ud = df_min_cap_inv.copy().loc[df_min_cap_inv.copy()['YEAR'] < min(get_years(start_year, end_year))] + df_res_cap_ud.rename(columns={'YEAR':'START_YEAR'}, + inplace=True) + df_res_cap_ud_final = pd.DataFrame(list(itertools.product(df_res_cap_ud['TECHNOLOGY'].unique(), + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'YEAR'] + ) + df_res_cap_ud_final = pd.merge(df_res_cap_ud_final, + df_res_cap_ud, + how='left', + on=['TECHNOLOGY']) + df_res_cap_ud_final['TECH'] = df_res_cap_ud_final['TECHNOLOGY'].str[3:6] + df_res_cap_ud_final.loc[df_res_cap_ud_final['TECHNOLOGY'].str.contains('TRN'), + 'TECH'] = 'TRN' + df_res_cap_ud_final['OP_LIFE'] = df_res_cap_ud_final['TECH'].map(op_life_dict) + df_res_cap_ud_final['END_YEAR'] = (df_res_cap_ud_final['OP_LIFE'] + + df_res_cap_ud_final['START_YEAR']) + df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] + >= df_res_cap_ud_final['START_YEAR']] + df_res_cap_ud_final = df_res_cap_ud_final.loc[df_res_cap_ud_final['YEAR'] + <= df_res_cap_ud_final['END_YEAR']] + df_res_cap_ud_final['REGION'] = region_name + df_res_cap_ud_final = df_res_cap_ud_final[['REGION', 'TECHNOLOGY', - 'YEAR'], - keep='last', - inplace=True) - - # Add IAR and OAR for custom technologies - tech_list = list(tech_capacity_trn_df['TECHNOLOGY'].unique()) - df_iar_custom = pd.DataFrame(list(itertools.product(tech_list, - [1, 2], - get_years(start_year, end_year)) - ), - columns = ['TECHNOLOGY', - 'MODE_OF_OPERATION', - 'YEAR'] - ) - df_oar_custom = pd.DataFrame(list(itertools.product(tech_list, - [1, 2], - get_years(start_year, end_year)) - ), - columns = ['TECHNOLOGY', - 'MODE_OF_OPERATION', - 'YEAR'] - ) - # IAR in modes 1 and 2 are primary electricity commodity ('ELC*01') in - # node_from and node_to, respectively. - # OAR is the inverse of the above - df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, - 'FUEL'] = ('ELC' + - df_iar_custom['TECHNOLOGY'].str[3:8] + - '01') - df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, - 'FUEL'] = ('ELC' + - df_iar_custom['TECHNOLOGY'].str[8:13] + - '01') - df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, - 'FUEL'] = ('ELC' + - df_oar_custom['TECHNOLOGY'].str[8:13] + - '01') - df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, - 'FUEL'] = ('ELC' + - df_oar_custom['TECHNOLOGY'].str[3:8] + - '01') - df_iar_custom['VALUE'] = df_iar_custom_val - df_oar_custom['VALUE'] = df_oar_custom_val - df_iar_custom['REGION'] = region_name - df_oar_custom['REGION'] = region_name - - df_iar_custom = df_iar_custom[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] - df_oar_custom = df_oar_custom[['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR', - 'VALUE',]] - - df_iar = pd.concat([df_iar_final, df_iar_custom]) - df_oar = pd.concat([df_oar_final, df_oar_custom]) - - df_iar.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR'], - keep='last', - inplace=True) + 'YEAR', + 'VALUE']] - df_oar.drop_duplicates(subset=['REGION', - 'TECHNOLOGY', - 'FUEL', - 'MODE_OF_OPERATION', - 'YEAR'], - keep='last', - inplace=True) + df_res_cap = pd.concat([df_res_cap, df_res_cap_ud_final + if not df_res_cap_ud_final.empty else None]) + + # For technologies with start year at or after model start year, add to + # TotalAnnualMinCapacityInvestment + df_min_cap_inv = df_min_cap_inv.loc[df_min_cap_inv['YEAR'] >= min(get_years(start_year, end_year))] + df_min_cap_inv.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'YEAR'], + keep='last', + inplace=True) + + # Add IAR and OAR for custom technologies + tech_list = list(tech_capacity_trn_df['TECHNOLOGY'].unique()) + df_iar_custom = pd.DataFrame(list(itertools.product(tech_list, + [1, 2], + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'MODE_OF_OPERATION', + 'YEAR'] + ) + df_oar_custom = pd.DataFrame(list(itertools.product(tech_list, + [1, 2], + get_years(start_year, end_year)) + ), + columns = ['TECHNOLOGY', + 'MODE_OF_OPERATION', + 'YEAR'] + ) + # IAR in modes 1 and 2 are primary electricity commodity ('ELC*01') in + # node_from and node_to, respectively. + # OAR is the inverse of the above + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, + 'FUEL'] = ('ELC' + + df_iar_custom['TECHNOLOGY'].str[3:8] + + '02') + df_iar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, + 'FUEL'] = ('ELC' + + df_iar_custom['TECHNOLOGY'].str[8:13] + + '02') + df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==1, + 'FUEL'] = ('ELC' + + df_oar_custom['TECHNOLOGY'].str[8:13] + + '01') + df_oar_custom.loc[df_iar_custom['MODE_OF_OPERATION']==2, + 'FUEL'] = ('ELC' + + df_oar_custom['TECHNOLOGY'].str[3:8] + + '01') + df_iar_custom['VALUE'] = df_iar_custom_val + df_oar_custom['VALUE'] = df_oar_custom_val + df_iar_custom['REGION'] = region_name + df_oar_custom['REGION'] = region_name - op_life_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) + df_iar_custom = df_iar_custom[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + df_oar_custom = df_oar_custom[['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR', + 'VALUE',]] + + df_iar = pd.concat([df_iar_final, df_iar_custom]) + df_oar = pd.concat([df_oar_final, df_oar_custom]) + + df_iar.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', + inplace=True) - op_life_custom.loc[op_life_custom['TECHNOLOGY'].str.contains('TRN'), - 'VALUE'] = op_life_dict.get('TRN') - op_life_custom['REGION'] = region_name - op_life_custom = op_life_custom[['REGION', - 'TECHNOLOGY', - 'VALUE']] + df_oar.drop_duplicates(subset=['REGION', + 'TECHNOLOGY', + 'FUEL', + 'MODE_OF_OPERATION', + 'YEAR'], + keep='last', + inplace=True) - op_life = pd.concat([op_life_base, op_life_custom]) - op_life.drop_duplicates(subset=['REGION', - 'TECHNOLOGY'], - keep='last', - inplace=True) - - # Update CapitalCost with user-defined costs by transmission line - tech_list = list(tech_capacity_trn_df['TECHNOLOGY'].unique()) - cap_cost_trn = pd.DataFrame(list(itertools.product(tech_list, - get_years(start_year, end_year))), - columns = ['TECHNOLOGY', - 'YEAR']) - - for each_trn in tech_list: - cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.startswith(each_trn), - 'VALUE'] = capex_dict[each_trn] + op_life_custom = pd.DataFrame({'TECHNOLOGY': tech_list}) - cap_cost_trn['REGION'] = region_name - cap_cost_trn = cap_cost_trn[['REGION', + op_life_custom.loc[op_life_custom['TECHNOLOGY'].str.contains('TRN'), + 'VALUE'] = op_life_dict.get('TRN') + op_life_custom['REGION'] = region_name + op_life_custom = op_life_custom[['REGION', 'TECHNOLOGY', - 'YEAR', 'VALUE']] - cap_cost = pd.concat([cap_cost_base, cap_cost_trn]) - cap_cost.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], - keep="last", - inplace=True) - - return(df_max_cap_inv, df_min_cap_inv, - df_res_cap, df_iar, df_oar, op_life, cap_cost) - \ No newline at end of file + + op_life = pd.concat([op_life_base, op_life_custom]) + op_life.drop_duplicates(subset=['REGION', + 'TECHNOLOGY'], + keep='last', + inplace=True) + + # Update CapitalCost with user-defined costs by transmission line + tech_list = list(tech_capacity_trn_df['TECHNOLOGY'].unique()) + cap_cost_trn = pd.DataFrame(list(itertools.product(tech_list, + get_years(start_year, end_year))), + columns = ['TECHNOLOGY', + 'YEAR']) + + for each_trn in tech_list: + cap_cost_trn.loc[cap_cost_trn['TECHNOLOGY'].str.startswith(each_trn), + 'VALUE'] = capex_dict[each_trn] + + cap_cost_trn['REGION'] = region_name + cap_cost_trn = cap_cost_trn[['REGION', + 'TECHNOLOGY', + 'YEAR', + 'VALUE']] + cap_cost = pd.concat([cap_cost_base, cap_cost_trn]) + cap_cost.drop_duplicates(subset=['REGION', 'TECHNOLOGY', 'YEAR'], + keep="last", + inplace=True) + + return(df_max_cap_inv, df_min_cap_inv, + df_res_cap, df_iar, df_oar, op_life, cap_cost) + \ No newline at end of file