out of domain error fixed

workflow/rules/preprocess.smk

@@ -21,7 +21,7 @@ power_plant_files = [
     'powerplant/OperationalLife',
     'powerplant/TotalAnnualMaxCapacityInvestment',
     'powerplant/TotalAnnualMinCapacityInvestment',
-    'FUEL',
+    'powerplant/FUEL',
     'powerplant/InputActivityRatio',
     'powerplant/OutputActivityRatio',
     'MODE_OF_OPERATION',
@@ -44,6 +44,7 @@ transmission_files = [
     'OutputActivityRatio',
     'ResidualCapacity',
     'TECHNOLOGY',
+    'FUEL'
     ]
 
 timeslice_files = [

workflow/scripts/osemosys_global/powerplant/main.py

@@ -204,12 +204,12 @@ def main(
 
     tech_set.to_csv(os.path.join(powerplant_data_dir, "TECHNOLOGY.csv"), index = None)
 
+    fuel_set.to_csv(os.path.join(powerplant_data_dir, "FUEL.csv"), index = None)
+
     # OUTPUT CSV's NOT USED AS INPUT FOR TRANMISSION RULE
 
     df_af_final.to_csv(os.path.join(output_data_dir, 'AvailabilityFactor.csv'), index=None)
 
-    fuel_set.to_csv(os.path.join(output_data_dir, "FUEL.csv"), index = None)
-
     years_set.to_csv(os.path.join(output_data_dir, "YEAR.csv"), index = None)
 
     mode_list_set.to_csv(os.path.join(output_data_dir, "MODE_OF_OPERATION.csv"), index = None)

workflow/scripts/osemosys_global/transmission/activity.py

@@ -1,8 +1,8 @@
 """Function to calaculate activity related to transmission."""
-import pandas as pd
 
+from typing import Optional
+import pandas as pd
 from data import format_transmission_name
-
 from utils import apply_dtypes
 
 def activity_transmission(df_iar_base, df_oar_base, df_pw_prop, df_trn_efficiencies,
@@ -218,19 +218,20 @@ def activity_transmission_limit(cross_border_trade, df_oar_trn_final):
 
     return df_crossborder_final
 
-def capact_transmission(df_capact_base, df_oar_trn_final):
-
-    # Create CapacityToActivityUnit csv
-    df_capact_trn_final = df_oar_trn_final[['REGION',
-                                    'TECHNOLOGY'
-                                    ]]
-    df_capact_trn_final = df_capact_trn_final.drop_duplicates()
-    df_capact_trn_final = df_capact_trn_final.loc[df_capact_trn_final['TECHNOLOGY'
-                                                          ].str.startswith('TRN')]
 
-    df_capact_trn_final['VALUE'] = 31.536
-    df_capact_trn_final.drop_duplicates(inplace=True)
+def create_trn_dist_capacity_activity(*dfs: pd.DataFrame, value: Optional[float] = 31.536, region: Optional[str] = "GLOBAL") -> pd.DataFrame:
+    """Creates tranmission and distribution capacity to activity unit data
     
-    df_capact_trn_final = pd.concat([df_capact_base, df_capact_trn_final])
+    Inputs are any number of dataframes with a TECHNOLOGY colums.
+    """
 
-    return df_capact_trn_final
+    techs = []
+    for df in dfs:
+        temp = df[(df.TECHNOLOGY.str.startswith("TRN")) | (df.TECHNOLOGY.str.startswith("PWRTRN"))]
+        techs += temp.TECHNOLOGY.to_list()
+
+    data = []
+    for tech in set(techs):
+        data.append([region, tech, value])
+
+    return pd.DataFrame(data, columns=["REGION", "TECHNOLOGY", "VALUE"])

workflow/scripts/osemosys_global/transmission/main.py

@@ -14,7 +14,7 @@
     import_max_cap_invest_base,
     import_min_cap_invest_base,
     import_res_cap_base,
-    import_tech_set_base
+    import_set_base
 )
 
 from constants import(
@@ -25,7 +25,7 @@
 from activity import(
     activity_transmission,
     activity_transmission_limit,
-    capact_transmission
+    create_trn_dist_capacity_activity
     )
 
 from costs import get_transmission_costs
@@ -36,7 +36,7 @@
 
 from user_defined_capacity import set_user_defined_capacity_trn
 
-from sets import create_tech_set_trn
+from sets import create_set_from_iterators, get_unique_fuels, get_unique_technologies
 
 def main(
     plexos_prop: pd.DataFrame,
@@ -53,84 +53,91 @@ def main(
     min_cap_invest_base: pd.DataFrame,
     res_cap_base: pd.DataFrame,
     tech_set_base: pd.DataFrame,
+    fuel_set_base: pd.DataFrame
 ):
 
     # CALL FUNCTIONS
 
     # Set activity ratios for transmission.
-    df_iar_trn_final, df_oar_trn_final = activity_transmission(iar_base, oar_base, 
+    iar_trn, oar_trn = activity_transmission(iar_base, oar_base, 
                                                                plexos_prop, interface_data,
                                                                start_year, end_year,
                                                                region_name)
 
+
     # Adjust activity limits if cross border trade is not allowed following user config.
-    df_crossborder_final = activity_transmission_limit(cross_border_trade, df_oar_trn_final)
-
-    # Set CapacityToActivityUnit.
-    df_capact_trn_final = capact_transmission(capact_base, df_oar_trn_final)
+    df_crossborder_final = activity_transmission_limit(cross_border_trade, oar_trn)
 
     # Set capital and fixed transmission costs.
-    df_cap_cost_trn_final, df_fix_cost_trn_final = get_transmission_costs(line_data, df_oar_trn_final,
+    df_cap_cost_trn_final, df_fix_cost_trn_final = get_transmission_costs(line_data, oar_trn,
                                                                     cap_cost_base, fix_cost_base,
                                                                     start_year, end_year,
                                                                     region_name)
 
     # Set operational life for transmission.
-    df_op_life_trn_final = set_op_life_transmission(df_iar_trn_final, df_oar_trn_final, 
+    df_op_life_trn_final = set_op_life_transmission(iar_trn, oar_trn, 
                                                     op_life_dict, op_life_base, region_name)
 
     # Set annual capacity investment constraints.
-    df_max_cap_invest_trn_final = cap_investment_constraints_trn(df_iar_trn_final, 
+    df_max_cap_invest_trn_final = cap_investment_constraints_trn(iar_trn, 
                                                                  max_cap_invest_base, 
                                                                  no_investment_techs, 
                                                                  start_year, 
                                                                  end_year, 
                                                                  region_name)
-
-    tech_set_trn_final = pd.concat([tech_set, create_tech_set_trn(interface_data)])
 
     # Alter output csv's based on user defined capacities following user config.
     if not tech_capacity_trn is None:
-        (tech_set_trn_final, 
-         df_max_cap_invest_trn_final, 
+        (df_max_cap_invest_trn_final, 
          df_min_cap_invest_trn_final, 
          df_res_cap_trn_final, 
-         df_iar_trn_final,
-         df_oar_trn_final,
+         iar_trn,
+         oar_trn,
          df_op_life_trn_final, 
-         df_capact_trn_final, 
          df_cap_cost_trn_final
          ) = set_user_defined_capacity_trn(
-             tech_capacity_trn, 
-             op_life_dict, 
-             tech_set_trn_final, 
-             min_cap_invest_base, 
-             df_max_cap_invest_trn_final, 
-             res_cap_base,
-             df_iar_trn_final,
-             df_oar_trn_final,
-             df_op_life_trn_final,
-             df_capact_trn_final,
-             df_cap_cost_trn_final,
-             start_year,
-             end_year,
-             region_name,
-             DF_IAR_CUSTOM_VAL, 
-             DF_OAR_CUSTOM_VAL
-             )
+            tech_capacity_trn, 
+            op_life_dict, 
+            min_cap_invest_base, 
+            df_max_cap_invest_trn_final, 
+            res_cap_base,
+            iar_trn,
+            oar_trn,
+            df_op_life_trn_final,
+            df_cap_cost_trn_final,
+            start_year,
+            end_year,
+            region_name,
+            DF_IAR_CUSTOM_VAL, 
+            DF_OAR_CUSTOM_VAL
+            )
 
+    # get new additions to fuel and technology sets
+    exising_techs = tech_set_base.VALUE.to_list()
+    iar_techs = get_unique_technologies(iar_trn)
+    oar_techs = get_unique_technologies(oar_trn)
+    tech_set = create_set_from_iterators(exising_techs, iar_techs, oar_techs)
+
+    exising_fuels = fuel_set_base.VALUE.to_list()
+    iar_fuels = get_unique_fuels(iar_trn)
+    oar_fuels = get_unique_fuels(oar_trn)
+    fuel_set = create_set_from_iterators(exising_fuels, iar_fuels, oar_fuels)
+
+    # assign capacity to activity unit to transmission + distribution techs
+    cap_activity_trn = create_trn_dist_capacity_activity(iar_trn, oar_trn)
+    cap_activity = pd.concat([capact_base, cap_activity_trn]).drop_duplicates(subset=["REGION", "TECHNOLOGY"], keep="last")
+
     # OUTPUT CSV's
 
-    df_oar_trn_final.to_csv(os.path.join(output_data_dir, "OutputActivityRatio.csv"), index=None)
+    oar_trn.to_csv(os.path.join(output_data_dir, "OutputActivityRatio.csv"), index=None)
 
-    df_iar_trn_final.to_csv(os.path.join(output_data_dir, "InputActivityRatio.csv"), index=None)
+    iar_trn.to_csv(os.path.join(output_data_dir, "InputActivityRatio.csv"), index=None)
 
     df_crossborder_final.to_csv(os.path.join(output_data_dir,
                                             "TotalTechnologyModelPeriodActivityUpperLimit.csv"),
                                 index = None)
 
-    df_capact_trn_final.to_csv(os.path.join(output_data_dir, 
-                                            "CapacityToActivityUnit.csv"), index = None)
+    cap_activity.to_csv(os.path.join(output_data_dir, "CapacityToActivityUnit.csv"), index = None)
 
     df_cap_cost_trn_final.to_csv(os.path.join(output_data_dir, "CapitalCost.csv"), index = None)
 
@@ -142,7 +149,8 @@ def main(
                                             'TotalAnnualMaxCapacityInvestment.csv'),
                                         index = None)
 
-    tech_set_trn_final.to_csv(os.path.join(output_data_dir, "TECHNOLOGY.csv"), index = None)
+    tech_set.to_csv(os.path.join(output_data_dir, "TECHNOLOGY.csv"), index = None)
+    fuel_set.to_csv(os.path.join(output_data_dir, "FUEL.csv"), index = None)
 
     if not tech_capacity_trn is None:
 
@@ -180,6 +188,7 @@ def main(
         file_min_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMinCapacityInvestment.csv'
         file_res_cap_base = f'{powerplant_data_dir}/ResidualCapacity.csv'
         file_tech_set = f'{powerplant_data_dir}/TECHNOLOGY.csv'        
+        file_fuel_set = f'{powerplant_data_dir}/FUEL.csv'        
 
     else:
         file_plexos = 'resources/data/PLEXOS_World_2015_Gold_V1.1.xlsx'
@@ -205,7 +214,8 @@ def main(
         file_max_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMaxCapacityInvestment.csv'
         file_min_cap_invest_base = f'{powerplant_data_dir}/TotalAnnualMinCapacityInvestment.csv'
         file_res_cap_base = f'{powerplant_data_dir}/ResidualCapacity.csv'
-        file_tech_set = f'{powerplant_data_dir}/TECHNOLOGY.csv'          
+        file_tech_set = f'{powerplant_data_dir}/TECHNOLOGY.csv'
+        file_fuel_set = f'{powerplant_data_dir}/FUEL.csv'
 
     # SET INPUT DATA
     plexos_prop = import_plexos_2015(file_plexos, "prop")
@@ -224,23 +234,25 @@ def main(
     max_cap_invest_base = import_max_cap_invest_base(file_max_cap_invest_base)
     min_cap_invest_base = import_min_cap_invest_base(file_min_cap_invest_base)
     res_cap_base = import_res_cap_base(file_res_cap_base)  
-    tech_set = import_tech_set_base(file_tech_set)  
+    tech_set_base = import_set_base(file_tech_set)  
+    fuel_set_base = import_set_base(file_tech_set)  
 
     input_data = {
         "plexos_prop": plexos_prop,
         "default_op_life": op_life,
         "line_data": trn_line,
         "interface_data": trn_interface,
-         "iar_base" : iar_base,
-         "oar_base" : oar_base,
-         "capact_base" : capact_base,
-         "cap_cost_base" : cap_cost_base,
-         "fix_cost_base" : fix_cost_base,
-         "op_life_base" : op_life_base,
-         "max_cap_invest_base" : max_cap_invest_base,
-         "min_cap_invest_base" : min_cap_invest_base,
-         "res_cap_base" : res_cap_base, 
-         "tech_set_base" : tech_set,     
+        "iar_base" : iar_base,
+        "oar_base" : oar_base,
+        "capact_base" : capact_base,
+        "cap_cost_base" : cap_cost_base,
+        "fix_cost_base" : fix_cost_base,
+        "op_life_base" : op_life_base,
+        "max_cap_invest_base" : max_cap_invest_base,
+        "min_cap_invest_base" : min_cap_invest_base,
+        "res_cap_base" : res_cap_base, 
+        "tech_set_base" : tech_set_base,
+        "fuel_set_base" : fuel_set_base,  
     }
 
     # CALL MAIN

workflow/scripts/osemosys_global/transmission/read.py

@@ -100,9 +100,6 @@ def import_res_cap_base(f: str) -> pd.DataFrame:
     """
     return pd.read_csv(f)
 
-def import_tech_set_base(f: str) -> pd.DataFrame:
-    """Imports TECHNOLOGY.csv as output from the Powerplant rule.
-    
-    TECHNOLOGY.csv
-    """
+def import_set_base(f: str) -> pd.DataFrame:
+    """Imports a set csv"""
     return pd.read_csv(f)

workflow/scripts/osemosys_global/transmission/sets.py

@@ -1,11 +1,23 @@
 """Function to create sets."""
 
-from data import format_transmission_name
+import pandas as pd
 
-def create_tech_set_trn(df_lines):
+def get_unique_technologies(ar: pd.DataFrame) -> list[str]:
+    """Gets unique technologies from activity ratio dataframe"""
+    return ar.TECHNOLOGY.unique().tolist()
 
-    df = df_lines[['From', 'To']].copy()
-    df = format_transmission_name(df)
-    df = df.rename(columns = {'TECHNOLOGY' : 'VALUE'})    
+def get_unique_fuels(ar: pd.DataFrame) -> list[str]:
+    """Gets unique fuels from activity ratio dataframe"""
+    return ar.FUEL.unique().tolist()
+
+def create_set_from_iterators(*args: list[str] | set[str]) -> pd.DataFrame:
+    """Creates a set formatted dataframe from an arbitrary number of arguments"""
+
+    data = []
+
+    for arg in args:
+        data += arg
+
+    data = list(set(data))
 
-    return df
+    return pd.Series(data, name="VALUE").to_frame()

workflow/scripts/osemosys_global/transmission/user_defined_capacity.py

@@ -5,9 +5,9 @@
 
 from data import get_years
 
-def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_tech_set, 
+def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, 
                                   df_min_cap_invest, df_max_cap_invest, df_res_cap,
-                                  df_iar_final, df_oar_final, op_life_base, cap_act_base,
+                                  df_iar_final, df_oar_final, op_life_base,
                                   cap_cost_base, start_year, end_year, region_name,
                                   df_iar_custom_val, df_oar_custom_val):
 
@@ -32,12 +32,6 @@ def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_tech_set,
         tech_capacity_trn_df['REGION'] = region_name
         tech_capacity_trn_df = tech_capacity_trn_df[['REGION', 'TECHNOLOGY', 'YEAR', 'VALUE']]
 
-        for each_tech in list(tech_capacity_trn_df['TECHNOLOGY'].unique()):
-            if each_tech not in list(df_tech_set['VALUE']):
-                df_tech_set = pd.concat([df_tech_set, pd.DataFrame({'VALUE':[each_tech]})])
-
-        df_tech_set.drop_duplicates(inplace=True)
-
         df_min_cap_inv = pd.concat([df_min_cap_invest, tech_capacity_trn_df])
         df_min_cap_inv.drop_duplicates(inplace=True)
 
@@ -207,16 +201,6 @@ def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_tech_set,
                                         'TECHNOLOGY'],
                                 keep='last',
                                 inplace=True)
-
-        cap_act_custom = pd.DataFrame({'TECHNOLOGY': tech_list})
-        cap_act_custom.loc[cap_act_custom['TECHNOLOGY'].str.contains('TRN'),
-                           'VALUE'] = 31.536
-        cap_act_custom['REGION'] = region_name
-        cap_act_custom = cap_act_custom[['REGION',
-                                         'TECHNOLOGY',
-                                         'VALUE']]
-        cap_act = pd.concat([cap_act_base, cap_act_custom])
-        cap_act.drop_duplicates(inplace=True)
 
         # Update CapitalCost with user-defined costs by transmission line
         tech_list = list(tech_capacity_trn_df['TECHNOLOGY'].unique())
@@ -239,6 +223,6 @@ def set_user_defined_capacity_trn(tech_capacity_trn, op_life_dict, df_tech_set,
                                  keep="last",
                                  inplace=True)
 
-        return(df_tech_set, df_max_cap_inv, df_min_cap_inv, 
-               df_res_cap, df_iar, df_oar, op_life, cap_act, cap_cost)
+        return(df_max_cap_inv, df_min_cap_inv, 
+               df_res_cap, df_iar, df_oar, op_life, cap_cost)