Merge pull request NHERI-SimCenter#75 from zsarnoczay/develop

Merge master updates to develop and fix some bugs

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.9', '3.10', '3.11','3.12']
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "pelicun/resources/DamageAndLossModelLibrary"]
+	path = pelicun/resources/DamageAndLossModelLibrary
+	url = https://github.com/NHERI-SimCenter/DamageAndLossModelLibrary

pelicun/assessment.py

@@ -44,7 +44,7 @@
 
 import json
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 import pandas as pd
@@ -53,6 +53,9 @@
 from pelicun.__init__ import __version__ as pelicun_version  # type: ignore
 from pelicun.base import EDP_to_demand_type, get
 
+if TYPE_CHECKING:
+    from pelicun.base import Logger
+
 default_dbs = {
     'fragility': {
         'FEMA P-58': 'damage_DB_FEMA_P58_2nd.csv',
@@ -120,9 +123,11 @@ def __init__(self, config_options: dict[str, Any] | None = None) -> None:
         """
         self.stories: int | None = None
         self.options = base.Options(config_options, self)
-        self.unit_conversion_factors = base.parse_units(self.options.units_file)
+        self.unit_conversion_factors: dict = base.parse_units(
+            self.options.units_file
+        )
 
-        self.log = self.options.log
+        self.log: Logger = self.options.log
         self.log.msg(
             f'pelicun {pelicun_version} | \n',
             prepend_timestamp=False,
@@ -640,7 +645,7 @@ def calculate_demand(  # noqa: C901
 
             if 'Units' in raw_demands.index:
                 raw_units = raw_demands.loc['Units', :]
-                raw_demands = raw_demands.drop('Units', axis=0)
+                raw_demands = raw_demands.drop('Units', axis=0).astype(float)
 
             else:
                 raw_units = None

pelicun/base.py

@@ -43,12 +43,12 @@
 from __future__ import annotations
 
 import argparse
-import datetime
 import json
 import pprint
 import sys
 import traceback
 import warnings
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Optional, TypeVar, overload
 
@@ -194,12 +194,9 @@ def __init__(
         self.sampling_method: str | None = None
         self.list_all_ds: bool | None = None
 
-        self._seed: float | None = None
-
-        self._rng = np.random.default_rng()
         merged_config_options = merge_default_config(user_config_options)
 
-        self._seed = merged_config_options['Seed']
+        self.seed = merged_config_options['Seed']
         self.sampling_method = merged_config_options['Sampling']['SamplingMethod']
         self.list_all_ds = merged_config_options['ListAllDamageStates']
 
@@ -283,7 +280,11 @@ def log_exception(
             f"{''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))}"
         )
         for logger in cls._loggers:
-            logger.warning(message)
+            logger.msg(message)
+
+        # Also call the default excepthook to print the exception to
+        # the console as is done by default.
+        sys.__excepthook__(exc_type, exc_value, exc_traceback)
 
 
 # Update sys.excepthook to log exceptions in all loggers
@@ -411,7 +412,9 @@ def msg(
 
         for msg_i, msg_line in enumerate(msg_lines):
             if prepend_timestamp and (msg_i == 0):
-                formatted_msg = f'{datetime.datetime.now().strftime(self.log_time_format)} {msg_line}'  # noqa: DTZ005
+                formatted_msg = (
+                    f'{datetime.now().strftime(self.log_time_format)} {msg_line}'  # noqa: DTZ005
+                )
             elif prepend_timestamp or prepend_blank_space:
                 formatted_msg = self.spaces + msg_line
             else:
@@ -486,9 +489,9 @@ def print_system_info(self) -> None:
         self.msg(
             'System Information:', prepend_timestamp=False, prepend_blank_space=False
         )
-        start = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')  # noqa: DTZ005
+        start = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')  # noqa: DTZ005
         self.msg(
-            f'local time zone: {datetime.datetime.utcnow().astimezone().tzinfo}\n'
+            f'local time zone: {datetime.now(timezone.utc).astimezone().tzinfo}\n'
             f'start time: {start}\n'
             f'python: {sys.version}\n'
             f'numpy: {np.__version__}\n'
@@ -882,7 +885,11 @@ def convert_dtypes(dataframe: pd.DataFrame) -> pd.DataFrame:
         The modified DataFrame.
 
     """
-    dataframe = dataframe.fillna(value=np.nan)
+    with (
+        pd.option_context('future.no_silent_downcasting', True),  # noqa: FBT003
+        pd.option_context('mode.copy_on_write', True),  # noqa: FBT003
+    ):
+        dataframe = dataframe.fillna(value=np.nan).infer_objects()
     # note: `axis=0` applies the function to the columns
     # note: ignoring errors is a bad idea and should never be done. In
     # this case, however, that's not what we do, despite the name of
@@ -1196,24 +1203,19 @@ def int_or_None(string: str) -> int | None:  # noqa: N802
         return None
 
 
-def with_parsed_str_na_values(df: pd.DataFrame) -> pd.DataFrame:
+def check_if_str_is_na(string: Any) -> bool:  # noqa: ANN401
     """
-    Identify string values interpretable as N/A.
-
-    Given a dataframe, this function identifies values that have
-    string type and can be interpreted as N/A, and replaces them with
-    actual NA's.
+    Check if the provided string can be interpreted as N/A.
 
     Parameters
     ----------
-    df: pd.DataFrame
-        Dataframe to process
+    string: object
+            The string to evaluate
 
     Returns
     -------
-    pd.DataFrame
-        The dataframe with proper N/A values.
-
+    bool
+        The evaluation result. Yes, if the string is considered N/A.
     """
     na_vals = {
         '',
@@ -1238,11 +1240,30 @@ def with_parsed_str_na_values(df: pd.DataFrame) -> pd.DataFrame:
     }
     # obtained from Pandas' internal STR_NA_VALUES variable.
 
+    return isinstance(string, str) and string in na_vals
+
+
+def with_parsed_str_na_values(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Identify string values interpretable as N/A.
+
+    Given a dataframe, this function identifies values that have
+    string type and can be interpreted as N/A, and replaces them with
+    actual NA's.
+
+    Parameters
+    ----------
+    df: pd.DataFrame
+        Dataframe to process
+
+    Returns
+    -------
+    pd.DataFrame
+        The dataframe with proper N/A values.
+    """
     # Replace string NA values with actual NaNs
     return df.apply(
-        lambda col: col.map(
-            lambda x: np.nan if isinstance(x, str) and x in na_vals else x
-        )
+        lambda col: col.map(lambda x: np.nan if check_if_str_is_na(x) else x)
     )
 
 

pelicun/model/demand_model.py

@@ -545,7 +545,9 @@ def parse_settings(  # noqa: C901
         ) -> None:
             def parse_str_to_float(in_str: str, context_string: str) -> float:
                 try:
-                    out_float = float(in_str)
+                    out_float = (
+                        np.nan if base.check_if_str_is_na(in_str) else float(in_str)
+                    )
 
                 except ValueError:
                     self.log.warning(
@@ -574,7 +576,7 @@ def parse_str_to_float(in_str: str, context_string: str) -> float:
                 cols = tuple(cols_lst)
 
             # load the distribution family
-            cal_df.loc[idx[cols, :, :], 'Family'] = settings['DistributionFamily']
+            cal_df.loc[list(cols), 'Family'] = settings['DistributionFamily']
 
             # load limits
             for lim in (
@@ -586,7 +588,7 @@ def parse_str_to_float(in_str: str, context_string: str) -> float:
                 if lim in settings:
                     val = parse_str_to_float(settings[lim], lim)
                     if not pd.isna(val):
-                        cal_df.loc[idx[cols, :, :], lim] = val
+                        cal_df.loc[list(cols), lim] = val
 
             # scale the censor and truncation limits, if needed
             scale_factor = self._asmnt.scale_factor(settings.get('Unit'))
@@ -609,7 +611,7 @@ def parse_str_to_float(in_str: str, context_string: str) -> float:
                 if settings['DistributionFamily'] == 'normal':
                     sig_increase *= scale_factor
 
-                cal_df.loc[idx[cols, :, :], 'SigIncrease'] = sig_increase
+                cal_df.loc[list(cols), 'SigIncrease'] = sig_increase
 
         def get_filter_mask(
             demand_sample: pd.DataFrame,

pelicun/model/loss_model.py

@@ -1714,8 +1714,10 @@ def save_sample(
 
         assert self.sample is not None
         assert self.loss_params is not None
-        cmp_units = self.loss_params['DV', 'Unit']
-        dv_units = pd.Series(index=self.sample.columns, name='Units', dtype='object')
+        cmp_units = self.loss_params['DV', 'Unit'].sort_index()
+        dv_units = pd.Series(
+            index=self.sample.columns, name='Units', dtype='object'
+        ).sort_index()
 
         valid_dv_types = dv_units.index.unique(level=0)
         valid_cmp_ids = dv_units.index.unique(level=1)

pelicun/model/pelicun_model.py

@@ -51,6 +51,7 @@
 
 if TYPE_CHECKING:
     from pelicun.assessment import AssessmentBase
+    from pelicun.base import Logger
 
 idx = base.idx
 
@@ -75,7 +76,7 @@ def __init__(self, assessment: AssessmentBase) -> None:
 
         # link logging methods as attributes enabling more
         # concise syntax
-        self.log = self._asmnt.log
+        self.log: Logger = self._asmnt.log
 
     def _convert_marginal_params(  # noqa: C901
         self,

pelicun/resources/SimCenterDBDL/new_locations.json

@@ -0,0 +1,20 @@
+{
+  "damage_DB_FEMA_P58_2nd.csv": "seismic/building/component/FEMA P-58 2nd Edition/fragility.csv",
+  "damage_DB_FEMA_P58_2nd.json": "seismic/building/component/FEMA P-58 2nd Edition/fragility.json",
+  "damage_DB_Hazus_EQ_bldg.csv": "seismic/building/portfolio/Hazus v5.1/fragility.csv",
+  "damage_DB_Hazus_EQ_bldg.json": "seismic/building/portfolio/Hazus v5.1/fragility.json",
+  "damage_DB_Hazus_EQ_trnsp.csv": "seismic/transportation_network/portfolio/Hazus v5.1/fragility.csv",
+  "damage_DB_Hazus_EQ_trnsp.json": "seismic/transportation_network/portfolio/Hazus v5.1/fragility.json",
+  "damage_DB_Hazus_EQ_water.csv": "seismic/water_network/portfolio/fragility.csv",
+  "damage_DB_SimCenter_HU.csv": "hurricane/building/component/fragility.csv",
+  "damage_DB_SimCenter_HU.json": "hurricane/building/component/fragility.json",
+  "damage_DB_SimCenter_Hazus_HU_bldg.csv": "hurricane/building/portfolio/Hazus v4.2/fragility_fitted.csv",
+  "loss_repair_DB_FEMA_P58_2nd.csv": "seismic/building/component/FEMA P-58 2nd Edition/consequence_repair.csv",
+  "loss_repair_DB_FEMA_P58_2nd.json": "seismic/building/component/FEMA P-58 2nd Edition/consequence_repair.json",
+  "loss_repair_DB_Hazus_EQ_bldg.csv": "seismic/building/portfolio/Hazus v5.1/consequence_repair.csv",
+  "loss_repair_DB_Hazus_EQ_bldg.json": "seismic/building/portfolio/Hazus v5.1/consequence_repair.json",
+  "loss_repair_DB_Hazus_EQ_trnsp.csv": "seismic/transportation_network/portfolio/Hazus v5.1/consequence_repair.csv",
+  "loss_repair_DB_Hazus_EQ_trnsp.json": "seismic/transportation_network/portfolio/Hazus v5.1/consequence_repair.json",
+  "loss_repair_DB_SimCenter_Hazus_HU_bldg.csv": "hurricane/building/portfolio/Hazus v4.2/consequence_repair_fitted.csv",
+  "combined_loss_matrices/Wind_Flood_Hazus_HU_bldg.csv": "hurricane/building/portfolio/Hazus v4.2/combined_loss_matrices/Wind_Flood.csv"
+}