Skip to content

Commit

Permalink
Implement README with full docs, TIMEIT functionality, full performan…
Browse files Browse the repository at this point in the history 
…ce benchmark (#11)

Also, a jupyter notebook to plot the timing results.
  • Loading branch information
@mtazzari
mtazzari authored Nov 5, 2021
1 parent 5a26520 commit 311dce1
Show file tree
Hide file tree
Showing 17 changed files with 868 additions and 63 deletions.
324 changes: 323 additions & 1 deletion README.md
View file

Large diffs are not rendered by default.

59 changes: 59 additions & 0 deletions benchmark/benchmark.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# bash script to run all the benchmarks

TIMINGS_LOGS_DIR="timings"


# simulator 0
export TIMEIT=1
export TIMEIT_CYCLES=100
export TIMEIT_LOGFILE="${TIMINGS_LOGS_DIR}/timings_s0.txt"

gethurricaneloss 10 2 0.001 30 1 0.000001 -n 10 -s0
gethurricaneloss 10 2 0.001 30 1 0.000001 -n 100 -s0
gethurricaneloss 10 2 0.001 30 1 0.000001 -n 1000 -s0
export TIMEIT_CYCLES=10
gethurricaneloss 10 2 0.001 30 1 0.000001 -n 100000 -s0
export TIMEIT_CYCLES=4
gethurricaneloss 10 2 0.001 30 1 0.000001 -n 1000000 -s0
export TIMEIT_CYCLES=2
gethurricaneloss 10 2 0.001 30 1 0.000001 -n 10000000 -s0 # <-- THIS TAKES **A LOT** (~30 mins on Macbook Pro 2019)


# simulators 1, 2, 3, 4
export TIMEIT_CYCLES=1000

num_monte_carlo_samples="10 100 1000 10000 100000" #manca 100000
simulator_ids="1 2 3 4 5"

for simulator_id in $simulator_ids; do
for num_monte_carlo_sample in $num_monte_carlo_samples; do
export TIMEIT_LOGFILE="${TIMINGS_LOGS_DIR}/timings_s${simulator_id}.txt";
gethurricaneloss 10 2 0.001 30 1 0.000001 -n ${num_monte_carlo_sample} -s${simulator_id};
done
done

# run the largest MC simulations with reduced TIMEIT_CYCLES
export TIMEIT_CYCLES=50

num_monte_carlo_samples="1000000"
simulator_ids="1 2 3 4 5"

for simulator_id in $simulator_ids; do
for num_monte_carlo_sample in $num_monte_carlo_samples; do
export TIMEIT_LOGFILE="${TIMINGS_LOGS_DIR}/timings_s${simulator_id}.txt";
gethurricaneloss 10 2 0.001 30 1 0.000001 -n ${num_monte_carlo_sample} -s${simulator_id};
done
done

# run the largest MC simulations with reduced TIMEIT_CYCLES
export TIMEIT_CYCLES=5

num_monte_carlo_samples="10000000"
simulator_ids="1 2 3 4 5"

for simulator_id in $simulator_ids; do
for num_monte_carlo_sample in $num_monte_carlo_samples; do
export TIMEIT_LOGFILE="${TIMINGS_LOGS_DIR}/timings_s${simulator_id}.txt";
gethurricaneloss 10 2 0.001 30 1 0.000001 -n ${num_monte_carlo_sample} -s${simulator_id};
done
done
Binary file added benchmark/execution_time_vs_num_monte_carlo_samples.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added benchmark/mean_loss_vs_num_monte_carlo_samples.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
281 changes: 281 additions & 0 deletions benchmark/plot_timing_results.ipynb
View file

Large diffs are not rendered by default.

Binary file added benchmark/speedup_vs_num_monte_carlo_samples.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
6 changes: 6 additions & 0 deletions benchmark/timings/timings_s0.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10.000000 100 0.000914 50.087758
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100.000000 100 0.010028 49.988582
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000.000000 100 0.101372 49.983956
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000.000000 100 1.040224 49.992674
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100000.000000 10 10.038142 49.980640
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000000.000000 4 105.049747 50.000562
7 changes: 7 additions & 0 deletions benchmark/timings/timings_s1.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10.000000 1000 0.000013 49.966121
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100.000000 1000 0.000133 50.037439
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000.000000 1000 0.001401 49.991665
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000.000000 1000 0.014170 50.000415
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100000.000000 1000 0.144798 49.999268
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000000.000000 50 1.464731 50.000486
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000000.000000 5 14.800176 50.001481
7 changes: 7 additions & 0 deletions benchmark/timings/timings_s2.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10.000000 1000 0.000042 49.797800
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100.000000 1000 0.000065 50.027264
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000.000000 1000 0.000302 50.002490
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000.000000 1000 0.002653 50.000720
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100000.000000 1000 0.029597 49.999877
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000000.000000 50 0.273513 49.998960
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000000.000000 5 2.935461 50.002490
7 changes: 7 additions & 0 deletions benchmark/timings/timings_s3.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10.000000 1000 0.000014 50.119241
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100.000000 1000 0.000139 49.950395
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000.000000 1000 0.001394 49.983407
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000.000000 1000 0.014457 50.006514
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100000.000000 1000 0.150629 49.999678
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000000.000000 50 1.609194 50.000586
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000000.000000 5 16.547855 50.000416
7 changes: 7 additions & 0 deletions benchmark/timings/timings_s4.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10.000000 1000 0.000040 49.928014
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100.000000 1000 0.000166 49.999524
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000.000000 1000 0.001432 49.987668
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000.000000 1000 0.014547 49.999203
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100000.000000 1000 0.150475 49.998460
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000000.000000 50 1.632011 49.999781
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000000.000000 5 16.701602 49.999752
7 changes: 7 additions & 0 deletions benchmark/timings/timings_s5.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10.000000 1000 0.000041 49.950534
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100.000000 1000 0.000063 49.969032
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000.000000 1000 0.000284 50.000781
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000.000000 1000 0.002494 49.997507
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 100000.000000 1000 0.026746 50.000624
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 1000000.000000 50 0.296471 50.002018
10.000000 0.693147 0.001000 30.000000 0.000000 0.000001 10000000.000000 5 3.050854 49.999555
35 changes: 26 additions & 9 deletions oasishurricane/cli.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env python
# coding=utf-8

import os
import sys
import argparse
import numpy as np
Expand All @@ -13,21 +13,22 @@
logging.config.dictConfig(LOGGING)
logger = logging.getLogger("cli")

from .model import Simulator, SIMULATORS

from .simulator import Simulator, SIMULATORS
from . import __version__

def parse_args():
"""
Parse arguments from CLI.
:return: [dict] Parsed arguments.
:return:
"""
parser = argparse.ArgumentParser(
description="A Python command-line utility for Linux that computes the economic loss for hurricanes in Florida and in the Gulf states.",
usage='use "%(prog)s --help" for more information',
formatter_class=argparse.RawTextHelpFormatter # for multi-line help text
)

# parser = parser.add_argument_group('parser arguments')

parser.add_argument("florida_landfall_rate",
action="store",
help="[float] annual rate of landfalling hurricanes in Florida.",
Expand Down Expand Up @@ -60,12 +61,11 @@ def parse_args():
default=10)
parser.add_argument("-s", "--simulator",
action="store",
help="[int] simulator id. Implemented simulators: (id:name) \n" + \
help="[int] simulator id (default=0). Implemented simulators: (id:name) \n" + \
"\n".join([f"{k}: {v['desc']}" for k, v in SIMULATORS.items()]),
type=int,
dest="simulator_id",
default=0)

args = vars(parser.parse_args()) # convert to dict for ease of use

return args
Expand Down Expand Up @@ -130,13 +130,26 @@ def validate_args(args):
for arg_k in numerical_args:
logger.info(f"{arg_k:>30s} = {validated_args[arg_k]:>10.5f}")

if os.getenv("TIMEIT"):
if os.getenv("TIMEIT_LOGFILE"):
logger.info(
f"Found TIMEIT and TIMEIT_LOGFILE: timings will be logged in {os.getenv('TIMEIT_LOGFILE')}")
else:
logger.info("Found TIMEIT: logging timings to the console.")
return validated_args


def main(args=None):
"""
Main function, called through the shell entrypoint.
# TODO: IMPROVE DOCS
If no args are passed, the function assumes it is called as a CLI and parses the args from the shell.
If args are passed (e.g., when testing), then the args are not parsed.
In any case, the args are validated.
If used as a CLI, the function terminates the program, otherwise it returns the mean loss.
:param args: [dict] CLI arguments (default=None).
:return mean_loss: [float,optional] The mean economic loss.
"""
as_CLI = False
Expand All @@ -146,6 +159,9 @@ def main(args=None):
as_CLI = True
args = parse_args()

# splash message
logger.info(f"gethurricaneloss v{__version__} by Marco Tazzari")

# validate (and transform, if necessary) arguments
validated_args = validate_args(args)

Expand All @@ -156,6 +172,7 @@ def main(args=None):
mean_loss = sim.simulate(**validated_args)

if as_CLI:
print(mean_loss)
sys.exit(0)
else:
return mean_loss
Expand Down
9 changes: 7 additions & 2 deletions oasishurricane/logs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import os

# setup the directories for namespacing
BASE_DIR = os.path.curdir
LOGS_DIR = BASE_DIR

Expand All @@ -12,11 +13,15 @@
DEVELOPMENT_LOGFILE = os.path.join(LOGS_DIR, DEV_LOGFILE)
PRODUCTION_LOGFILE = os.path.join(LOGS_DIR, PROD_LOGFILE)


# define logging config
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'concise': {
'format': '[%(asctime)s] %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'
},
'simple': {
'format': '[%(asctime)s] %(levelname)6s %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'
Expand All @@ -30,7 +35,7 @@
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'simple'
'formatter': 'concise'
},
'development_logfile': {
'level': 'DEBUG',
Expand Down
Loading

0 comments on commit 311dce1

Please sign in to comment.