analyze.py

"""
Auxiliary script to analyze output data.

Options include comparing benchmark metrics, performing regression, plotting timeseries
performance, and more. Refer to the README for usage.
"""

import argparse

from userdefs import get_sample_methods
from yolov3 import parallelize

from retrain import utils
import analysis.benchmark as bench
from analysis import charts


def get_args(prefixes):
    parser = argparse.ArgumentParser(description="Analyze training output data")
    parser.add_argument(
        "--config", required=True, help="configuration file for output data"
    )
    parser.add_argument(
        "--prefix", choices=prefixes, default=None, help="prefix of model to test"
    )

    group = parser.add_mutually_exclusive_group()

    group.add_argument("--avg", type=int, action="store", nargs="?", default=False)
    group.add_argument("--roll_avg", type=int, default=None)
    parser.add_argument("--batch_test", type=int, default=None)

    parser.add_argument("--delta", type=int, default=4)

    parser.add_argument("--tabulate", action="store_true", default=False)
    parser.add_argument("--benchmark", action="store_true", default=False)
    parser.add_argument("--visualize_conf", default=None)
    parser.add_argument("--view_benchmark", default=None)

    parser.add_argument("--filter_sample", action="store_true", default=False)
    parser.add_argument("--compare_init", action="store_true", default=False)
    parser.add_argument("--aggr_median", action="store_true", default=False)

    metric_names = [
        "prec",
        "acc",
        "recall",
        "conf",
        "conf_std",
        "detect_conf_std",
        "epochs_trained",
    ]
    parser.add_argument("--metric", choices=metric_names, default="prec")
    parser.add_argument("--metric2", choices=metric_names, default=None)

    opt = parser.parse_args()
    config = utils.parse_retrain_config(opt.config)
    return opt, config


def benchmark_all(prefixes, config, opt):
    # Benchmark the inference results before the start of each sample batch
    # and as a time series
    batch_args = list()
    for prefix in prefixes:
        batch_args.append((prefix, config, opt))
        if not config["parallel"]:
            bench.benchmark_next_batch(*batch_args[-1])

    if config["parallel"]:
        parallelize.run_parallel(bench.benchmark_next_batch, batch_args)


def benchmark_batch_test(prefixes, config, opt, num_batches):
    batch_args = list()
    for prefix in prefixes:
        batch_args.append((prefix, config, opt, num_batches))
        if not config["parallel"]:
            bench.benchmark_batch_test_set(*batch_args[-1])
    if config["parallel"]:
        parallelize.run_parallel(bench.benchmark_batch_test_set, batch_args)


def get_benchmark_suffix(opt):
    bench_suffix = "_*.csv"
    if opt.batch_test is not None:
        # Default is rolling average on batch test set
        bench_suffix = "_test" + bench_suffix
        if opt.avg:
            # This provides the linear-spaced variant
            bench_suffix = "_avg" + bench_suffix
    elif opt.roll_avg is not None:
        # Rolling average for batch splits and series
        bench_suffix = "_roll" + bench_suffix
    else:
        # View batch splits; generated by default
        bench_suffix = "_avg_1" + bench_suffix
    return bench_suffix


def main():
    prefixes = ["init"] + list(get_sample_methods().keys())

    opt, config = get_args(prefixes)
    bench_suffix = get_benchmark_suffix(opt)

    if opt.benchmark:
        if opt.prefix is not None:
            prefixes = [opt.prefix]
            bench.series_benchmark(config, opt, opt.prefix)
        if opt.batch_test is None:
            benchmark_all(prefixes, config, opt)
        else:
            benchmark_batch_test(prefixes, config, opt, opt.batch_test)

    if opt.tabulate:
        if opt.prefix is not None:
            # Specify a sampling prefix to view all metrics (conf, prec, acc, recall train length)
            # on a per-batch basis
            charts.tabulate_batch_samples(
                config,
                opt.prefix,
                filter_samp=opt.filter_sample,
                bench_suffix=bench_suffix,
            )
        else:
            # View the specified metric (with precision as default) for each batch,
            # across all sampling methods
            charts.compare_benchmarks(
                config,
                prefixes,
                opt.metric,
                opt.metric2,
                bench_suffix,
                compare_init=opt.compare_init,
                filter_sample=opt.filter_sample,
                use_median=opt.aggr_median,
            )

    elif opt.visualize_conf:
        # Generate a PDF graph of the confidence distributions for a specified benchmark file
        charts.visualize_conf(
            opt.prefix, opt.visualize_conf, opt.filter_sample, config["pos_thres"]
        )
    elif opt.view_benchmark is not None:
        charts.display_benchmark(opt.view_benchmark, config)

    elif opt.prefix is not None:
        charts.tabulate_batch_samples(config, opt.prefix, bench_suffix=bench_suffix)
        charts.display_series(config, opt)


if __name__ == "__main__":
    main()