train.py

"""
Train a diffusion model on images.
"""

import argparse
from datetime import datetime
import os
import torch.distributed as dist
import sys

from improved_diffusion import dist_util, logger
from improved_diffusion.resample import create_named_schedule_sampler
from improved_diffusion.script_util import (
    model_and_diffusion_defaults,
    create_model_and_diffusion,
    args_to_dict,
    add_dict_to_argparser,
)
from improved_diffusion.train_util import TrainLoop
from ood_utils import load_data, dict2namespace, yield_
import yaml


def main():
    args = create_argparser().parse_args()

    with open(args.config, 'r') as fp:
        config = yaml.safe_load(fp)
    config = dict2namespace(config)

    # merge config into args
    for arg_name, arg_value in vars(config).items():
        setattr(args, arg_name, arg_value)

    dist_util.setup_dist(1)
    timestamp = datetime.now().strftime("%Y_%m_%d_%H%M%S")
    logger.configure(os.path.join("results", args.dataset, timestamp))
    
    # log hyperparameters and running command
    if dist.get_rank() == 0:
        command_line = ' '.join(sys.argv)
        with open(os.path.join(logger.get_dir(), "args.txt"), 'w') as f:
            f.write(command_line)
            for arg_name, arg_value in vars(args).items():
                f.write(f"{arg_name}: {arg_value}\n")

    logger.log("creating model and diffusion...")
    model, diffusion = create_model_and_diffusion(
        **args_to_dict(args, model_and_diffusion_defaults().keys())
    )
    model.to(dist_util.dev())
    schedule_sampler = create_named_schedule_sampler(args.schedule_sampler, diffusion)
    
    logger.log("creating data loader...")
    data = load_data(
        dataset=args.dataset,
        data_dir=args.data_dir,
        batch_size=args.batch_size,
        image_size=args.image_size,
        train=True
    )
    data = yield_(data)

    logger.log("training...")
    TrainLoop(
        model=model,
        diffusion=diffusion,
        data=data,
        batch_size=args.batch_size,
        microbatch=args.microbatch,
        lr=args.lr,
        n_training_iters=args.n_training_iters,
        ema_rate=args.ema_rate,
        log_interval=args.log_interval,
        save_interval=args.save_interval,
        plot_interval=args.plot_interval,
        resume_checkpoint=args.resume_checkpoint,
        use_fp16=args.use_fp16,
        fp16_scale_growth=args.fp16_scale_growth,
        schedule_sampler=schedule_sampler,
        weight_decay=args.weight_decay,
        lr_anneal_steps=args.lr_anneal_steps
    ).run_loop()


def create_argparser():
    defaults = dict(
        config="",
        data_dir="",
        dataset="",
        weight_decay=0.0,
        lr_anneal_steps=0,
        microbatch=-1,  # -1 disables microbatches
        ema_rate="0.9999",  # comma-separated list of EMA values
        resume_checkpoint="",
        use_fp16=False,
        fp16_scale_growth=1e-3,
        clip_denoised=True
    )
    defaults.update(model_and_diffusion_defaults())
    parser = argparse.ArgumentParser()
    add_dict_to_argparser(parser, defaults)
    return parser


if __name__ == "__main__":
    main()