Sketch: Crawling Job Runs

import json
from pathlib import Path

def crawl_job_runs(job_directory):
    job_directory = Path(job_directory)
    job_json = job_directory / 'job.json'

    with open(job_json, 'r') as f:
        job_data = json.load(f)

    run_data = []
    runs_dir = job_directory / 'runs'
    if runs_dir.exists():
        for run_directory in runs_dir.iterdir():
            run_json = run_directory / 'run.json'
            with open(run_json, 'r') as f:
                run_info = json.load(f)
                run_info.update(job_data)
                run_data.append(run_info)

    return run_data

data = crawl_job_runs("output/my-super-job")