standardize plotting and gather procedures

.gitignore

@@ -105,15 +105,16 @@ ENV/
 .DS_Store
 .vscode
 /data
-log/
+log
+saved
 processed_log/
 notebooks/
 *.prof
 *.cpp
 !sound_law/rl/mcts_cpp/*.cpp
 scripts/*
 dump/*
-cmdl/*
+cmd/*
 .testmondata
 overleaf/*
 grid/*
@@ -125,3 +126,4 @@ ini/*
 *.c
 *.html
 *.o
+out/

scripts/gen_matching_scores.py

@@ -1,14 +1,38 @@
-from argparse import ArgumentParser
 import re
 import subprocess
-from tqdm import tqdm
-import torch
+from argparse import ArgumentParser
 from pathlib import Path
+from typing import Iterator, Tuple
+
+import torch
+from tqdm import tqdm
+
+
+def should_print(out_path_str: str, overwrite: bool) -> bool:
+    out_path = Path(out_path_str)
+    return not out_path.exists() or out_path.stat().st_size == 0 or overwrite
+
+
+def get_eval_paths(eval_folder: Path) -> Iterator[Tuple[int, Path]]:
+    """Yields tuples of (epoch, path) of saved action sequences (system output) given the folder.
+
+    Args:
+        eval_folder (Path): path to the evaluation folder.
+
+    Yields:
+        Iterator[Tuple[int, Path]]: an iterator of tuples (epoch, path).
+    """
+    for eval_path in eval_folder.glob('*.path'):
+        match = re.match(r'(\d+).path$', eval_path.name)
+        if match is not None:
+            epoch = int(match.group(1))
+            yield epoch, eval_path
+
 
 if __name__ == '__main__':
     parser = ArgumentParser()
     parser.add_argument('saved_folder', help='Path to the saved folder.')
-    parser.add_argument('mode', choices=['full', 'truncate', 'epoch', 'regress', 'irreg', 'merger'])
+    parser.add_argument('mode', choices=['full', 'truncate', 'epoch', 'regress', 'irreg', 'merger', 'evaluate'])
     parser.add_argument('--overwrite', action='store_true', help='Flag to override previous saved files.')
     args = parser.parse_args()
 
@@ -25,15 +49,11 @@
         tgt_lang = saved_dict['tgt_lang'].value
 
         if args.mode == 'epoch':
-            for eval_path in run.glob('eval/*.path'):
-                match = re.match(r'(\d+).path', eval_path.name)
-                if match is not None:
-                    epoch = match.group(1)
-                    base_cmd = f'python sound_law/evaluate/ilp.py --config OPRLPgmc{tgt_lang[0].upper()}{tgt_lang[1:]} --mcts_config SmallSims --cand_path {eval_path} --use_greedy_growth --silent'
-                    for m in [0.2, 0.4, 0.6, 0.8, 1.0]:
-                        out_path = f'{run}/eval/epoch{epoch}-{m}-100-10.pkl'
-                        if Path(out_path).exists() and not args.overwrite:
-                            continue
+            for epoch, eval_path in get_eval_paths(run / 'eval'):
+                base_cmd = f'python sound_law/evaluate/ilp.py --config OPRLPgmc{tgt_lang[0].upper()}{tgt_lang[1:]} --mcts_config SmallSims --cand_path {eval_path} --use_greedy_growth --silent'
+                for m in [0.2, 0.4, 0.6, 0.8, 1.0]:
+                    out_path = f'{run}/eval/epoch{epoch}-{m}-100-10.pkl'
+                    if should_print(out_path, args.overwrite):
                         print(
                             base_cmd + f' --match_proportion {m} --k_matches 100 --max_power_set_size 10 --out_path {out_path} --message {run}-epoch{epoch}-{m}-100-10')
         elif args.mode in ['full', 'truncate']:
@@ -45,10 +65,9 @@
                     for k in [10, 20, 30, 50, 100]:
                         for p in [1, 2, 3, 5, 10]:
                             out_path = f'{run}/eval/full-{m}-{k}-{p}.pkl'
-                            if Path(out_path).exists() and not args.overwrite:
-                                continue
-                            print(
-                                base_cmd + f' --match_proportion {m} --k_matches {k} --max_power_set_size {p} --out_path {out_path} --message {run}-full-{m}-{k}-{p}')
+                            if should_print(out_path, args.overwrite):
+                                print(
+                                    base_cmd + f' --match_proportion {m} --k_matches {k} --max_power_set_size {p} --out_path {out_path} --message {run}-full-{m}-{k}-{p}')
 
             else:
                 # Generate matching scores with different truncate lengths.
@@ -57,42 +76,40 @@
                                         text=True, capture_output=True, check=True).stdout
                 max_length = int(output)
                 for m in [0.2, 0.4, 0.6, 0.8, 1.0]:
-                    for l in range(5, max_length + 5, 5):
-                        out_path = f'{run}/eval/truncate-{m}-100-10-{l}.pkl'
-                        if Path(out_path).exists() and not args.overwrite:
-                            continue
-                        print(
-                            base_cmd + f' --match_proportion {m} --k_matches {100} --max_power_set_size {10} --out_path {out_path} --cand_length {l} --message {run}-truncate-{m}-100-10-{l}')
+                    for tl in range(5, max_length + 5, 5):
+                        pl = min(tl, max_length)
+                        out_path = f'{run}/eval/truncate-{m}-100-10-{pl}.pkl'
+                        if should_print(out_path, args.overwrite):
+                            print(
+                                base_cmd + f' --match_proportion {m} --k_matches {100} --max_power_set_size {10} --out_path {out_path} --cand_length {pl} --message {run}-truncate-{m}-100-10-{pl}')
+        elif args.mode == 'evaluate':
+            # Generate all commands to evaluate paths.
+            base_cmd = f'python scripts/evaluate.py --config OPRLPgmc{tgt_lang[0].upper()}{tgt_lang[1:]} --mcts_config SmallSims'
+            for epoch, eval_path in get_eval_paths(run / 'eval'):
+                in_path = f'{run}/eval/{epoch}.path'
+                out_path = in_path + '.scores'
+                if should_print(out_path, args.overwrite):
+                    print(base_cmd + f' --in_path {in_path} --out_path {out_path} --message {out_path}')
         else:
             base_cmd = f'python sound_law/evaluate/ilp.py --config OPRLPgmcGot --mcts_config SmallSims --in_path data/wikt/pgmc-{tgt_lang}/action_seq.tsv --cand_path {run}/eval/{best_run}.path --use_greedy_growth --silent --tgt_lang {tgt_lang}'
             if args.mode == 'irreg':
                 # Generate matching scores for synthetic runs, grouped by number of irregular changes.
                 for m in [0.2, 0.4, 0.6, 0.8, 1.0]:
                     out_path = f'{run}/eval/irreg-{m}-100-10.pkl'
-                    if Path(out_path).exists() and not args.overwrite:
-                        continue
-                    print(
-                        base_cmd + f' --match_proportion {m} --k_matches 100 --max_power_set_size 10 --out_path {out_path} --message {run}-irreg-{m}-100-10')
+                    if should_print(out_path, args.overwrite):
+                        print(
+                            base_cmd + f' --match_proportion {m} --k_matches 100 --max_power_set_size 10 --out_path {out_path} --message {run}-irreg-{m}-100-10')
             elif args.mode == 'regress':
                 # Generate matching scores for synthetic runs, grouped by number of regressive rules.
                 for m in [0.2, 0.4, 0.6, 0.8, 1.0]:
                     out_path = f'{run}/eval/regress-{m}-100-10.pkl'
-                    if Path(out_path).exists() and not args.overwrite:
-                        continue
-                    print(
-                        base_cmd + f' --match_proportion {m} --k_matches 100 --max_power_set_size 10 --out_path {out_path} --message {run}-regress-{m}-100-10')
+                    if should_print(out_path, args.overwrite):
+                        print(
+                            base_cmd + f' --match_proportion {m} --k_matches 100 --max_power_set_size 10 --out_path {out_path} --message {run}-regress-{m}-100-10')
             else:
                 # Generate matching scores for synthetic runs, grouped by number of mergers.
                 for m in [0.2, 0.4, 0.6, 0.8, 1.0]:
                     out_path = f'{run}/eval/merger-{m}-100-10.pkl'
-                    if Path(out_path).exists() and not args.overwrite:
-                        continue
-                    print(
-                        base_cmd + f' --match_proportion {m} --k_matches 100 --max_power_set_size 10 --out_path {out_path} --message {run}-merger-{m}-100-10')
-
-            # # Generate all commands to evaluate paths.
-            # out_path = Path(f'{run}/eval/{best_run}.path.scores')
-            # cmd = f'python scripts/evaluate.py --config OPRLPgmc{tgt_lang[0].upper()}{tgt_lang[1:]} --mcts_config SmallSims --in_path {run}/eval/{best_run}.path --out_path {out_path}'
-            # if Path(out_path).exists() and not args.overwrite:
-            #     continue
-            # print(cmd)
+                    if should_print(out_path, args.overwrite):
+                        print(
+                            base_cmd + f' --match_proportion {m} --k_matches 100 --max_power_set_size 10 --out_path {out_path} --message {run}-merger-{m}-100-10')

scripts/get_run_df.py

@@ -0,0 +1,64 @@
+import re
+from argparse import ArgumentParser
+from pathlib import Path
+
+import pandas as pd
+import torch
+from sound_law.utils import (load_event, load_stats, read_distance_metrics,
+                             read_matching_metrics)
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('folder', type=str)
+    parser.add_argument('mode', choices=['run', 'irreg', 'regress', 'merger'], type=str)
+    parser.add_argument('prefix', type=str)
+    args = parser.parse_args()
+
+    folder = Path(args.folder)
+
+    if args.mode == 'run':
+        runs = list(folder.glob('*/'))
+        # Get all matching scores.
+        match_dfs = list()
+        event_dfs = list()
+        dist_dfs = list()
+        meta_data = list()
+        for run in runs:
+            saved_dict = {k: v.value for k, v in torch.load(run / 'hparams.pth').items()}
+            lang = saved_dict['tgt_lang']
+            with open(run / 'best_run', 'r') as fin:
+                meta_record = {'best_epoch': int(fin.read(-1)), 'run': str(run)}
+                meta_record.update(saved_dict)
+                meta_data.append(meta_record)
+
+            # FIXME(j_luo) check we are not missing any entry for the dfs.
+            match_df = read_matching_metrics(run).assign(run=str(run))
+            event_df = load_event(run).assign(run=str(run))
+            dist_df = read_distance_metrics(run).assign(run=str(run))
+            match_dfs.append(match_df)
+            event_dfs.append(event_df)
+            dist_dfs.append(dist_df)
+
+        all_match_df = pd.concat(match_dfs, ignore_index=True)
+        all_event_df = pd.concat(event_dfs, ignore_index=True)
+        all_dist_df = pd.concat(dist_dfs, ignore_index=True)
+        meta_df = pd.DataFrame(meta_data)
+        all_match_df.to_csv(f'{args.prefix}_match.tsv', sep='\t', index=False)
+        all_event_df.to_csv(f'{args.prefix}_event.tsv', sep='\t', index=False)
+        all_dist_df.to_csv(f'{args.prefix}_dist.tsv', sep='\t', index=False)
+        meta_df.to_csv(f'{args.prefix}_meta.tsv', sep='\t', index=False)
+    else:
+        # Get all the data folders based on the mode.
+        if args.mode == 'irreg':
+            runs = [f'data/wikt/pgmc-rand{i}' for i in range(1, 51)]
+        elif args.mode == 'regress':
+            runs = [f'data/wikt/pgmc-rand-regress{i}' for i in range(1, 51)]
+        else:
+            runs = [f'data/wikt/pgmc-rand-merger{i}' for i in range(1, 51)]
+
+        stats_dfs = list()
+        for run in runs:
+            stats_df = load_stats(run).assign(data_folder=str(run))
+            stats_dfs.append(stats_df)
+        all_stats_df = pd.concat(stats_dfs, ignore_index=True)
+        all_stats_df.to_csv(f'{args.prefix}_stats.tsv', sep='\t', index=False)

sound_law/main.py

@@ -4,10 +4,10 @@
 
 import numpy as np
 import torch
-
 from dev_misc import Initiator, add_argument, g, parse_args
 from dev_misc.devlib.named_tensor import patch_named_tensors
 from dev_misc.trainlib import set_random_seeds
+
 from sound_law.config import a2c_reg, mcts_reg, reg, s2s_reg
 from sound_law.train.manager import OnePairManager, OneToManyManager
 

sound_law/utils.py

@@ -80,22 +80,24 @@ def read_matching_metrics(folder_path: PathLike) -> PDF:
 
     Returns:
         pd.DataFrame: the dataframe that summarizes all metrics. It contains the following columns:
-            match_name, match_proportion, k_matches, max_power_set_size, truncate_length and match_score.
+            match_name, match_proportion, k_matches, max_power_set_size, path_length, epoch, and match_score.
+            `epoch` is set to be `None`, reserved for the best run.
     """
     folder_path = Path(folder_path)
     # Pattern used for the majority of all matching results.
-    match_pat1 = re.compile(r'(?P<name>\w+)-(?P<mp>\d+)-(?P<km>\d+)-(?P<mpss>\d+)(-(?P<tl>\d+))?.pkl')
+    num = r'[\d\.]+'
+    match_pat1 = re.compile(rf'(?P<name>[a-zA-Z_]+)-(?P<mp>{num})-(?P<km>{num})-(?P<mpss>{num})(-(?P<pl>{num}))?\.pkl')
     # Pattern used for matching results with epoch number.
-    match_pat2 = re.compile(r'epoch(?P<epoch>\d+)-(?P<mp>\d+)-(?P<km>\d+)-(?P<mpss>\d+).pkl')
+    match_pat2 = re.compile(rf'epoch(?P<epoch>{num})-(?P<mp>{num})-(?P<km>{num})-(?P<mpss>{num})\.pkl')
 
     records = list()
-    for file_path in folder_path.glob('*.pkl'):
+    for file_path in folder_path.glob('eval/*.pkl'):
         match = match_pat1.match(file_path.name)
         if match is None:
-            matched_pat = 1
-        else:
             match = match_pat2.match(file_path.name)
             matched_pat = 2
+        else:
+            matched_pat = 1
 
         if match is None:
             continue
@@ -104,14 +106,14 @@ def read_matching_metrics(folder_path: PathLike) -> PDF:
         mp = match.group('mp')
         km = match.group('km')
         mpss = match.group('mpss')
-        tl = match.group('tl') if matched_pat == 1 else None
+        pl = match.group('pl') if matched_pat == 1 else None
         epoch = match.group('epoch') if matched_pat == 2 else None
         score = read_matching_score(file_path)
         records.append({'match_name': name,
                         'match_proportion': mp,
                         'k_matches': km,
                         'max_power_set_size': mpss,
-                        'truncate_length': tl,
+                        'path_length': pl,
                         'epoch': epoch,
                         'match_score': score})
     match_df = pd.DataFrame(records)
@@ -125,7 +127,7 @@ class Record:
     wall_time: float
     tag: str
     value: float
-    step: Optional[int] = None
+    epoch: Optional[int] = None
 
 
 class EventFile:
@@ -161,11 +163,11 @@ def __iter__(self) -> Iterator[Record]:
                 if abs(value - 2.0) < 1e-6 and tag == 'best_score':
                     value = 1.0
                 try:
-                    step = int(e['step'])
+                    epoch = int(e['step'])
                 except KeyError:
-                    step = default_step[tag]
+                    epoch = default_step[tag]
                     default_step[tag] += 1
-                yield Record(wall_time, tag, value, step=step)
+                yield Record(wall_time, tag, value, epoch=epoch)
             except KeyError:
                 pass
 
@@ -197,7 +199,7 @@ def read_distance_metrics(run_folder: PathLike) -> PDF:
 
     Returns:
         PDF: a dataframe summarizing all results. It includes columns:
-            truncate_length, epoch, and distance_score.
+            path_length, epoch, and distance_score.
     """
     run_folder = Path(run_folder)
     records = list()
@@ -208,8 +210,8 @@ def read_distance_metrics(run_folder: PathLike) -> PDF:
             for line in fin:
                 truncated_dists.append(float(line))
         start_dist = truncated_dists[0]
-        for dist, l in enumerate(truncated_dists[1:], 1):
-            records.append({'truncate_length': l,
+        for l, dist in enumerate(truncated_dists[1:], 1):
+            records.append({'path_length': l,
                             'epoch': epoch,
                             'distance_score': 1.0 - dist / start_dist})
     record_df = pd.DataFrame(records)