From 488759d2c9a55ad3ed55d55417b7733e95e37667 Mon Sep 17 00:00:00 2001 From: LSinev Date: Sun, 28 Jan 2024 23:44:08 +0300 Subject: [PATCH] Apply some best practices and guideline recommendations to code (#1363) * raise Exception, not a string Additional info https://peps.python.org/pep-0352/#exception-hierarchy-changes https://docs.python.org/3.8/tutorial/errors.html#raising-exceptions * Apply PEP8 recommendation to prefer isinstance "Object type comparisons should always use isinstance() instead of comparing types directly" https://peps.python.org/pep-0008/ * Remove dangerous default mutable values in arguments https://pylint.readthedocs.io/en/stable/user_guide/messages/warning/dangerous-default-value.html * Format logging messages with fstring (not with format) Additional info https://pylint.readthedocs.io/en/stable/user_guide/messages/warning/logging-format-interpolation.html There are also discussions about the speed of formatting while logging or some unintended code executions https://github.com/pylint-dev/pylint/issues/2395 https://stackoverflow.com/a/54368109 but at least one format (fstring one) will be used throughout the project * Specify utf-8 encoding for `open` explicitly If not specified, it may be supposed differently in different environments, OSes, and Python versions. See https://peps.python.org/pep-0597/ https://docs.python.org/3.11/library/locale.html#locale.getencoding https://docs.python.org/3.10/library/os.html#utf8-mode https://pylint.readthedocs.io/en/stable/user_guide/messages/warning/unspecified-encoding.html Helps also if some code from English language tasks is taken as inspiration for tasks in non-English languages. * Use inline-ignoring comments to pass pre-commit instead of identity process https://flake8.pycqa.org/en/3.0.1/user/ignoring-errors.html#in-line-ignoring-errors https://www.flake8rules.com/rules/F841.html flake8 comments are supported by ruff: https://docs.astral.sh/ruff/linter/#error-suppression --- lm_eval/__main__.py | 4 ++-- lm_eval/api/registry.py | 8 ++------ lm_eval/decontamination/archiver.py | 6 ++++-- lm_eval/decontamination/decontaminate.py | 2 +- lm_eval/evaluator.py | 18 +++++++++-------- lm_eval/filters/transformation.py | 4 +++- lm_eval/models/huggingface.py | 3 +-- lm_eval/prompts/__init__.py | 2 +- lm_eval/tasks/__init__.py | 14 ++++++------- lm_eval/tasks/bbh/_generate_configs.py | 4 ++-- lm_eval/tasks/belebele/_generate_configs.py | 6 +++--- lm_eval/tasks/bigbench/generate_tasks.py | 2 +- lm_eval/tasks/blimp/generate_configs.py | 2 +- lm_eval/tasks/ceval/_generate_configs.py | 6 +++--- lm_eval/tasks/cmmlu/_generate_configs.py | 6 +++--- lm_eval/tasks/code_x_glue/code-text/bleu.py | 2 +- lm_eval/tasks/csatqa/_generate_configs.py | 4 ++-- lm_eval/tasks/mmlu/_generate_configs.py | 8 ++++---- .../advanced_ai_risk/_generate_configs.py | 2 +- .../persona/_generate_configs.py | 2 +- lm_eval/tasks/qasper/utils.py | 2 +- lm_eval/tasks/xwinograd/utils.py | 2 +- scripts/build_benchmark.py | 4 ++-- .../clean_training_data/generate_13_grams.py | 4 ++-- .../clean_training_data/investigate_pile.py | 4 ++-- scripts/make_table_results.py | 6 +++--- scripts/make_table_tasks.py | 2 +- scripts/regression.py | 6 +++++- scripts/write_out.py | 2 +- scripts/zeno_visualize.py | 20 +++++++++++-------- 30 files changed, 83 insertions(+), 74 deletions(-) diff --git a/lm_eval/__main__.py b/lm_eval/__main__.py index 37fdabc6df..4f862710bf 100644 --- a/lm_eval/__main__.py +++ b/lm_eval/__main__.py @@ -171,7 +171,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: task_names = ALL_TASKS elif args.tasks == "list": eval_logger.info( - "Available Tasks:\n - {}".format("\n - ".join(sorted(ALL_TASKS))) + f"Available Tasks:\n - {(os.linesep + ' - ').join(sorted(ALL_TASKS))}" ) sys.exit() else: @@ -257,7 +257,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: batch_sizes = ",".join(map(str, results["config"]["batch_sizes"])) if args.output_path: - output_path_file.open("w").write(dumped) + output_path_file.open("w", encoding="utf-8").write(dumped) if args.log_samples: for task_name, config in results["configs"].items(): diff --git a/lm_eval/api/registry.py b/lm_eval/api/registry.py index 5fb9c011fc..7354e8b6f1 100644 --- a/lm_eval/api/registry.py +++ b/lm_eval/api/registry.py @@ -152,18 +152,14 @@ def get_aggregation(name): try: return AGGREGATION_REGISTRY[name] except KeyError: - eval_logger.warning( - "{} not a registered aggregation metric!".format(name), - ) + eval_logger.warning(f"{name} not a registered aggregation metric!") def get_metric_aggregation(name): try: return METRIC_AGGREGATION_REGISTRY[name] except KeyError: - eval_logger.warning( - "{} metric is not assigned a default aggregation!".format(name), - ) + eval_logger.warning(f"{name} metric is not assigned a default aggregation!") def is_higher_better(metric_name): diff --git a/lm_eval/decontamination/archiver.py b/lm_eval/decontamination/archiver.py index e6bff33f0c..fa8a715f78 100644 --- a/lm_eval/decontamination/archiver.py +++ b/lm_eval/decontamination/archiver.py @@ -30,7 +30,9 @@ def __init__(self, file_path: str, compression_level: int = 3) -> None: self.cctx = zstandard.ZstdCompressor(level=compression_level) self.compressor = self.cctx.stream_writer(self.fh) - def add_data(self, data, meta={}) -> None: + def add_data(self, data, meta=None) -> None: + if meta is None: + meta = {} self.compressor.write( json.dumps({"text": data, "meta": meta}, default=json_serial).encode( "UTF-8" @@ -108,7 +110,7 @@ def __init__(self, file_path) -> None: def read_tqdm(self, update_frequency: int = 10000): current_file_position = 0 line_counter = 0 - with open(self.file_path, "r") as fh, tqdm.tqdm( + with open(self.file_path, "r", encoding="utf-8") as fh, tqdm.tqdm( total=os.path.getsize(self.file_path), dynamic_ncols=True, unit="byte", diff --git a/lm_eval/decontamination/decontaminate.py b/lm_eval/decontamination/decontaminate.py index f5b4157c67..3874eb58be 100644 --- a/lm_eval/decontamination/decontaminate.py +++ b/lm_eval/decontamination/decontaminate.py @@ -38,7 +38,7 @@ def get_train_overlap(docs_by_task_set: dict, ngrams_path: str, limit: int) -> d # return get_train_overlap_stub(docs, ngrams_path, ngrams_n_size) info_dict_path = os.path.join(ngrams_path, "info.json") - info_dict = json.load(open(info_dict_path, "r")) + info_dict = json.load(open(info_dict_path, "r", encoding="utf-8")) ngrams_n_size = info_dict["ngram_size"] janitor = Janitor() diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index 9c430e779b..657fdcfef0 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -25,7 +25,7 @@ def simple_evaluate( model, model_args=None, - tasks=[], + tasks=None, num_fewshot=None, batch_size=None, max_batch_size=None, @@ -80,6 +80,8 @@ def simple_evaluate( 1234 ) # TODO: this may affect training runs that are run with evaluation mid-run. + if tasks is None: + tasks = [] assert ( tasks != [] ), "No tasks specified, or no tasks found. Please verify the task names." @@ -122,7 +124,7 @@ def simple_evaluate( task_dict = lm_eval.tasks.get_task_dict(tasks) for task_name in task_dict.keys(): task_obj = task_dict[task_name] - if type(task_obj) == tuple: + if isinstance(task_obj, tuple): group, task_obj = task_obj if task_obj is None: continue @@ -242,7 +244,7 @@ def evaluate( # get lists of each type of request for task_name, task in task_dict.items(): - if type(task) == tuple: + if isinstance(task, tuple): group_name, task = task task_hierarchy[group_name].append(task_name) versions[group_name] = "N/A" @@ -316,7 +318,7 @@ def evaluate( ### Run LM on inputs, get all outputs ### # execute each type of request for reqtype, reqs in requests.items(): - eval_logger.info("Running {} requests".format(reqtype)) + eval_logger.info(f"Running {reqtype} requests") # create `K` copies of each request `req` based off `K = req.repeats` cloned_reqs = [] for req in reqs: @@ -339,7 +341,7 @@ def evaluate( ### Postprocess outputs ### # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately) for task_name, task in task_dict.items(): - if type(task) == tuple: + if isinstance(task, tuple): group, task = task if task is None: continue @@ -350,7 +352,7 @@ def evaluate( # unpack results and sort back in order and return control to Task for task_name, task in task_dict.items(): - if type(task) == tuple: + if isinstance(task, tuple): group, task = task if task is None: continue @@ -401,7 +403,7 @@ def evaluate( vals_torch = collections.defaultdict(list) for (task_name, key, metric), items in vals.items(): numitem = 0 - if type(items[0]) == tuple: + if isinstance(items[0], tuple): numitem = len(items[0]) if isinstance(items[0], (str, list, tuple)): @@ -447,7 +449,7 @@ def evaluate( task = task_dict[task_name] metric_key = metric + "," + key - if type(task) == tuple: + if isinstance(task, tuple): group_name, task = task else: group_name = None diff --git a/lm_eval/filters/transformation.py b/lm_eval/filters/transformation.py index f254b0db1b..41d03df7e1 100644 --- a/lm_eval/filters/transformation.py +++ b/lm_eval/filters/transformation.py @@ -24,7 +24,7 @@ def filter_set(inst): class MapFilter(Filter): - def __init__(self, mapping_dict: dict = {}, default_value=None) -> None: + def __init__(self, mapping_dict: dict = None, default_value=None) -> None: """ Initializes the MapFilter with a given mapping dictionary and default value. @@ -37,6 +37,8 @@ def __init__(self, mapping_dict: dict = {}, default_value=None) -> None: Example: mapper = MapFilter({'A': 1, 'B': 2}, default_value=0) """ + if mapping_dict is None: + mapping_dict = {} assert isinstance( mapping_dict, dict ), "Provided mapping_dict is not a dictionary" diff --git a/lm_eval/models/huggingface.py b/lm_eval/models/huggingface.py index e7ebcd11d6..d5eb645b65 100644 --- a/lm_eval/models/huggingface.py +++ b/lm_eval/models/huggingface.py @@ -603,8 +603,7 @@ def forward_batch(batch_size): (batch_size, max_length), device=self.device ).long() for _ in range(5): - out = F.log_softmax(self._model_call(test_batch, **call_kwargs), dim=-1) - out = out # Identity process so that it passes pre-commit + out = F.log_softmax(self._model_call(test_batch, **call_kwargs), dim=-1) # noqa: F841 return batch_size diff --git a/lm_eval/prompts/__init__.py b/lm_eval/prompts/__init__.py index d8b62e7deb..c505113a3d 100644 --- a/lm_eval/prompts/__init__.py +++ b/lm_eval/prompts/__init__.py @@ -117,7 +117,7 @@ def apply(self, doc): # TODO need a way to process doc_to_choice if "doc_to_choice" in self.prompt_string: - raise "Not yet implemented to accept doc_to_choice" + raise Exception("Not yet implemented to accept doc_to_choice") text_string = utils.apply_template(doc_to_text, doc) target_string = utils.apply_template(doc_to_target, doc) diff --git a/lm_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py index 5b7142496e..fed0c1d0ed 100644 --- a/lm_eval/tasks/__init__.py +++ b/lm_eval/tasks/__init__.py @@ -43,7 +43,7 @@ def register_configurable_task(config: Dict[str, str]) -> int: if "group" in config: if config["group"] == config["task"]: raise ValueError("task and group name cannot be the same") - elif type(config["group"]) == str: + elif isinstance(config["group"], str): group_name = [config["group"]] else: group_name = config["group"] @@ -57,8 +57,8 @@ def register_configurable_task(config: Dict[str, str]) -> int: def register_configurable_group(config: Dict[str, str], yaml_path: str = None) -> int: group = config["group"] all_task_list = config["task"] - config_list = [task for task in all_task_list if type(task) != str] - task_list = [task for task in all_task_list if type(task) == str] + config_list = [task for task in all_task_list if not isinstance(task, str)] + task_list = [task for task in all_task_list if isinstance(task, str)] for task_config in config_list: @@ -68,7 +68,7 @@ def register_configurable_group(config: Dict[str, str], yaml_path: str = None) - task_name = task_config["task"] if task_name in ALL_TASKS: task_obj = TASK_REGISTRY[task_name] - if type(task_obj) == tuple: + if isinstance(task_obj, tuple): _, task_obj = task_obj if task_obj is not None: @@ -166,10 +166,10 @@ def include_task_folder(task_dir: str, register_task: bool = True) -> None: ) for config in all_configs: if register_task: - if type(config["task"]) == str: + if isinstance(config["task"], str): register_configurable_task(config) else: - if type(config["task"]) == list: + if isinstance(config["task"], list): register_configurable_group(config, yaml_path) # Log this silently and show it only when @@ -243,7 +243,7 @@ def get_task_dict(task_name_list: List[Union[str, Dict, Task]], **kwargs): task_name_from_config_dict = {} task_name_from_object_dict = {} - if type(task_name_list) != list: + if not isinstance(task_name_list, list): task_name_list = [task_name_list] for task_element in task_name_list: diff --git a/lm_eval/tasks/bbh/_generate_configs.py b/lm_eval/tasks/bbh/_generate_configs.py index 18a55c705a..0d085a1d0a 100644 --- a/lm_eval/tasks/bbh/_generate_configs.py +++ b/lm_eval/tasks/bbh/_generate_configs.py @@ -28,7 +28,7 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) base_doc_to_text = "Q: {{input}}\nA:" @@ -70,7 +70,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"/{task}.yaml" utils.eval_logger.info(f"Saving yaml for subset {task} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/belebele/_generate_configs.py b/lm_eval/tasks/belebele/_generate_configs.py index 54274a419e..f1d76cad7e 100644 --- a/lm_eval/tasks/belebele/_generate_configs.py +++ b/lm_eval/tasks/belebele/_generate_configs.py @@ -27,13 +27,13 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) if args.cot_prompt_path is not None: import json - with open(args.cot_prompt_path) as f: + with open(args.cot_prompt_path, encoding="utf-8") as f: cot_file = json.load(f) def query(): @@ -54,7 +54,7 @@ def query(): file_save_path = args.save_prefix_path + f"_{lang}.yaml" logging.info(f"Saving yaml for subset {lang} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/bigbench/generate_tasks.py b/lm_eval/tasks/bigbench/generate_tasks.py index 3fd5cd6c2b..08fd0c0a59 100644 --- a/lm_eval/tasks/bigbench/generate_tasks.py +++ b/lm_eval/tasks/bigbench/generate_tasks.py @@ -181,7 +181,7 @@ def main() -> None: for task in all_subtasks: file_name = f"{task}.yaml" try: - with open(f"{path}/{file_name}", "w") as f: + with open(f"{path}/{file_name}", "w", encoding="utf-8") as f: f.write("# Generated by utils.py\n") yaml.dump( { diff --git a/lm_eval/tasks/blimp/generate_configs.py b/lm_eval/tasks/blimp/generate_configs.py index dfc4b4dc95..a768196172 100644 --- a/lm_eval/tasks/blimp/generate_configs.py +++ b/lm_eval/tasks/blimp/generate_configs.py @@ -75,7 +75,7 @@ def main() -> None: for task in all_subtasks: file_name = f"{task}.yaml" try: - with open(f"{file_name}", "w") as f: + with open(f"{file_name}", "w", encoding="utf-8") as f: f.write("# Generated by utils.py\n") yaml.dump( { diff --git a/lm_eval/tasks/ceval/_generate_configs.py b/lm_eval/tasks/ceval/_generate_configs.py index 2b96e00713..2df8ca31e4 100644 --- a/lm_eval/tasks/ceval/_generate_configs.py +++ b/lm_eval/tasks/ceval/_generate_configs.py @@ -79,13 +79,13 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) if args.cot_prompt_path is not None: import json - with open(args.cot_prompt_path) as f: + with open(args.cot_prompt_path, encoding="utf-8") as f: cot_file = json.load(f) for subject_eng, subject_zh in tqdm(SUBJECTS.items()): @@ -107,7 +107,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"_{subject_eng}.yaml" eval_logger.info(f"Saving yaml for subset {subject_eng} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/cmmlu/_generate_configs.py b/lm_eval/tasks/cmmlu/_generate_configs.py index 07553bb1ea..3afb15bf84 100644 --- a/lm_eval/tasks/cmmlu/_generate_configs.py +++ b/lm_eval/tasks/cmmlu/_generate_configs.py @@ -94,13 +94,13 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) if args.cot_prompt_path is not None: import json - with open(args.cot_prompt_path) as f: + with open(args.cot_prompt_path, encoding="utf-8") as f: cot_file = json.load(f) for subject_eng, subject_zh in tqdm(SUBJECTS.items()): @@ -122,7 +122,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"_{subject_eng}.yaml" eval_logger.info(f"Saving yaml for subset {subject_eng} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/code_x_glue/code-text/bleu.py b/lm_eval/tasks/code_x_glue/code-text/bleu.py index a90fc46b17..7f89404649 100644 --- a/lm_eval/tasks/code_x_glue/code-text/bleu.py +++ b/lm_eval/tasks/code_x_glue/code-text/bleu.py @@ -184,7 +184,7 @@ def splitPuncts(line): def computeMaps(predictions, goldfile): predictionMap: Dict[str, list] = {} goldMap: Dict[str, list] = {} - gf = open(goldfile, "r") + gf = open(goldfile, "r", encoding="utf-8") for row in predictions: cols = row.strip().split("\t") diff --git a/lm_eval/tasks/csatqa/_generate_configs.py b/lm_eval/tasks/csatqa/_generate_configs.py index 56fe825a90..bd849c0ae6 100644 --- a/lm_eval/tasks/csatqa/_generate_configs.py +++ b/lm_eval/tasks/csatqa/_generate_configs.py @@ -25,7 +25,7 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) for name in tqdm(SUBSETS): @@ -39,7 +39,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"_{name.lower()}.yaml" eval_logger.info(f"Saving yaml for subset {name} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/mmlu/_generate_configs.py b/lm_eval/tasks/mmlu/_generate_configs.py index e6271bc4c2..1424814e7d 100644 --- a/lm_eval/tasks/mmlu/_generate_configs.py +++ b/lm_eval/tasks/mmlu/_generate_configs.py @@ -85,13 +85,13 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) if args.cot_prompt_path is not None: import json - with open(args.cot_prompt_path) as f: + with open(args.cot_prompt_path, encoding="utf-8") as f: cot_file = json.load(f) ALL_CATEGORIES = [] @@ -120,7 +120,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"_{subject}.yaml" eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, @@ -142,7 +142,7 @@ def parse_args(): file_save_path = args.save_prefix_path + ".yaml" eval_logger.info(f"Saving benchmark config to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( { "group": f"mmlu_{args.task_prefix}" diff --git a/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py b/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py index aecb40a5eb..3a2bac5923 100644 --- a/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py +++ b/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py @@ -9,7 +9,7 @@ def main() -> None: for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): file_name = f"{task}.yaml" try: - with open(f"{file_name}", "w") as f: + with open(f"{file_name}", "w", encoding="utf-8") as f: f.write("# Generated by _generate_configs.py\n") yaml.dump( { diff --git a/lm_eval/tasks/model_written_evals/persona/_generate_configs.py b/lm_eval/tasks/model_written_evals/persona/_generate_configs.py index 7aff892f03..811e0b1b62 100644 --- a/lm_eval/tasks/model_written_evals/persona/_generate_configs.py +++ b/lm_eval/tasks/model_written_evals/persona/_generate_configs.py @@ -9,7 +9,7 @@ def main() -> None: for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): file_name = f"{task}.yaml" try: - with open(f"{file_name}", "w") as f: + with open(f"{file_name}", "w", encoding="utf-8") as f: f.write("# Generated by _generate_configs.py\n") yaml.dump( { diff --git a/lm_eval/tasks/qasper/utils.py b/lm_eval/tasks/qasper/utils.py index 7a02237a78..21f5be2435 100644 --- a/lm_eval/tasks/qasper/utils.py +++ b/lm_eval/tasks/qasper/utils.py @@ -50,7 +50,7 @@ def _flatten(doc): obs_list["abstract"].append(abstract) obs_list["question"].append(question) obs_list["answer_type"].append(answer_type) - if type(answer) == list: + if isinstance(answer, list): answer = ", ".join(answer) obs_list["answer"].append(answer) diff --git a/lm_eval/tasks/xwinograd/utils.py b/lm_eval/tasks/xwinograd/utils.py index 3bb6b1452e..97c93c7072 100644 --- a/lm_eval/tasks/xwinograd/utils.py +++ b/lm_eval/tasks/xwinograd/utils.py @@ -51,7 +51,7 @@ def gen_lang_yamls(output_dir: str, overwrite: bool) -> None: for lang in LANGUAGES: file_name = f"xwinograd_{lang}.yaml" try: - with open(f"{output_dir}/{file_name}", "w" if overwrite else "x") as f: + with open(f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf-8") as f: f.write("# Generated by utils.py\n") yaml.dump( { diff --git a/scripts/build_benchmark.py b/scripts/build_benchmark.py index ce4b661681..fc99b5ec37 100644 --- a/scripts/build_benchmark.py +++ b/scripts/build_benchmark.py @@ -23,7 +23,7 @@ def parse_args(): if __name__ == "__main__": args = parse_args() - with open(args.benchmark_path) as file: + with open(args.benchmark_path, encoding="utf-8") as file: TASK_LIST = yaml.full_load(file) for task in tqdm(TASK_LIST): eval_logger.info(f"Processing {task}") @@ -57,5 +57,5 @@ def parse_args(): file_save_path = os.path.join(file_path, full_file_name) eval_logger.info(f"Save to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump(config_dict, yaml_file) diff --git a/scripts/clean_training_data/generate_13_grams.py b/scripts/clean_training_data/generate_13_grams.py index 66fa0ff45b..e508f266e9 100644 --- a/scripts/clean_training_data/generate_13_grams.py +++ b/scripts/clean_training_data/generate_13_grams.py @@ -119,7 +119,7 @@ def close_buckets(self): def do_ngrams_in_buckets(n_value, working_directory, bucket_count): - pile_statistics = json.load(open("pile_statistics.json", "r")) + pile_statistics = json.load(open("pile_statistics.json", "r", encoding="utf-8")) pile_document_count = pile_statistics["Document Count"] start_offsets = pile_statistics["File Start Offsets"] @@ -212,4 +212,4 @@ def do_ngrams_in_buckets(n_value, working_directory, bucket_count): info_dict = {"title": "dataset ngrams", "ngram_size": 13} info_dict_path = os.path.join(args.working_directory, "info.json") - json.dump(info_dict, open(info_dict_path, "w")) + json.dump(info_dict, open(info_dict_path, "w", encoding="utf-8")) diff --git a/scripts/clean_training_data/investigate_pile.py b/scripts/clean_training_data/investigate_pile.py index c1d348d463..681b591ced 100644 --- a/scripts/clean_training_data/investigate_pile.py +++ b/scripts/clean_training_data/investigate_pile.py @@ -79,7 +79,7 @@ def on_error(_): stats_file_path = "pile_statistics.json" if os.path.exists(stats_file_path): - stats = json.load(open(stats_file_path, "r")) + stats = json.load(open(stats_file_path, "r", encoding="utf-8")) else: document_count, total_document_size_chars, start_offsets = get_stats() stats = { @@ -88,7 +88,7 @@ def on_error(_): "Total Pile Characters": total_document_size_chars, "File Start Offsets": start_offsets, } - json.dump(stats, open(stats_file_path, "w"), indent=4) + json.dump(stats, open(stats_file_path, "w", encoding="utf-8"), indent=4) print(f"document_count: {stats['Document Count']}") print(f"total_chars: {stats['Total Pile Characters']}") diff --git a/scripts/make_table_results.py b/scripts/make_table_results.py index 72af524ffe..2893c2b0e5 100644 --- a/scripts/make_table_results.py +++ b/scripts/make_table_results.py @@ -61,14 +61,14 @@ def make_table(result_dict): if not filenames: continue path_readme = os.path.join(dirpath, "README.md") - with open(path_readme, "w") as f: + with open(path_readme, "w", encoding="utf-8") as f: # get path name, only last folder path_name = dirpath.split("/")[-1] f.write(f"# {path_name} \n\n") for filename in sorted([f for f in filenames if f.endswith(".json")]): path = os.path.join(dirpath, filename) - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: result_dict = json.load(f) - with open(path_readme, "a") as f: + with open(path_readme, "a", encoding="utf-8") as f: f.write(f"## {filename} \n") f.write(f"{make_table(result_dict)} \n") diff --git a/scripts/make_table_tasks.py b/scripts/make_table_tasks.py index ded7c1a596..0c8c44bc65 100644 --- a/scripts/make_table_tasks.py +++ b/scripts/make_table_tasks.py @@ -50,5 +50,5 @@ def check(tf): values.append(v) writer.value_matrix = values table = writer.dumps() - with open(args.output, "w") as f: + with open(args.output, "w", encoding="utf-8") as f: f.write(table) diff --git a/scripts/regression.py b/scripts/regression.py index 2b8167c0eb..75258dcb64 100644 --- a/scripts/regression.py +++ b/scripts/regression.py @@ -94,7 +94,11 @@ def eval_models(args, branch=None): ret = os.system(command) - results[model] = json.load(open(output_path)) if ret == 0 else {"results": {}} + results[model] = ( + json.load(open(output_path, encoding="utf-8")) + if ret == 0 + else {"results": {}} + ) end_time = time.time() diff --git a/scripts/write_out.py b/scripts/write_out.py index 360b0b6271..3e8dcab7f5 100644 --- a/scripts/write_out.py +++ b/scripts/write_out.py @@ -53,7 +53,7 @@ def main(): os.makedirs(args.output_base_path, exist_ok=True) for task_name, task in task_dict.items(): - if type(task) == tuple: + if isinstance(task, tuple): group_name, task = task rnd = random.Random() rnd.seed(args.seed) diff --git a/scripts/zeno_visualize.py b/scripts/zeno_visualize.py index f4012afeca..c7b75a80e2 100644 --- a/scripts/zeno_visualize.py +++ b/scripts/zeno_visualize.py @@ -69,18 +69,20 @@ def main(): model_args = re.sub( "/|=", "__", - json.load(open(Path(args.data_path, model, "results.json")))["config"][ - "model_args" - ], + json.load( + open(Path(args.data_path, model, "results.json"), encoding="utf-8") + )["config"]["model_args"], ) with open( - Path(args.data_path, model, f"{model_args}_{task}.jsonl"), "r" + Path(args.data_path, model, f"{model_args}_{task}.jsonl"), + "r", + encoding="utf-8", ) as file: data = json.loads(file.read()) - configs = json.load(open(Path(args.data_path, model, "results.json")))[ - "configs" - ] + configs = json.load( + open(Path(args.data_path, model, "results.json"), encoding="utf-8") + )["configs"] config = configs[task] if model_index == 0: # Only need to assemble data for the first model @@ -124,7 +126,9 @@ def tasks_for_model(model: str, data_path: str): list: A list of tasks for the model. """ dir_path = Path(data_path, model) - config = (json.load(open(Path(dir_path, "results.json")))["configs"],) + config = ( + json.load(open(Path(dir_path, "results.json"), encoding="utf-8"))["configs"], + ) return list(config[0].keys())