From 1ba7d1b620ea7586a2707f7af955c8c83c495035 Mon Sep 17 00:00:00 2001 From: Arshaan Date: Thu, 9 Nov 2023 21:20:17 +0530 Subject: [PATCH 1/3] fix default config --- langtest/data/config/wino_huggingface_config.yml | 4 ++-- langtest/data/config/wino_openai_config.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/langtest/data/config/wino_huggingface_config.yml b/langtest/data/config/wino_huggingface_config.yml index 925b4415b..b6d1b2c1d 100644 --- a/langtest/data/config/wino_huggingface_config.yml +++ b/langtest/data/config/wino_huggingface_config.yml @@ -2,7 +2,7 @@ tests: defaults: min_pass_rate: 1.0 - wino-bias: - gender-occupational-stereotype: + stereotype: + wino-bias: min_pass_rate: 0.70 diff_threshold: 0.03 \ No newline at end of file diff --git a/langtest/data/config/wino_openai_config.yml b/langtest/data/config/wino_openai_config.yml index 80f32721d..de7f4cab6 100644 --- a/langtest/data/config/wino_openai_config.yml +++ b/langtest/data/config/wino_openai_config.yml @@ -5,7 +5,7 @@ model_parameters: tests: defaults: min_pass_rate: 1.0 - - wino-bias: - gender-occupational-stereotype: + + stereotype: + wino-bias: min_pass_rate: 0.70 \ No newline at end of file From 108408240a2c43656093cf54c2e5e7a28348a8e3 Mon Sep 17 00:00:00 2001 From: Arshaan Date: Thu, 9 Nov 2023 21:37:49 +0530 Subject: [PATCH 2/3] update DisinformationTestFactory and NB --- demo/tutorials/llm_notebooks/Disinformation_Test.ipynb | 4 ++-- langtest/transform/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/demo/tutorials/llm_notebooks/Disinformation_Test.ipynb b/demo/tutorials/llm_notebooks/Disinformation_Test.ipynb index 8a8eea97e..ac1951ab1 100644 --- a/demo/tutorials/llm_notebooks/Disinformation_Test.ipynb +++ b/demo/tutorials/llm_notebooks/Disinformation_Test.ipynb @@ -175,9 +175,9 @@ "source": [ "model = {\"model\": \"j2-jumbo-instruct\", \"hub\":\"ai21\"}\n", "\n", - "data = {\"data_source\": \"Prompt-Injection-Attack\", \"split\":\"test-tiny\"}\n", + "data = {\"data_source\": \"Narrative-Wedging\"}\n", "\n", - "harness = Harness(task={\"task\": \"text-generation\", \"category\": \"disinformation\"}, model=model, data=data)" + "harness = Harness(task={\"task\": \"text-generation\", \"category\": \"disinformation-test\"}, model=model, data=data)" ] }, { diff --git a/langtest/transform/__init__.py b/langtest/transform/__init__.py index 0d2d075b7..aefa82b84 100644 --- a/langtest/transform/__init__.py +++ b/langtest/transform/__init__.py @@ -1492,7 +1492,7 @@ class DisinformationTestFactory(ITests): alias_name = "disinformation" supported_tasks = [ - "disinformation-test", + "disinformation-test", "text-generation", ] def __init__(self, data_handler: List[Sample], tests: Dict = None, **kwargs) -> None: From 6881d6ef531ebeee34237fbebcc4a02eb7bcc9d5 Mon Sep 17 00:00:00 2001 From: Arshaan Date: Thu, 9 Nov 2023 21:45:53 +0530 Subject: [PATCH 3/3] fix linting --- langtest/transform/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/langtest/transform/__init__.py b/langtest/transform/__init__.py index aefa82b84..95069469e 100644 --- a/langtest/transform/__init__.py +++ b/langtest/transform/__init__.py @@ -1492,7 +1492,8 @@ class DisinformationTestFactory(ITests): alias_name = "disinformation" supported_tasks = [ - "disinformation-test", "text-generation", + "disinformation-test", + "text-generation", ] def __init__(self, data_handler: List[Sample], tests: Dict = None, **kwargs) -> None: