diff --git a/demo/tutorials/llm_notebooks/Disinformation_Test.ipynb b/demo/tutorials/llm_notebooks/Disinformation_Test.ipynb index 641f3c8c9..5bfa7a2b9 100644 --- a/demo/tutorials/llm_notebooks/Disinformation_Test.ipynb +++ b/demo/tutorials/llm_notebooks/Disinformation_Test.ipynb @@ -175,9 +175,9 @@ "source": [ "model = {\"model\": \"text-davinci-003\", \"hub\":\"openai\"}\n", "\n", - "data={\"data_source\" :'Toxicity',\"split\":\"test\"}\n", + "data = {\"data_source\": \"Narrative-Wedging\"}\n", "\n", - "harness = Harness(task={\"task\": \"text-generation\", \"category\": \"toxicity\"}, model=model, data=data)" + "harness = Harness(task={\"task\": \"text-generation\", \"category\": \"disinformation-test\"}, model=model, data=data)" ] }, { diff --git a/langtest/data/config/wino_huggingface_config.yml b/langtest/data/config/wino_huggingface_config.yml index 925b4415b..b6d1b2c1d 100644 --- a/langtest/data/config/wino_huggingface_config.yml +++ b/langtest/data/config/wino_huggingface_config.yml @@ -2,7 +2,7 @@ tests: defaults: min_pass_rate: 1.0 - wino-bias: - gender-occupational-stereotype: + stereotype: + wino-bias: min_pass_rate: 0.70 diff_threshold: 0.03 \ No newline at end of file diff --git a/langtest/data/config/wino_openai_config.yml b/langtest/data/config/wino_openai_config.yml index 80f32721d..de7f4cab6 100644 --- a/langtest/data/config/wino_openai_config.yml +++ b/langtest/data/config/wino_openai_config.yml @@ -5,7 +5,7 @@ model_parameters: tests: defaults: min_pass_rate: 1.0 - - wino-bias: - gender-occupational-stereotype: + + stereotype: + wino-bias: min_pass_rate: 0.70 \ No newline at end of file diff --git a/langtest/transform/__init__.py b/langtest/transform/__init__.py index 0d2d075b7..95069469e 100644 --- a/langtest/transform/__init__.py +++ b/langtest/transform/__init__.py @@ -1493,6 +1493,7 @@ class DisinformationTestFactory(ITests): alias_name = "disinformation" supported_tasks = [ "disinformation-test", + "text-generation", ] def __init__(self, data_handler: List[Sample], tests: Dict = None, **kwargs) -> None: