From 41cb9097414d853dd9c248a39f949b21fcca4094 Mon Sep 17 00:00:00 2001 From: Kiarash Jamali Date: Fri, 13 Jan 2023 10:50:07 +0000 Subject: [PATCH] Change default dropout value in documentation Documentation says default is 0.1, but the code has attention_dropout default at 0.0 --- flash_attn/flash_attention.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flash_attn/flash_attention.py b/flash_attn/flash_attention.py index 138a90be1..db6804e5d 100644 --- a/flash_attn/flash_attention.py +++ b/flash_attn/flash_attention.py @@ -16,7 +16,7 @@ class FlashAttention(nn.Module): (default: 1/sqrt(d_keys) where d_keys is computed at runtime) attention_dropout: The dropout rate to apply to the attention - (default: 0.1) + (default: 0.0) """ def __init__(self, softmax_scale=None, attention_dropout=0.0, device=None, dtype=None): super().__init__()