Deploying to gh-pages from @ 8f4d134 🚀

OpenNMT · Sep 9, 2024 · be1828f · be1828f
1 parent 9976be1
commit be1828f
Show file tree

Hide file tree

Showing 100 changed files with 268 additions and 167 deletions.
diff --git a/.buildinfo b/.buildinfo
@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 99553ed5bb7519ec6ff4af06b2dd71b5
+config: 2a4bd9441600d6f87ec45cdb9c8c85e0
 tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle
diff --git a/.doctrees/python/ctranslate2.GenerationResult.doctree b/.doctrees/python/ctranslate2.GenerationResult.doctree
diff --git a/.doctrees/python/ctranslate2.GenerationStepResult.doctree b/.doctrees/python/ctranslate2.GenerationStepResult.doctree
diff --git a/.doctrees/python/ctranslate2.Generator.doctree b/.doctrees/python/ctranslate2.Generator.doctree
diff --git a/.doctrees/python/ctranslate2.TranslationResult.doctree b/.doctrees/python/ctranslate2.TranslationResult.doctree
diff --git a/.doctrees/python/ctranslate2.Translator.doctree b/.doctrees/python/ctranslate2.Translator.doctree
diff --git a/.doctrees/python/ctranslate2.models.Whisper.doctree b/.doctrees/python/ctranslate2.models.Whisper.doctree
diff --git a/.doctrees/python/ctranslate2.specs.RotaryScalingType.doctree b/.doctrees/python/ctranslate2.specs.RotaryScalingType.doctree
diff --git a/.doctrees/python/ctranslate2.specs.TransformerDecoderModelSpec.doctree b/.doctrees/python/ctranslate2.specs.TransformerDecoderModelSpec.doctree
diff --git a/.doctrees/python/ctranslate2.specs.TransformerDecoderSpec.doctree b/.doctrees/python/ctranslate2.specs.TransformerDecoderSpec.doctree
diff --git a/.doctrees/python/ctranslate2.specs.Wav2Vec2Spec.doctree b/.doctrees/python/ctranslate2.specs.Wav2Vec2Spec.doctree
diff --git a/.doctrees/quantization.doctree b/.doctrees/quantization.doctree
diff --git a/_sources/python/ctranslate2.GenerationResult.rst.txt b/_sources/python/ctranslate2.GenerationResult.rst.txt
@@ -10,6 +10,7 @@ GenerationResult
 
     **Attributes:**
 
+    - :obj:`~ctranslate2.GenerationResult.logits`
     - :obj:`~ctranslate2.GenerationResult.scores`
     - :obj:`~ctranslate2.GenerationResult.sequences`
     - :obj:`~ctranslate2.GenerationResult.sequences_ids`
diff --git a/_sources/python/ctranslate2.GenerationStepResult.rst.txt b/_sources/python/ctranslate2.GenerationStepResult.rst.txt
@@ -14,6 +14,7 @@ GenerationStepResult
     - :obj:`~ctranslate2.GenerationStepResult.hypothesis_id`
     - :obj:`~ctranslate2.GenerationStepResult.is_last`
     - :obj:`~ctranslate2.GenerationStepResult.log_prob`
+    - :obj:`~ctranslate2.GenerationStepResult.logits`
     - :obj:`~ctranslate2.GenerationStepResult.step`
     - :obj:`~ctranslate2.GenerationStepResult.token`
     - :obj:`~ctranslate2.GenerationStepResult.token_id`
diff --git a/_sources/python/ctranslate2.TranslationResult.rst.txt b/_sources/python/ctranslate2.TranslationResult.rst.txt
@@ -12,4 +12,5 @@ TranslationResult
 
     - :obj:`~ctranslate2.TranslationResult.attention`
     - :obj:`~ctranslate2.TranslationResult.hypotheses`
+    - :obj:`~ctranslate2.TranslationResult.logits`
     - :obj:`~ctranslate2.TranslationResult.scores`
diff --git a/_sources/python/ctranslate2.specs.TransformerDecoderSpec.rst.txt b/_sources/python/ctranslate2.specs.TransformerDecoderSpec.rst.txt
@@ -8,6 +8,10 @@ TransformerDecoderSpec
 
     **Inherits from:** :class:`ctranslate2.specs.LayerSpec`
 
+    **Attributes:**
+
+    - :obj:`~ctranslate2.specs.TransformerDecoderSpec.config`
+
     **Methods:**
 
     - :obj:`~ctranslate2.specs.TransformerDecoderSpec.optimize`

diff --git a/_sources/quantization.md.txt b/_sources/quantization.md.txt
@@ -6,6 +6,7 @@ Quantization is a technique that can reduce the model size and accelerate its ex
 * 16-bit integers (INT16)
 * 16-bit floating points (FP16)
 * 16-bit brain floating points (BF16)
+* 4-bit AWQ Quantization
 
 ```{tip}
 See the benchmark results in the main [README](https://github.com/OpenNMT/CTranslate2#benchmarks) to compare the performance and memory usage with and without quantization.
@@ -161,3 +162,21 @@ In this mode, all model weights are stored in half precision and all layers are
 * NVIDIA GPU with Compute Capability >= 8.0
 
 In this mode, all model weights are stored in BF16 and all layers are run with this type.
+
+### 4-bit AWQ
+
+The compute type would be `int32_float16`
+
+**Supported on:**
+
+* NVIDIA GPU with Compute Capability >= 7.5
+
+In this mode, all model weights are stored in half precision and all layers are run in half precision. Other parameters like scale and zero are stored in ``int32``.
+
+For example,
+
+```bash
+ ct2-transformers-converter --model TheBloke/Llama-2-7B-AWQ --copy_files tokenizer.model --output_dir ct2_model
+```
+
+We have to quantize the model with AWQ first, then convert it to CT2 format.
diff --git a/_static/documentation_options.js b/_static/documentation_options.js
@@ -1,6 +1,6 @@
 var DOCUMENTATION_OPTIONS = {
     URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
-    VERSION: '4.3.1',
+    VERSION: '4.4.0',
     LANGUAGE: 'en',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',

diff --git a/conversion.html b/conversion.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Model conversion &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Model conversion &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">

diff --git a/decoding.html b/decoding.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Decoding features &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Decoding features &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">

diff --git a/encoding.html b/encoding.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Text encoding &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Text encoding &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">

diff --git a/environment_variables.html b/environment_variables.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Environment variables &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Environment variables &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">

diff --git a/faq.html b/faq.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Frequently asked questions &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Frequently asked questions &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">

diff --git a/generation.html b/generation.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Text generation &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Text generation &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">

diff --git a/genindex.html b/genindex.html
@@ -3,7 +3,7 @@
 <head>
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Index &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Index &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -29,7 +29,7 @@
             <a href="index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
@@ -248,6 +248,8 @@ <h2 id="C">C</h2>
         <li><a href="python/ctranslate2.specs.SequenceToSequenceModelSpec.html#ctranslate2.specs.SequenceToSequenceModelSpec.config">(ctranslate2.specs.SequenceToSequenceModelSpec property)</a>
 </li>
         <li><a href="python/ctranslate2.specs.TransformerDecoderModelSpec.html#ctranslate2.specs.TransformerDecoderModelSpec.config">(ctranslate2.specs.TransformerDecoderModelSpec property)</a>
+</li>
+        <li><a href="python/ctranslate2.specs.TransformerDecoderSpec.html#ctranslate2.specs.TransformerDecoderSpec.config">(ctranslate2.specs.TransformerDecoderSpec property)</a>
 </li>
         <li><a href="python/ctranslate2.specs.TransformerEncoderModelSpec.html#ctranslate2.specs.TransformerEncoderModelSpec.config">(ctranslate2.specs.TransformerEncoderModelSpec property)</a>
 </li>
@@ -587,6 +589,8 @@ <h2 id="L">L</h2>
       <li><a href="python/ctranslate2.specs.LayerSpec.html#ctranslate2.specs.LayerSpec">LayerSpec (class in ctranslate2.specs)</a>
 </li>
       <li><a href="python/ctranslate2.specs.RotaryScalingType.html#ctranslate2.specs.RotaryScalingType.Linear">Linear (ctranslate2.specs.RotaryScalingType attribute)</a>
+</li>
+      <li><a href="python/ctranslate2.specs.RotaryScalingType.html#ctranslate2.specs.RotaryScalingType.Llama3">Llama3 (ctranslate2.specs.RotaryScalingType attribute)</a>
 </li>
       <li><a href="python/ctranslate2.converters.TransformersConverter.html#ctranslate2.converters.TransformersConverter.load_model">load_model() (ctranslate2.converters.TransformersConverter method)</a>
 
@@ -610,6 +614,14 @@ <h2 id="L">L</h2>
 </li>
       <li><a href="python/ctranslate2.ScoringResult.html#ctranslate2.ScoringResult.log_probs">log_probs (ctranslate2.ScoringResult property)</a>
 </li>
+      <li><a href="python/ctranslate2.GenerationResult.html#ctranslate2.GenerationResult.logits">logits (ctranslate2.GenerationResult property)</a>
+
+      <ul>
+        <li><a href="python/ctranslate2.GenerationStepResult.html#ctranslate2.GenerationStepResult.logits">(ctranslate2.GenerationStepResult property)</a>
+</li>
+        <li><a href="python/ctranslate2.TranslationResult.html#ctranslate2.TranslationResult.logits">(ctranslate2.TranslationResult property)</a>
+</li>
+      </ul></li>
   </ul></td>
 </tr></table>
 

diff --git a/guides/fairseq.html b/guides/fairseq.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Fairseq &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Fairseq &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="../_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="../index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">

diff --git a/guides/marian.html b/guides/marian.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Marian &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Marian &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="../_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="../index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">

diff --git a/guides/opennmt_py.html b/guides/opennmt_py.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>OpenNMT-py &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>OpenNMT-py &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="../_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="../index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">

diff --git a/guides/opennmt_tf.html b/guides/opennmt_tf.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>OpenNMT-tf &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>OpenNMT-tf &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="../_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="../index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">

diff --git a/guides/opus_mt.html b/guides/opus_mt.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>OPUS-MT &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>OPUS-MT &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="../_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="../index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">

diff --git a/guides/transformers.html b/guides/transformers.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Transformers &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Transformers &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="../_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="../index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">

diff --git a/hardware_support.html b/hardware_support.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Hardware support &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Hardware support &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">

diff --git a/index.html b/index.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Index &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Index &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -31,7 +31,7 @@
             <a href="#" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">

diff --git a/installation.html b/installation.html
@@ -4,7 +4,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>Installation &mdash; CTranslate2 4.3.1 documentation</title>
+  <title>Installation &mdash; CTranslate2 4.4.0 documentation</title>
       <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="_static/custom.css" type="text/css" />
@@ -32,7 +32,7 @@
             <a href="index.html" class="icon icon-home"> CTranslate2
           </a>
               <div class="version">
-                4.3
+                4.4
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="search.html" method="get">