Add plugin for Automatic1111 Stable-diffusion-webui

README.md

@@ -11,11 +11,14 @@ Our AI plugins offer the following key features:
 
 ## List of Plugins
 
-### Stable Diffusion plugin for GIMP
+### <img src="plugins\gimp\stable-diffusion\docs\resources\gimp.png" alt="drawing" style="width:30px;"/> Stable Diffusion plugin for GIMP
 - [Overview and usage](plugins/gimp/stable-diffusion/README.md)
 - [Quick Installation](plugins/gimp/stable-diffusion/docs/install.md)
 - [Build Instructions](plugins/gimp/stable-diffusion/docs/build.md)
 
-## License Information
+### <img src="plugins\stable-diffusion-webui\qairt_accelerate\docs\resources\main_icon.png" alt="drawing" style="width:30px;border-radius: 50%;"/> AUTOMATIC1111 stable-diffusion-webui Extension
+- [Overview and Installation](plugins/stable-diffusion-webui/qairt_accelerate/README.md)
+
+##  License Information
 
 This project is licensed under the [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). For the full license text, please refer to the [LICENSE](LICENSE) file in this repository.

plugins/stable-diffusion-webui/qairt_accelerate/README.md

@@ -0,0 +1,42 @@
+
+# <img src="docs\resources\main_icon.png" alt="drawing" style="width:40px;border-radius: 50%;"/> AUTOMATIC1111 stable-diffusion-webui Extension
+
+Stable Diffusion WebUI can now be run on Qualcomm X-Elite NPU with [Qualcomm AI Runtime (QAIRT)](https://www.qualcomm.com/developer/software/qualcomm-ai-engine-direct-sdk). QAIRT support is provided through an custom extension script. The custom script uses QAIRT python APIs to run context binaries (.bin) generated by QAIRT SDK. We provide optimal performance by using QAIRT to run AI models on Qualcomm X-Elite NPU. These models are hosted on [Qualcomm AI hub](https://aihub.qualcomm.com/compute/models/stable_diffusion_v1_5_quantized?searchTerm=stable).
+
+![sd1_5](docs/resources/qairt.gif)
+<sub><sup>This clip is at 2x speed</sup></sup>
+
+
+> **_NOTE:_**  Majority of the AUTOMATIC1111's features are not supported by this extension as of now and the extension is in active development. New feature support will be added incrementally. We actively welcome feedback and contributions from the community.
+
+## Supported features
+
+* Original txt2img mode with SD 1.5 and 2.1
+* Sampling methods: DPM++ 2M
+* Upscaling methods: ESRGAN-x4
+
+## Instructions to run WebUI with QAIRT (Windows):
+
+### Step 1: Download AUTOMATIC1111 stable-diffusion-webui
+Run below commands in Windows PowerShell terminal.
+
+```
+# Make sure Python version is >=3.10.6 and <3.10.14
+git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui.git
+cd stable-diffusion-webui
+```
+
+### Step 2: Download Extension
+Download `stable-diffusion-webui-qairt-extension.zip` from the latest release.
+unzip and place the `qairt_accelerate` extension under `stable-diffusion-webui\extensions`
+
+### Step 3: Launch the WebUI
+
+```
+$env:TORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"
+$env:WEBUI_LAUNCH_LIVE_OUTPUT=1
+
+.\webui.bat --skip-torch-cuda-test --no-half --precision full --ui-config-file .\extensions\qairt_accelerate\ui-config.json
+```
+
+The steps above will create a virtual environment and install the required packages into this environment.

plugins/stable-diffusion-webui/qairt_accelerate/common_utils.py

@@ -0,0 +1,59 @@
+# =============================================================================
+#
+# Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# =============================================================================
+
+import requests
+import os
+import subprocess
+
+def download_url(url, save_path, chunk_size=128):
+    r = requests.get(url, stream=True)
+    with open(save_path, "wb") as fd:
+        for chunk in r.iter_content(chunk_size=chunk_size):
+            fd.write(chunk)
+
+def run_command(command, live: bool = True):
+    try:
+        env = os.environ.copy()
+        env['PYTHONPATH'] = f"{os.path.abspath('.')}{os.pathsep}{env.get('PYTHONPATH', '')}"
+
+        stdout = run(command, errdesc=f"Error running command", live=live).strip()
+        if stdout:
+            print(stdout)
+    except Exception as e:
+        print(str(e))
+        exit()
+
+def run(command, desc=None, errdesc=None, custom_env=None, live: bool = True) -> str:
+    if desc is not None:
+        print(desc)
+
+    run_kwargs = {
+        "args": command,
+        "shell": True,
+        "env": os.environ if custom_env is None else custom_env,
+        "errors": 'ignore',
+    }
+
+    if not live:
+        run_kwargs["stdout"] = run_kwargs["stderr"] = subprocess.PIPE
+
+    result = subprocess.run(**run_kwargs)
+
+    if result.returncode != 0:
+        error_bits = [
+            f"{errdesc or 'Error running command'}.",
+            f"Command: {command}",
+            f"Error code: {result.returncode}",
+        ]
+        if result.stdout:
+            error_bits.append(f"stdout: {result.stdout}")
+        if result.stderr:
+            error_bits.append(f"stderr: {result.stderr}")
+        raise RuntimeError("\n".join(error_bits))
+
+    return (result.stdout or "")

plugins/stable-diffusion-webui/qairt_accelerate/install.py

@@ -0,0 +1,137 @@
+# =============================================================================
+#
+# Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# =============================================================================
+
+import launch
+import os
+import qairt_constants as consts
+import platform
+
+
+# check python version to be 3.10.6+. As qai_appbuilder is built on 3.10.6
+if (not platform.python_version().startswith("3.10.")) or (int(platform.python_version().split(".")[2])<6):
+    raise Exception("Python version needs to be >=3.10.6 and <3.10.14")
+
+if not launch.is_installed("qai_appbuilder"):
+    launch.run_pip(
+        f"install {consts.QAI_APPBUILDER_WHEEL}",
+        "python QNN",
+    )
+if not launch.is_installed("diffusers"):
+    launch.run_pip("install diffusers", "diffusers")
+if not launch.is_installed("onnx"):
+    launch.run_pip("install onnx", "onnx")
+
+from qairt_sd_pipeline import model_path_1_5, model_path_2_1
+from huggingface_hub import hf_hub_download
+import common_utils as utils
+import zipfile
+import shutil
+
+def download_qairt_sdk():
+    # Setup QAIRT SDK
+    if not os.path.isdir(consts.QNN_SDK_ROOT):
+        os.makedirs(consts.QAIRT_DIR, exist_ok=True)
+        print(f"Downloading QAIRT SDK...")
+        utils.download_url(consts.QNN_SDK_DOWNLOAD_URL, consts.SDK_SAVE_PATH)
+        print(f"QAIRT SDK downloaded.")
+
+        with zipfile.ZipFile(consts.SDK_SAVE_PATH, "r") as zip_ref:
+            zip_ref.extractall(consts.EXTENSION_WS)
+        shutil.move(
+            os.path.join(consts.EXTENSION_WS, "qairt", consts.QAIRT_VERSION),
+            os.path.join(consts.QNN_SDK_ROOT, ".."),
+        )
+        shutil.rmtree(os.path.join(consts.EXTENSION_WS, "qairt"))
+        os.remove(consts.SDK_SAVE_PATH)
+
+def setup_qairt_env():
+    # Preparing all the binaries and libraries for execution.
+    SDK_lib_dir = consts.QNN_SDK_ROOT + "\\lib\\arm64x-windows-msvc"
+    SDK_skel = consts.QNN_SDK_ROOT + "\\lib\\hexagon-v{}\\unsigned\\libQnnHtpV{}Skel.so".format(
+        consts.DSP_ARCH, consts.DSP_ARCH
+    )
+
+    # Copy necessary libraries to a common location
+    libs = [
+        "QnnHtp.dll",
+        "QnnSystem.dll",
+        "QnnHtpPrepare.dll",
+        "QnnHtpV{}Stub.dll".format(consts.DSP_ARCH),
+    ]
+    for lib in libs:
+        if not os.path.isfile(os.path.join(consts.QNN_LIBS_DIR, lib)):
+            shutil.copy(os.path.join(SDK_lib_dir, lib), consts.QNN_LIBS_DIR)
+
+    if not os.path.isfile(os.path.join(consts.QNN_LIBS_DIR, SDK_skel)):
+        shutil.copy(SDK_skel, consts.QNN_LIBS_DIR)
+
+
+def create_venv_for_qai_hub():
+    if not os.path.isdir(consts.QAI_HUB_VENV_PATH):
+        utils.run_command(f"python -m venv {consts.QAI_HUB_VENV_PATH}")
+
+def install_qai_hub():
+    utils.run_command(f"{consts.QAI_HUB_VENV_PYTHON_PATH} -m pip install qai-hub")
+    utils.run_command(f"{consts.QAI_HUB_VENV_PYTHON_PATH} -m pip install qai_hub_models")
+    utils.run_command(f"{consts.QAI_HUB_VENV_PATH}\\Scripts\\qai-hub.exe configure --api_token {consts.HUB_ID} > NUL", False)
+
+
+print(f"Downloading QAIRT model bin files...")
+SD_MODEL_1_5_REVISION="120de88f304daa9d5fa726ddccdfe086b6349801"
+SD_MODEL_2_1_REVISION="52f821ad5420d1b0408a8b856733f9e372e7776a"
+
+hf_hub_download(
+    repo_id="qualcomm/Stable-Diffusion-v1.5",
+    filename="UNet_Quantized.bin",
+    local_dir=model_path_1_5,
+    revision=SD_MODEL_1_5_REVISION,
+)
+hf_hub_download(
+    repo_id="qualcomm/Stable-Diffusion-v1.5",
+    filename="TextEncoder_Quantized.bin",
+    local_dir=model_path_1_5,
+    revision=SD_MODEL_1_5_REVISION,
+)
+hf_hub_download(
+    repo_id="qualcomm/Stable-Diffusion-v1.5",
+    filename="VAEDecoder_Quantized.bin",
+    local_dir=model_path_1_5,
+    revision=SD_MODEL_1_5_REVISION,
+)
+
+hf_hub_download(
+    repo_id="qualcomm/Stable-Diffusion-v2.1",
+    filename="UNet_Quantized.bin",
+    local_dir=model_path_2_1,
+    revision=SD_MODEL_2_1_REVISION,
+)
+hf_hub_download(
+    repo_id="qualcomm/Stable-Diffusion-v2.1",
+    filename="TextEncoder_Quantized.bin",
+    local_dir=model_path_2_1,
+    revision=SD_MODEL_2_1_REVISION,
+)
+hf_hub_download(
+    repo_id="qualcomm/Stable-Diffusion-v2.1",
+    filename="VAEDecoder_Quantized.bin",
+    local_dir=model_path_2_1,
+    revision=SD_MODEL_2_1_REVISION,
+)
+print(f"QAIRT model bin files downloaded.")
+
+os.makedirs(consts.CACHE_DIR, exist_ok=True)
+os.makedirs(consts.QNN_LIBS_DIR, exist_ok=True)
+
+download_qairt_sdk()
+setup_qairt_env()
+create_venv_for_qai_hub()
+install_qai_hub()
+
+print("Downloading required models using qai-hub...")
+utils.run_command(f"{consts.QAI_HUB_VENV_PYTHON_PATH} {consts.EXTENSION_WS}//qairt_hub_models.py")
+print("Downloaded required models.")

plugins/stable-diffusion-webui/qairt_accelerate/qairt_constants.py

@@ -0,0 +1,32 @@
+# =============================================================================
+#
+# Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# =============================================================================
+
+
+import os
+from modules.paths_internal import script_path, extensions_dir
+
+QNN_SDK_DOWNLOAD_URL="https://softwarecenter.qualcomm.com/api/download/software/qualcomm_neural_processing_sdk/v2.24.0.240710.zip"
+QAI_APPBUILDER_WHEEL="https://github.com/quic/ai-engine-direct-helper/releases/download/v2.24.0/qai_appbuilder-2.24.0-cp310-cp310-win_amd64.whl"
+
+QAIRT_VERSION = "2.24.0.240626"
+DSP_ARCH = "73"  # For X-Elite device.
+EXTENSION_WS = os.path.join(extensions_dir, "qairt_accelerate")
+VENV_PYTHON_PATH = f"{EXTENSION_WS}\\..\\..\\venv\\Scripts\\python.exe"
+QAI_HUB_VENV_PATH = f"{EXTENSION_WS}\\qai_hub_venv"
+QAI_HUB_VENV_PYTHON_PATH = f"{EXTENSION_WS}\\qai_hub_venv\\Scripts\\python.exe"
+QNN_LIBS_DIR = os.path.join(EXTENSION_WS, "qnn_assets", "qnn_libs")
+CACHE_DIR = os.path.join(EXTENSION_WS, "qnn_assets", "cache")
+SDK_SAVE_PATH= EXTENSION_WS + f"\\{QAIRT_VERSION}.zip"
+QAIRT_DIR=f"C:\\Qualcomm\\AIStack\\QAIRT"
+QNN_SDK_ROOT=f"C:\\Qualcomm\\AIStack\\QAIRT\\{QAIRT_VERSION}"
+
+
+HUB_ID="aac24f12d047e7f558d8effe4b2fdad0f5c2c341"
+CONVERTION_DIR = os.path.join(EXTENSION_WS, "model_conversion")
+ESRGAN_X4_MODEL_ID="meq29lx7n"
+ESRGAN_X4_MODEL_PATH=os.path.join(CONVERTION_DIR,"esrgan.bin")

plugins/stable-diffusion-webui/qairt_accelerate/qairt_hub_models.py

@@ -0,0 +1,30 @@
+# =============================================================================
+#
+# Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# =============================================================================
+
+import common_utils as utils
+import os
+import qairt_constants as consts
+import shutil
+import qai_hub
+
+def convert_and_download_models():
+    os.makedirs(consts.CONVERTION_DIR, exist_ok=True)
+    if not os.path.isfile(consts.ESRGAN_X4_MODEL_PATH):
+        try:
+            model = qai_hub.get_model(consts.ESRGAN_X4_MODEL_ID)
+            model.download(filename=consts.ESRGAN_X4_MODEL_PATH)
+        except Exception:
+            utils.run_command(f"{consts.QAI_HUB_VENV_PYTHON_PATH} -m qai_hub_models.models.esrgan.export " +
+                        f"--device \"Snapdragon X Elite CRD\" --height 512 --width 512 --target-runtime qnn " +
+                        f"--skip-profiling --skip-inferencing --skip-summary --output-dir {consts.CONVERTION_DIR}")
+            shutil.move(
+                os.path.join(consts.CONVERTION_DIR,"esrgan.so"),
+                consts.ESRGAN_X4_MODEL_PATH,
+            )
+
+convert_and_download_models()