Gradio Local Editor cookbook

cookbooks/Gradio/README.md

@@ -0,0 +1,56 @@
+# Gradio Workbook Editor
+
+## Prompt IDE with LastMile AI
+
+This cookbook is meant to demonstrate the capabilities of the AI Workbook editor. It can run inference against locally hosted or remote models from many inference providers, including Hugging Face, OpenAI and others.
+
+It supports text, image and audio model formats, allowing you to easily chain them together in a single notebook session!
+
+With `aiconfig`, it lets you save the state in a single json config file which you can version control or share with others.
+
+In addition to editing the `*.aiconfig.json` file through the AI Workbook interface, you can also use the AIConfig SDK to use it in application code, providing a single interface to run inference across any model and modality (media formats).
+
+## Tech Stack
+
+What you see here is a "local editor" -- a React frontend and a Flask server which allow you to edit `.aiconfig.json` files in a notebook-like UI.
+
+- [React Frontend code](https://github.com/lastmile-ai/aiconfig/tree/main/python/src/aiconfig/editor/client)
+
+- [Backend server code](https://github.com/lastmile-ai/aiconfig/tree/main/python/src/aiconfig/editor/server)
+
+### Gradio custom component
+
+The Gradio custom component is currently WIP.
+
+**Note**: We already have the Gradio backend that corresponds to the Flask server in the [`gradio-workbook`](https://github.com/lastmile-ai/gradio-workbook) repo.
+
+We are working on using `sveltris` to package our React frontend to work with Gradio. Once that works, the same experience you see in this cookbook will be possible inside a Gradio custom component.
+
+## Getting Started
+
+**Instructions**:
+
+- Clone https://github.com/lastmile-ai/aiconfig
+- Go back to top-level directory: `cd <aiconfig>`
+
+- `cd <aiconfig>/cookbooks/Gradio`
+
+- `pip3 install -r requirements.txt`
+
+- Install `python-aiconfig-test` package from `test-pypi`:
+
+```
+pip3 install --index-url https://test.pypi.org/simple --extra-index-url https://pypi.org/simple python-aiconfig-test==1.1.27 --force
+```
+
+Now run this command to start the AIConfig editor:
+
+```bash
+aiconfig edit --aiconfig-path=huggingface.aiconfig.json --parsers-module-path=hf_model_parsers.py
+```
+
+## TODO
+
+- Publish new version of aiconfig_extension_hugging_face package
+- Update huggingface.aiconfig.json with clean examples
+- Add video demo

cookbooks/Gradio/hf_model_parsers.py

@@ -0,0 +1,33 @@
+from aiconfig_extension_hugging_face import (
+    HuggingFaceAutomaticSpeechRecognitionTransformer,
+    HuggingFaceImage2TextTransformer,
+    HuggingFaceTextSummarizationTransformer,
+    HuggingFaceText2ImageDiffusor,
+    HuggingFaceText2SpeechTransformer,
+    HuggingFaceTextGenerationTransformer,
+    HuggingFaceTextTranslationTransformer,
+)
+
+from aiconfig import AIConfigRuntime
+
+
+def register_model_parsers() -> None:
+    """Register model parsers for HuggingFace models."""
+    automatic_speech_recognition = HuggingFaceAutomaticSpeechRecognitionTransformer()
+    AIConfigRuntime.register_model_parser(automatic_speech_recognition, automatic_speech_recognition.id())
+
+    image_to_text = HuggingFaceImage2TextTransformer()
+    AIConfigRuntime.register_model_parser(image_to_text, image_to_text.id())
+
+    text_to_image = HuggingFaceText2ImageDiffusor()
+    AIConfigRuntime.register_model_parser(text_to_image, text_to_image.id())
+
+    text_to_speech = HuggingFaceText2SpeechTransformer()
+    AIConfigRuntime.register_model_parser(text_to_speech, text_to_speech.id())
+
+    text_generation = HuggingFaceTextGenerationTransformer()
+    AIConfigRuntime.register_model_parser(text_generation, text_generation.id())
+    text_summarization = HuggingFaceTextSummarizationTransformer()
+    AIConfigRuntime.register_model_parser(text_summarization, text_summarization.id())
+    text_translation = HuggingFaceTextTranslationTransformer()
+    AIConfigRuntime.register_model_parser(text_translation, text_translation.id())

cookbooks/Gradio/huggingface.aiconfig.json

@@ -0,0 +1,179 @@
+{
+  "name": "The Tale of the Quick Brown Fox",
+  "schema_version": "latest",
+  "metadata": {
+    "parameters": {},
+    "models": {
+      "stevhliu/my_awesome_billsum_model": {
+        "model": "stevhliu/my_awesome_billsum_model",
+        "min_length": 10,
+        "max_length": 30
+      },
+      "Salesforce/blip-image-captioning-base": {
+        "model": "Salesforce/blip-image-captioning-base"
+      }
+    },
+    "default_model": "stevhliu/my_awesome_billsum_model",
+    "model_parsers": {
+      "Salesforce/blip-image-captioning-base": "HuggingFaceImage2TextTransformer",
+      "suno/bark": "HuggingFaceText2SpeechTransformer"
+    }
+  },
+  "description": "Welcome to the first Hugging Face AIConfig! Let's showcase the many capabilities of Hugging Face models, all in one place.",
+  "prompts": [
+    {
+      "name": "image_caption",
+      "input": {
+        "attachments": [
+          {
+            "data": "https://s3.amazonaws.com/lastmileai.aiconfig.public/uploads/2024_1_10_18_41_31/1742/bear_with_honey.png",
+            "mime_type": "image/png"
+          },
+          {
+            "data": "https://s3.amazonaws.com/lastmileai.aiconfig.public/uploads/2024_1_10_18_41_31/7275/fox_in_forest.png",
+            "mime_type": "image/png"
+          }
+        ]
+      },
+      "metadata": {
+        "model": {
+          "name": "HuggingFaceImage2TextTransformer",
+          "settings": {
+            "max_new_tokens": 10,
+            "model": "Salesforce/blip-image-captioning-base"
+          }
+        },
+        "parameters": {}
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "execution_count": 0,
+          "data": "a bear sitting on a rock eating honey",
+          "metadata": {}
+        },
+        {
+          "output_type": "execute_result",
+          "execution_count": 1,
+          "data": "a red fox in the woods",
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "name": "translation",
+      "input": "Once upon a time, in a lush and vibrant forest, there lived a magnificent creature known as the Quick Brown Fox. This fox was unlike any other, possessing incredible speed and agility that awed all the animals in the forest. With its fur as golden as the sun and its eyes as sharp as emeralds, the Quick Brown Fox was admired by everyone, from the tiniest hummingbird to the mightiest bear. The fox had a kind heart and would often lend a helping paw to those in need. The Quick Brown Fox had a particular fondness for games and challenges. It loved to test its skills against others, always seeking new adventures to satisfy its boundless curiosity. Its favorite game was called \"The Great Word Hunt,\" where it would embark on a quest to find hidden words scattered across the forest. \n\nOne day it got very old and died",
+      "metadata": {
+        "model": {
+          "name": "HuggingFaceTextTranslationTransformer",
+          "settings": {
+            "model": "Helsinki-NLP/opus-mt-en-fr",
+            "max_length": 100,
+            "min_length": 50,
+            "num_beams": 1
+          }
+        },
+        "parameters": {}
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "execution_count": 0,
+          "data": " Il \u00e9tait une fois, dans une for\u00eat luxuriante et vibrante, vivait une magnifique cr\u00e9ature connue sous le nom de Foxe Brun. Ce renard \u00e9tait diff\u00e9rent des autres, poss\u00e9dant une vitesse et une agilit\u00e9 incroyables qui amenaient tous les animaux dans la for\u00eat. Avec sa fourrure aussi dor\u00e9e que le soleil et ses yeux aussi aiguis\u00e9s que les \u00e9meraudes, le Foxe Brun Brun \u00e9tait admir\u00e9 par tout le monde, du plus petit colibri au",
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "name": "translation_with_parameters",
+      "input": "Tell the tale of {{topic}}",
+      "metadata": {
+        "model": {
+          "name": "HuggingFaceTextTranslationTransformer",
+          "settings": {
+            "model": "Helsinki-NLP/opus-mt-en-fr",
+            "max_length": 100,
+            "min_length": 50,
+            "num_beams": 1
+          }
+        },
+        "parameters": {
+          "topic": "the quick brown fox"
+        }
+      },
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "execution_count": 0,
+          "data": " Racontez l'histoire du renard brun rapide. - C'est \u00e7a. - C'est \u00e7a. - C'est \u00e7a. - C'est \u00e7a. - C'est \u00e7a. - C'est \u00e7a. - C'est \u00e7a.",
+          "metadata": {}
+        }
+      ]
+    },
+    {
+      "name": "summarize_story",
+      "input": "Once upon a time, in a lush and vibrant forest, there lived a magnificent creature known as the Quick Brown Fox. This fox was unlike any other, possessing incredible speed and agility that awed all the animals in the forest. With its fur as golden as the sun and its eyes as sharp as emeralds, the Quick Brown Fox was admired by everyone, from the tiniest hummingbird to the mightiest bear. The fox had a kind heart and would often lend a helping paw to those in need. The Quick Brown Fox had a particular fondness for games and challenges. It loved to test its skills against others, always seeking new adventures to satisfy its boundless curiosity. Its favorite game was called \"The Great Word Hunt,\" where it would embark on a quest to find hidden words scattered across the forest.",
+      "metadata": {
+        "model": "stevhliu/my_awesome_billsum_model",
+        "parameters": {}
+      },
+      "outputs": []
+    },
+    {
+      "name": "generate_audio_title",
+      "input": "The Quick Brown Fox was admired by all the animals in the forest.",
+      "metadata": {
+        "model": {
+          "name": "suno/bark",
+          "settings": {}
+        },
+        "parameters": {}
+      },
+      "outputs": []
+    },
+    {
+      "name": "generate_caption",
+      "input": {
+        "attachments": [
+          {
+            "data": "/Users/jonathan/Desktop/pic.png",
+            "mime_type": "image/png"
+          }
+        ]
+      },
+      "metadata": {
+        "model": "Salesforce/blip-image-captioning-base",
+        "parameters": {}
+      },
+      "outputs": []
+    },
+    {
+      "name": "openai_gen_itinerary",
+      "input": "Generate an itinerary for a 2 day trip to NYC ordered by {{order_by}}.",
+      "metadata": {
+        "model": "gpt-4",
+        "parameters": {
+          "order_by": "geographic location"
+        }
+      },
+      "outputs": []
+    },
+    {
+      "name": "asr",
+      "input": {
+        "attachments": [
+          {
+            "data": "./hi.mp3",
+            "mime_type": "audio/mpeg"
+          }
+        ]
+      },
+      "metadata": {
+        "model": "openai/whisper-small",
+        "parameters": {}
+      },
+      "outputs": []
+    }
+  ],
+  "$schema": "https://json.schemastore.org/aiconfig-1.0"
+}

cookbooks/Gradio/requirements.txt

@@ -0,0 +1,5 @@
+# AIConfig
+python-aiconfig
+
+# Hugging Face Extension for AIConfig
+aiconfig-extension-hugging-face >= 0.0.2

cookbooks/Gradio/travel.aiconfig.json

@@ -0,0 +1,40 @@
+{
+  "name": "NYC Trip Planner",
+  "schema_version": "latest",
+  "metadata": {
+    "parameters": {
+      "": ""
+    },
+    "models": {
+      "gpt-3.5-turbo": {
+        "model": "gpt-3.5-turbo",
+        "top_p": 1,
+        "temperature": 1
+      },
+      "gpt-4": {
+        "model": "gpt-4",
+        "max_tokens": 3000,
+        "system_prompt": "You are an expert travel coordinator with exquisite taste."
+      }
+    },
+    "default_model": "gpt-3.5-turbo"
+  },
+  "description": "Intrepid explorer with ChatGPT and AIConfig",
+  "prompts": [
+    {
+      "name": "get_activities",
+      "input": "Tell me 10 fun attractions to do in NYC."
+    },
+    {
+      "name": "gen_itinerary",
+      "input": "Generate an itinerary ordered by {{order_by}} for these activities: {{get_activities.output}}.",
+      "metadata": {
+        "model": "gpt-4",
+        "parameters": {
+          "order_by": "geographic location"
+        }
+      }
+    }
+  ],
+  "$schema": "https://json.schemastore.org/aiconfig-1.0"
+}