Develop

src/airunner/aihandler/engine.py

@@ -6,7 +6,6 @@
 from airunner.aihandler.llm import LLM
 from airunner.aihandler.logger import Logger as logger
 from airunner.aihandler.runner import SDRunner
-from airunner.data.managers import SettingsManager
 from airunner.aihandler.tts import TTS
 
 
@@ -33,7 +32,6 @@ def __init__(self, **kwargs):
             engine=self
         )
         self.tts = TTS()
-        self.settings_manager = SettingsManager()
         self.tts_thread = threading.Thread(target=self.tts.run)
         self.tts_thread.start()
 
@@ -51,11 +49,12 @@ def generator_sample(self, data: dict):
         logger.info("generator_sample called")
         is_llm = self.is_llm_request(data)
         is_tts = self.is_tts_request(data)
+        self.request_data = data["request_data"]
         if is_llm and self.model_type != "llm":
             logger.info("Switching to LLM model")
             self.model_type = "llm"
-            do_unload_model = self.settings_manager.settings.unload_unused_model
-            do_move_to_cpu = not do_unload_model and self.settings_manager.settings.move_unused_model_to_cpu
+            do_unload_model = self.request_data["unload_unused_model"]
+            do_move_to_cpu = not do_unload_model and self.request_data["move_unused_model_to_cpu"]
             if do_move_to_cpu:
                 self.move_pipe_to_cpu()
             elif do_unload_model:
@@ -100,9 +99,12 @@ def unload_llm(self):
         VRAM to keep the LLM loaded while
         using other models.
         """
-        do_unload_model = self.settings_manager.settings.unload_unused_model
-        do_move_to_cpu = not do_unload_model and self.settings_manager.settings.move_unused_model_to_cpu
-        dtype = self.settings_manager.llm_generator_settings.dtype
+        do_unload_model = self.request_data["unload_unused_model"]
+        move_unused_model_to_cpu = self.request_data["move_unused_model_to_cpu"]
+
+
+        do_move_to_cpu = not do_unload_model and move_unused_model_to_cpu
+        dtype = self.request_data["dtype"]
         if dtype in ["2bit", "4bit", "8bit"]:
             do_unload_model = True
             do_move_to_cpu = False

src/airunner/aihandler/transformer_runner.py

@@ -3,16 +3,10 @@
 
 from PyQt6.QtCore import QObject
 
-# import AutoTokenizer
-# import BitsAndBytesConfig
 from transformers import BitsAndBytesConfig
 from transformers import AutoModelForCausalLM, AutoModelForSeq2SeqLM, AutoTokenizer
 from transformers import InstructBlipForConditionalGeneration
 from transformers import InstructBlipProcessor
-
-from airunner.data.managers import SettingsManager
-from airunner.data.models import LLMGenerator
-from airunner.data.session_scope import session_scope
 from airunner.aihandler.logger import Logger
 
 
@@ -83,7 +77,6 @@ def __init__(self, *args, **kwargs):
         app = kwargs.pop("app", None)
         self.app = app
         super().__init__(*args, **kwargs)
-        self.settings_manager = SettingsManager()
         # self.llm_api = LLMAPI(app=app)
 
     def move_to_cpu(self):
@@ -175,7 +168,7 @@ def load_model(self, local_files_only = None):
             "device_map": self.device_map,
             "use_cache": self.use_cache,
             "torch_dtype": torch.float16 if self.dtype != "32bit" else torch.float32,
-            "token": self.settings_manager.settings.hf_api_key_read_key,
+            "token": self.request_data.get("hf_api_key_read_key", ""),
             "trust_remote_code": True
         }
 
@@ -205,14 +198,6 @@ def load_model(self, local_files_only = None):
                     return self.load_model(local_files_only=False)
                 else:
                     Logger.error(e)
-
-        # if self.requested_generator_name == "casuallm":
-        #     self.pipeline = AutoModelForCausalLM.from_pretrained(
-        #         self.model,
-        #         torch_dtype=torch.float16 if self.dtype != "32bit" else torch.float32,
-        #         trust_remote_code=False,
-        #         device_map="auto"
-        #     )
 
 
     def process_data(self, data):
@@ -314,7 +299,7 @@ def load_tokenizer(self, local_files_only = None):
                 self.tokenizer = AutoTokenizer.from_pretrained(
                     self.current_model_path,
                     local_files_only=local_files_only,
-                    token=self.settings_manager.settings.hf_api_key_read_key,
+                    token=self.request_data.get("hf_api_key_read_key"),
                     device_map=self.device_map,
                 )
             except OSError as e:

src/airunner/alembic/versions/83be186bd388_removes_llm_generator_setting_from_db.py

@@ -0,0 +1,54 @@
+"""removes llm generator setting from db
+
+Revision ID: 83be186bd388
+Revises: ca640e5c9d6d
+Create Date: 2024-01-11 16:39:24.547116
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '83be186bd388'
+down_revision: Union[str, None] = 'ca640e5c9d6d'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('llm_generator_settings')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('llm_generator_settings',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('created_at', sa.DATETIME(), nullable=True),
+    sa.Column('top_p', sa.INTEGER(), nullable=True),
+    sa.Column('max_length', sa.INTEGER(), nullable=True),
+    sa.Column('repetition_penalty', sa.INTEGER(), nullable=True),
+    sa.Column('min_length', sa.INTEGER(), nullable=True),
+    sa.Column('length_penalty', sa.INTEGER(), nullable=True),
+    sa.Column('num_beams', sa.INTEGER(), nullable=True),
+    sa.Column('ngram_size', sa.INTEGER(), nullable=True),
+    sa.Column('temperature', sa.INTEGER(), nullable=True),
+    sa.Column('sequences', sa.INTEGER(), nullable=True),
+    sa.Column('top_k', sa.INTEGER(), nullable=True),
+    sa.Column('seed', sa.INTEGER(), nullable=True),
+    sa.Column('do_sample', sa.BOOLEAN(), nullable=True),
+    sa.Column('eta_cutoff', sa.INTEGER(), nullable=True),
+    sa.Column('early_stopping', sa.BOOLEAN(), nullable=True),
+    sa.Column('random_seed', sa.BOOLEAN(), nullable=True),
+    sa.Column('model_version', sa.VARCHAR(), nullable=True),
+    sa.Column('generator_id', sa.INTEGER(), nullable=True),
+    sa.Column('dtype', sa.VARCHAR(), nullable=True),
+    sa.Column('use_gpu', sa.BOOLEAN(), nullable=True),
+    sa.ForeignKeyConstraint(['generator_id'], ['llm_generator.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###

src/airunner/data/db.py

@@ -6,7 +6,7 @@
 from airunner.data.models import ControlnetModel, LLMPromptTemplate, Pipeline, Document, \
     MetadataSettings, AIModel, \
     ImageFilter, ImageFilterValue, Scheduler, ActionScheduler, \
-    LLMGeneratorSetting, LLMGenerator, LLMModelVersion, StandardImageWidgetSettings
+    LLMGenerator, LLMModelVersion, StandardImageWidgetSettings
 from airunner.data.session_scope import session_scope, engine
 from alembic.config import Config
 from alembic import command
@@ -172,13 +172,6 @@ def prepare_database():
                 generator = LLMGenerator(name=generator_name)
                 my_session.add(generator)
 
-                # create GeneratorSetting with property, value and property_type based on value type
-                setting = LLMGeneratorSetting()
-                setting.generator = generator
-                for k, v in generator_data["generator_settings"].items():
-                    setting.__setattr__(k, v)
-                my_session.add(setting)
-
                 if "model_versions" in generator_data:
                     model_versions = []
                     for name in generator_data["model_versions"]:

src/airunner/data/managers.py

@@ -10,7 +10,6 @@
     models_scope,
     session_scope,
     llm_generator_scope,
-    llm_generator_settings_scope,
     canvas_settings_scope,
     memory_settings_scope,
     metadata_settings_scope
@@ -69,7 +68,6 @@ class SettingsManager(QObject):
         "standard_image_settings": standard_image_widget_settings_scope,
         "models": models_scope,
         "llm_generator": llm_generator_scope,
-        "llm_generator_settings": llm_generator_settings_scope,
         "canvas_settings": canvas_settings_scope,
         "memory_settings": memory_settings_scope,
     }
@@ -86,7 +84,6 @@ def __init__(self):
         self.standard_image_settings = Modelmanager(standard_image_widget_settings_scope)
         self.models = Modelmanager(models_scope)
         self.llm_generator = Modelmanager(llm_generator_scope)
-        self.llm_generator_settings = Modelmanager(llm_generator_settings_scope)
         self.canvas_settings = Modelmanager(canvas_settings_scope)
         self.memory_settings = Modelmanager(memory_settings_scope)
         self.metadata_settings = Modelmanager(metadata_settings_scope)

src/airunner/data/models.py

@@ -327,31 +327,6 @@ class LLMGenerator(BaseModel):
     prompt_template = Column(String, default=prompt_template_seed_data[0]["name"])
 
 
-class LLMGeneratorSetting(BaseModel):
-    __tablename__ = 'llm_generator_settings'
-    id = Column(Integer, primary_key=True)
-    created_at = Column(DateTime, default=datetime.datetime.utcnow)
-    top_p = Column(Integer, default=90)
-    max_length = Column(Integer, default=50)
-    repetition_penalty = Column(Integer, default=100)
-    min_length = Column(Integer, default=10)
-    length_penalty = Column(Integer, default=100)
-    num_beams = Column(Integer, default=1)
-    ngram_size = Column(Integer, default=0)
-    temperature = Column(Integer, default=100)
-    sequences = Column(Integer, default=1)
-    top_k = Column(Integer, default=0)
-    seed = Column(Integer, default=0)
-    do_sample = Column(Boolean, default=False)
-    eta_cutoff = Column(Integer, default=10)
-    early_stopping = Column(Boolean, default=False)
-    random_seed = Column(Boolean, default=False)
-    model_version = Column(String, default="google/flan-t5-xl")
-    generator_id = Column(Integer, ForeignKey('llm_generator.id'))
-    dtype = Column(String, default="4bit")
-    use_gpu = Column(Boolean, default=True)
-
-
 class LLMModelVersion(BaseModel):
     __tablename__ = 'llm_model_version'
     id = Column(Integer, primary_key=True)

src/airunner/data/session_scope.py

@@ -60,13 +60,6 @@ def llm_generator_scope():
         llm_generator = session.query(LLMGenerator).options(joinedload('*')).first()
         yield llm_generator
 
-@contextmanager
-def llm_generator_settings_scope():
-    from airunner.data.models import LLMGeneratorSetting
-    with session_scope() as session:
-        llm_generator = session.query(LLMGeneratorSetting).options(joinedload('*')).first()
-        yield llm_generator
-
 @contextmanager
 def canvas_settings_scope():
     from airunner.data.models import CanvasSettings

src/airunner/widgets/canvas_plus/canvas_plus_widget.py

@@ -212,6 +212,8 @@ def __init__(self, parent=None):
         self.image.fill(Qt.GlobalColor.transparent)
         self.item = QGraphicsPixmapItem(QPixmap.fromImage(self.image))
         self.addItem(self.item)
+        # self.item should always be on top
+        self.item.setZValue(1)
 
         # self.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff)
         # self.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff)
@@ -274,7 +276,7 @@ def wheelEvent(self, event):
     def eraseAt(self, position):
         painter = QPainter(self.image)
         painter.setPen(QPen(Qt.GlobalColor.white, self.app.brush_settings["size"], Qt.PenStyle.SolidLine, Qt.PenCapStyle.RoundCap))
-        painter.setCompositionMode(QPainter.CompositionMode.CompositionMode_Source)
+        painter.setCompositionMode(QPainter.CompositionMode.CompositionMode_Clear)
 
         # Create a QPainterPath
         path = QPainterPath()

src/airunner/widgets/llm/chat_prompt_widget.py

@@ -212,11 +212,15 @@ def action_button_clicked_send(self, image_override=None, prompt_override=None,
                 LLMPromptTemplate.name == self.llm_generator.prompt_template
             ).first()
 
+            llm_generator_settings = self.app.llm_generator_settings
+
             data = {
                 "llm_request": True,
                 "request_data": {
+                    "unload_unused_model": self.app.unload_unused_models,
+                    "move_unused_model_to_cpu": self.app.move_unused_model_to_cpu,
                     "generator_name": generator_name,
-                    "model_path": self.llm_generator_settings.model_version,
+                    "model_path": llm_generator_settings["model_version"],
                     "stream": True,
                     "prompt": prompt,
                     "do_summary": False,
@@ -225,32 +229,34 @@ def action_button_clicked_send(self, image_override=None, prompt_override=None,
                     "generator": self.llm_generator,
                     "prefix": self.prefix,
                     "suffix": self.suffix,
-                    "dtype": self.llm_generator_settings.dtype,
-                    "use_gpu": self.llm_generator_settings.use_gpu,
+                    "dtype": llm_generator_settings["dtype"],
+                    "use_gpu": llm_generator_settings["use_gpu"],
                     "request_type": "image_caption_generator",
                     "username": self.llm_generator.username,
                     "botname": self.llm_generator.botname,
                     "prompt_template": prompt_template.template,
+                    "hf_api_key_read_key": self.app.hf_api_key_read_key,
                     "parameters": {
                         "override_parameters": self.llm_generator.override_parameters,
-                        "top_p": self.llm_generator_settings.top_p / 100.0,
-                        "max_length": self.llm_generator_settings.max_length,
-                        "repetition_penalty": self.llm_generator_settings.repetition_penalty / 100.0,
-                        "min_length": self.llm_generator_settings.min_length,
-                        "length_penalty": self.llm_generator_settings.length_penalty / 100,
-                        "num_beams": self.llm_generator_settings.num_beams,
-                        "ngram_size": self.llm_generator_settings.ngram_size,
-                        "temperature": self.llm_generator_settings.temperature / 10000.0,
-                        "sequences": self.llm_generator_settings.sequences,
-                        "top_k": self.llm_generator_settings.top_k,
-                        "eta_cutoff": self.llm_generator_settings.eta_cutoff / 100.0,
-                        "seed": self.llm_generator_settings.do_sample,
-                        "early_stopping": self.llm_generator_settings.early_stopping,
+                        "top_p": llm_generator_settings["top_p"] / 100.0,
+                        "max_length": llm_generator_settings["max_length"],
+                        "repetition_penalty": llm_generator_settings["repetition_penalty"] / 100.0,
+                        "min_length": llm_generator_settings["min_length"],
+                        "length_penalty": llm_generator_settings["length_penalty"] / 100,
+                        "num_beams": llm_generator_settings["num_beams"],
+                        "ngram_size": llm_generator_settings["ngram_size"],
+                        "temperature": llm_generator_settings["temperature"] / 10000.0,
+                        "sequences": llm_generator_settings["sequences"],
+                        "top_k": llm_generator_settings["top_k"],
+                        "eta_cutoff": llm_generator_settings['eta_cutoff'] / 100.0,
+                        "seed": llm_generator_settings["do_sample"],
+                        "early_stopping": llm_generator_settings["early_stopping"],
                     },
                     "image": image,
                     "callback": callback
                 }
             }
+            print(data)
             message_object = Message(
                 name=self.llm_generator.username,
                 message=self.prompt,