Lift to SpeziLLM

Package.swift

@@ -26,9 +26,13 @@
     dependencies: [
         .package(url: "https://github.com/apple/FHIRModels", .upToNextMinor(from: "0.5.0")),
         .package(url: "https://github.com/StanfordBDHG/HealthKitOnFHIR", .upToNextMinor(from: "0.2.4")),
-        .package(url: "https://github.com/StanfordSpezi/Spezi", .upToNextMinor(from: "0.8.0")),
-        .package(url: "https://github.com/StanfordSpezi/SpeziHealthKit.git", .upToNextMinor(from: "0.4.0")),
-        .package(url: "https://github.com/StanfordSpezi/SpeziML.git", .upToNextMinor(from: "0.3.1"))
+        .package(url: "https://github.com/StanfordSpezi/Spezi", from: "1.1.0"),
+        .package(url: "https://github.com/StanfordSpezi/SpeziHealthKit.git", .upToNextMinor(from: "0.5.0")),
+        //.package(url: "https://github.com/StanfordSpezi/SpeziLLM.git", branch: "feat/structural-improvments"),
+        .package(path: "../SpeziLLM"),
+        .package(url: "https://github.com/StanfordSpezi/SpeziStorage.git", from: "1.0.0"),
+        .package(url: "https://github.com/StanfordSpezi/SpeziChat.git", .upToNextMinor(from: "0.1.4")),
+        .package(url: "https://github.com/StanfordSpezi/SpeziSpeech.git", from: "1.0.0")
     ],
     targets: [
         .target(
@@ -54,7 +58,11 @@
                 .target(name: "SpeziFHIR"),
                 .product(name: "Spezi", package: "Spezi"),
                 .product(name: "ModelsR4", package: "FHIRModels"),
-                .product(name: "SpeziOpenAI", package: "SpeziML")
+                .product(name: "SpeziLLM", package: "SpeziLLM"),
+                .product(name: "SpeziLLMOpenAI", package: "SpeziLLM"),
+                .product(name: "SpeziLocalStorage", package: "SpeziStorage"),
+                .product(name: "SpeziChat", package: "SpeziChat"),
+                .product(name: "SpeziSpeechSynthesizer", package: "SpeziSpeech")
             ],
             resources: [
                 .process("Resources")
@@ -63,7 +71,8 @@
         .target(
             name: "SpeziFHIRMockPatients",
             dependencies: [
-                .target(name: "SpeziFHIR")
+                .target(name: "SpeziFHIR"),
+                .product(name: "ModelsR4", package: "FHIRModels")
             ],
             resources: [
                 .process("Resources")

Sources/SpeziFHIRInterpretation/FHIRInterpretation/MultipleResources/FHIRGetResourceLLMFunction.swift

@@ -0,0 +1,109 @@
+//
+// This source file is part of the Stanford LLM on FHIR project
+//
+// SPDX-FileCopyrightText: 2023 Stanford University
+//
+// SPDX-License-Identifier: MIT
+//
+
+import os
+import SpeziFHIR
+import SpeziLLMOpenAI
+
+
+struct FHIRGetResourceLLMFunction: LLMFunction {
+    static let logger = Logger(subsystem: "edu.stanford.spezi.fhir", category: "SpeziFHIRInterpretation")
+
+    static let name = "get_resources"
+    static let description = String(localized: "FUNCTION_DESCRIPTION")
+
+    private let fhirStore: FHIRStore
+    private let resourceSummary: FHIRResourceSummary
+    private let allResourcesFunctionCallIdentifier: [String]
+
+
+    @Parameter var resources: [String]
+
+
+    init(fhirStore: FHIRStore, resourceSummary: FHIRResourceSummary, allResourcesFunctionCallIdentifier: [String]) {
+        self.fhirStore = fhirStore
+        self.resourceSummary = resourceSummary
+        self.allResourcesFunctionCallIdentifier = allResourcesFunctionCallIdentifier
+
+        _resources = Parameter(description: String(localized: "PARAMETER_DESCRIPTION"), enumValues: allResourcesFunctionCallIdentifier)
+    }
+
+
+    func execute() async throws -> String? {
+        var functionOutput: [String] = []
+
+        try await withThrowingTaskGroup(of: [String].self) { outerGroup in
+            // Iterate over all requested resources by the LLM
+            for requestedResource in resources {
+                outerGroup.addTask {
+                    // Fetch relevant FHIR resources matching the resources requested by the LLM
+                    var fittingResources = fhirStore.llmRelevantResources.filter { $0.functionCallIdentifier.contains(requestedResource) }
+
+                    // Stores output of nested task group summarizing fitting resources
+                    var nestedFunctionOutputResults = [String]()
+
+                    guard !fittingResources.isEmpty else {
+                        nestedFunctionOutputResults.append(
+                            String(
+                                localized: "The medical record does not include any FHIR resources for the search term \(requestedResource)."
+                            )
+                        )
+                        return []
+                    }
+
+                    // Filter out fitting resources (if greater than 64 entries)
+                    fittingResources = filterFittingResources(fittingResources)
+
+                    try await withThrowingTaskGroup(of: String.self) { innerGroup in
+                        // Iterate over fitting resources and summarizing them
+                        for resource in fittingResources {
+                            innerGroup.addTask {
+                                try await summarizeResource(fhirResource: resource, resourceType: requestedResource)
+                            }
+                        }
+
+                        for try await nestedResult in innerGroup {
+                            nestedFunctionOutputResults.append(nestedResult)
+                        }
+                    }
+
+                    return nestedFunctionOutputResults
+                }
+            }
+
+            for try await result in outerGroup {
+                functionOutput.append(contentsOf: result)
+            }
+        }
+
+        return functionOutput.joined(separator: "\n\n")
+    }
+
+    private func summarizeResource(fhirResource: FHIRResource, resourceType: String) async throws -> String {
+        let summary = try await resourceSummary.summarize(resource: fhirResource)
+        Self.logger.debug("Summary of appended FHIR resource \(resourceType): \(summary.description)")
+        return String(localized: "This is the summary of the requested \(resourceType):\n\n\(summary.description)")
+    }
+
+    private func filterFittingResources(_ fittingResources: [FHIRResource]) -> [FHIRResource] {
+        Self.logger.debug("Overall fitting Resources: \(fittingResources.count)")
+
+        var fittingResources = fittingResources
+
+        if fittingResources.count > 64 {
+            fittingResources = fittingResources.lazy.sorted(by: { $0.date ?? .distantPast < $1.date ?? .distantPast }).suffix(64)
+            Self.logger.debug(
+                """
+                Reduced to the following 64 resources: \(fittingResources.map { $0.functionCallIdentifier }.joined(separator: ","))
+                """
+            )
+        }
+
+        return fittingResources
+    }
+}

Sources/SpeziFHIRInterpretation/FHIRInterpretation/MultipleResources/FHIRMultipleResourceInterpreter.swift

@@ -0,0 +1,123 @@
+//
+// This source file is part of the Stanford LLM on FHIR project
+//
+// SPDX-FileCopyrightText: 2023 Stanford University
+//
+// SPDX-License-Identifier: MIT
+//
+
+import os
+import Spezi
+import SpeziChat
+import SpeziFHIR
+import SpeziLLM
+import SpeziLLMOpenAI
+import SpeziLocalStorage
+import SpeziViews
+import SwiftUI
+
+
+private enum FHIRMultipleResourceInterpreterConstants {
+    static let chat = "FHIRMultipleResourceInterpreter.chat"
+}
+
+
+@Observable
+class FHIRMultipleResourceInterpreter {
+    static let logger = Logger(subsystem: "edu.stanford.spezi.fhir", category: "SpeziFHIRInterpretation")
+
+    private let localStorage: LocalStorage
+    private let llmRunner: LLMRunner
+    private let llmSchema: any LLMSchema
+    private let fhirStore: FHIRStore
+
+    var llm: (any LLMSession)?
+
+
+    required init(
+        localStorage: LocalStorage,
+        llmRunner: LLMRunner,
+        llmSchema: any LLMSchema,
+        fhirStore: FHIRStore
+    ) {
+        self.localStorage = localStorage
+        self.llmRunner = llmRunner
+        self.llmSchema = llmSchema
+        self.fhirStore = fhirStore
+    }
+
+
+    @MainActor
+    func resetChat() {
+        var context = Chat()
+        context.append(systemMessage: FHIRPrompt.interpretMultipleResources.prompt)
+        if let patient = fhirStore.patient {
+            context.append(systemMessage: patient.jsonDescription)
+        }
+
+        llm?.context = context
+    }
+
+    @MainActor
+    func prepareLLM() async {
+        guard llm == nil else {
+            return
+        }
+
+        var llm = await llmRunner(with: llmSchema)
+        // Read initial conversation from storage
+        if let storedContext: Chat = try? localStorage.read(storageKey: FHIRMultipleResourceInterpreterConstants.chat) {
+            llm.context = storedContext
+        } else {
+            llm.context.append(systemMessage: FHIRPrompt.interpretMultipleResources.prompt)
+            if let patient = fhirStore.patient {
+                llm.context.append(systemMessage: patient.jsonDescription)
+            }
+        }
+
+        self.llm = llm
+    }
+
+    @MainActor
+    func queryLLM() {
+        guard let llm,
+              llm.context.last?.role == .user || !(llm.context.contains(where: { $0.role == .assistant }) ?? false) else {
+            return
+        }
+
+        Task {
+            Self.logger.debug("The Multiple Resource Interpreter has access to \(self.fhirStore.llmRelevantResources.count) resources.")
+
+            guard let stream = try? await llm.generate() else {
+                return
+            }
+
+            for try await token in stream {
+                llm.context.append(assistantOutput: token)
+            }
+
+            // Store conversation to storage
+            try localStorage.store(llm.context, storageKey: FHIRMultipleResourceInterpreterConstants.chat)
+        }
+    }
+}
+
+
+extension FHIRPrompt {
+    /// Prompt used to interpret multiple FHIR resources
+    ///
+    /// This prompt is used by the ``FHIRMultipleResourceInterpreter``.
+    public static let interpretMultipleResources: FHIRPrompt = {
+        FHIRPrompt(
+            storageKey: "prompt.interpretMultipleResources",
+            localizedDescription: String(
+                localized: "Interpretation Prompt",
+                comment: "Title of the multiple resources interpretation prompt."
+            ),
+            defaultPrompt: String(
+                localized: "Interpretation Prompt Content",
+                comment: "Content of the multiple resources interpretation prompt."
+            )
+        )
+    }()
+}