Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Current state of moving LLMonFHIR code to SpeziFHIR
- Loading branch information
1 parent
b58ba7a
commit fd7ac71
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| // | ||
| // This source file is part of the Stanford LLM on FHIR project | ||
| // | ||
| // SPDX-FileCopyrightText: 2023 Stanford University | ||
| // | ||
| // SPDX-License-Identifier: MIT | ||
| // | ||
|
|
||
| import os | ||
| import SpeziFHIR | ||
| import SpeziLLMOpenAI | ||
|
|
||
|
|
||
| struct FHIRGetResourceLLMFunction: LLMFunction { | ||
| static let logger = Logger(subsystem: "edu.stanford.spezi.fhir", category: "SpeziFHIRInterpretation") | ||
|
|
||
| static let name = "get_resources" | ||
| static let description = String(localized: "FUNCTION_DESCRIPTION") | ||
|
|
||
| private let fhirStore: FHIRStore | ||
| private let resourceSummary: FHIRResourceSummary | ||
| private let allResourcesFunctionCallIdentifier: [String] | ||
|
|
||
|
|
||
| @Parameter var resources: [String] | ||
|
|
||
|
|
||
| init(fhirStore: FHIRStore, resourceSummary: FHIRResourceSummary, allResourcesFunctionCallIdentifier: [String]) { | ||
| self.fhirStore = fhirStore | ||
| self.resourceSummary = resourceSummary | ||
| self.allResourcesFunctionCallIdentifier = allResourcesFunctionCallIdentifier | ||
|
|
||
| _resources = Parameter(description: String(localized: "PARAMETER_DESCRIPTION"), enumValues: allResourcesFunctionCallIdentifier) | ||
| } | ||
|
|
||
|
|
||
| func execute() async throws -> String? { | ||
| var functionOutput: [String] = [] | ||
|
|
||
| try await withThrowingTaskGroup(of: [String].self) { outerGroup in | ||
| // Iterate over all requested resources by the LLM | ||
| for requestedResource in resources { | ||
| outerGroup.addTask { | ||
| // Fetch relevant FHIR resources matching the resources requested by the LLM | ||
| var fittingResources = fhirStore.llmRelevantResources.filter { $0.functionCallIdentifier.contains(requestedResource) } | ||
|
|
||
| // Stores output of nested task group summarizing fitting resources | ||
| var nestedFunctionOutputResults = [String]() | ||
|
|
||
| guard !fittingResources.isEmpty else { | ||
| nestedFunctionOutputResults.append( | ||
| String( | ||
| localized: "The medical record does not include any FHIR resources for the search term \(requestedResource)." | ||
| ) | ||
| ) | ||
| return [] | ||
| } | ||
|
|
||
| // Filter out fitting resources (if greater than 64 entries) | ||
| fittingResources = filterFittingResources(fittingResources) | ||
|
|
||
| try await withThrowingTaskGroup(of: String.self) { innerGroup in | ||
| // Iterate over fitting resources and summarizing them | ||
| for resource in fittingResources { | ||
| innerGroup.addTask { | ||
| try await summarizeResource(fhirResource: resource, resourceType: requestedResource) | ||
| } | ||
| } | ||
|
|
||
| for try await nestedResult in innerGroup { | ||
| nestedFunctionOutputResults.append(nestedResult) | ||
| } | ||
| } | ||
|
|
||
| return nestedFunctionOutputResults | ||
| } | ||
| } | ||
|
|
||
| for try await result in outerGroup { | ||
| functionOutput.append(contentsOf: result) | ||
| } | ||
| } | ||
|
|
||
| return functionOutput.joined(separator: "\n\n") | ||
| } | ||
|
|
||
| private func summarizeResource(fhirResource: FHIRResource, resourceType: String) async throws -> String { | ||
| let summary = try await resourceSummary.summarize(resource: fhirResource) | ||
| Self.logger.debug("Summary of appended FHIR resource \(resourceType): \(summary.description)") | ||
| return String(localized: "This is the summary of the requested \(resourceType):\n\n\(summary.description)") | ||
| } | ||
|
|
||
| private func filterFittingResources(_ fittingResources: [FHIRResource]) -> [FHIRResource] { | ||
| Self.logger.debug("Overall fitting Resources: \(fittingResources.count)") | ||
|
|
||
| var fittingResources = fittingResources | ||
|
|
||
| if fittingResources.count > 64 { | ||
| fittingResources = fittingResources.lazy.sorted(by: { $0.date ?? .distantPast < $1.date ?? .distantPast }).suffix(64) | ||
| Self.logger.debug( | ||
| """ | ||
| Reduced to the following 64 resources: \(fittingResources.map { $0.functionCallIdentifier }.joined(separator: ",")) | ||
| """ | ||
| ) | ||
| } | ||
|
|
||
| return fittingResources | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| // | ||
| // This source file is part of the Stanford LLM on FHIR project | ||
| // | ||
| // SPDX-FileCopyrightText: 2023 Stanford University | ||
| // | ||
| // SPDX-License-Identifier: MIT | ||
| // | ||
|
|
||
| import os | ||
| import Spezi | ||
| import SpeziChat | ||
| import SpeziFHIR | ||
| import SpeziLLM | ||
| import SpeziLLMOpenAI | ||
| import SpeziLocalStorage | ||
| import SpeziViews | ||
| import SwiftUI | ||
|
|
||
|
|
||
| private enum FHIRMultipleResourceInterpreterConstants { | ||
| static let chat = "FHIRMultipleResourceInterpreter.chat" | ||
| } | ||
|
|
||
|
|
||
| @Observable | ||
| class FHIRMultipleResourceInterpreter { | ||
| static let logger = Logger(subsystem: "edu.stanford.spezi.fhir", category: "SpeziFHIRInterpretation") | ||
|
|
||
| private let localStorage: LocalStorage | ||
| private let llmRunner: LLMRunner | ||
| private let llmSchema: any LLMSchema | ||
| private let fhirStore: FHIRStore | ||
|
|
||
| var llm: (any LLMSession)? | ||
|
|
||
|
|
||
| required init( | ||
| localStorage: LocalStorage, | ||
| llmRunner: LLMRunner, | ||
| llmSchema: any LLMSchema, | ||
| fhirStore: FHIRStore | ||
| ) { | ||
| self.localStorage = localStorage | ||
| self.llmRunner = llmRunner | ||
| self.llmSchema = llmSchema | ||
| self.fhirStore = fhirStore | ||
| } | ||
|
|
||
|
|
||
| @MainActor | ||
| func resetChat() { | ||
| var context = Chat() | ||
| context.append(systemMessage: FHIRPrompt.interpretMultipleResources.prompt) | ||
| if let patient = fhirStore.patient { | ||
| context.append(systemMessage: patient.jsonDescription) | ||
| } | ||
|
|
||
| llm?.context = context | ||
| } | ||
|
|
||
| @MainActor | ||
| func prepareLLM() async { | ||
| guard llm == nil else { | ||
| return | ||
| } | ||
|
|
||
| var llm = await llmRunner(with: llmSchema) | ||
| // Read initial conversation from storage | ||
| if let storedContext: Chat = try? localStorage.read(storageKey: FHIRMultipleResourceInterpreterConstants.chat) { | ||
| llm.context = storedContext | ||
| } else { | ||
| llm.context.append(systemMessage: FHIRPrompt.interpretMultipleResources.prompt) | ||
| if let patient = fhirStore.patient { | ||
| llm.context.append(systemMessage: patient.jsonDescription) | ||
| } | ||
| } | ||
|
|
||
| self.llm = llm | ||
| } | ||
|
|
||
| @MainActor | ||
| func queryLLM() { | ||
| guard let llm, | ||
| llm.context.last?.role == .user || !(llm.context.contains(where: { $0.role == .assistant }) ?? false) else { | ||
| return | ||
| } | ||
|
|
||
| Task { | ||
| Self.logger.debug("The Multiple Resource Interpreter has access to \(self.fhirStore.llmRelevantResources.count) resources.") | ||
|
|
||
| guard let stream = try? await llm.generate() else { | ||
| return | ||
| } | ||
|
|
||
| for try await token in stream { | ||
| llm.context.append(assistantOutput: token) | ||
| } | ||
|
|
||
| // Store conversation to storage | ||
| try localStorage.store(llm.context, storageKey: FHIRMultipleResourceInterpreterConstants.chat) | ||
| } | ||
| } | ||
| } | ||
|
|
||
|
|
||
| extension FHIRPrompt { | ||
| /// Prompt used to interpret multiple FHIR resources | ||
| /// | ||
| /// This prompt is used by the ``FHIRMultipleResourceInterpreter``. | ||
| public static let interpretMultipleResources: FHIRPrompt = { | ||
| FHIRPrompt( | ||
| storageKey: "prompt.interpretMultipleResources", | ||
| localizedDescription: String( | ||
| localized: "Interpretation Prompt", | ||
| comment: "Title of the multiple resources interpretation prompt." | ||
| ), | ||
| defaultPrompt: String( | ||
| localized: "Interpretation Prompt Content", | ||
| comment: "Content of the multiple resources interpretation prompt." | ||
| ) | ||
| ) | ||
| }() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| // | ||
| // This source file is part of the Stanford LLM on FHIR project | ||
| // | ||
| // SPDX-FileCopyrightText: 2023 Stanford University | ||
| // | ||
| // SPDX-License-Identifier: MIT | ||
| // | ||
|
|
||
| import SpeziChat | ||
| import SpeziFHIR | ||
| import SpeziLLM | ||
| import SpeziLLMOpenAI | ||
| import SpeziSpeechSynthesizer | ||
| import SpeziViews | ||
| import SwiftUI | ||
|
|
||
|
|
||
| public struct MultipleResourcesChatView: View { | ||
| @Environment(FHIRMultipleResourceInterpreter.self) private var multipleResourceInterpreter | ||
| @Environment(\.dismiss) private var dismiss | ||
|
|
||
| @Binding private var textToSpeech: Bool | ||
| private let navigationTitle: Text | ||
|
|
||
|
|
||
| public var body: some View { | ||
| @Bindable var multipleResourceInterpreter = multipleResourceInterpreter | ||
| NavigationStack { | ||
| Group { | ||
| if let llm = multipleResourceInterpreter.llm { | ||
| let contextBinding = Binding { llm.context } set: { llm.context = $0 } | ||
| ChatView( | ||
| contextBinding, | ||
| disableInput: llm.state.representation == .processing | ||
| ) | ||
| .viewStateAlert(state: llm.state) | ||
| .onChange(of: llm.context, initial: true) { old, new in | ||
|
Check failure on line 37 in Sources/SpeziFHIRInterpretation/FHIRInterpretation/MultipleResources/MultipleResourcesChatView.swift
|
||
| if llm.state != .generating { | ||
| multipleResourceInterpreter.queryLLM() | ||
| } | ||
| } | ||
| } else { | ||
| ChatView( | ||
| .constant([]) | ||
| ) | ||
| } | ||
| } | ||
| .navigationTitle(navigationTitle) | ||
| .toolbar { | ||
| toolbar | ||
| } | ||
| .task { | ||
| await multipleResourceInterpreter.prepareLLM() | ||
| } | ||
| } | ||
| .interactiveDismissDisabled() | ||
| } | ||
|
|
||
|
|
||
| @MainActor @ToolbarContentBuilder private var toolbar: some ToolbarContent { | ||
| ToolbarItem(placement: .cancellationAction) { | ||
| if multipleResourceInterpreter.llm?.state.representation == .processing { | ||
| ProgressView() | ||
| } else { | ||
| Button("Close") { | ||
| dismiss() | ||
| } | ||
| } | ||
| } | ||
| ToolbarItem(placement: .primaryAction) { | ||
| Button( | ||
| action: { | ||
| textToSpeech.toggle() | ||
| }, | ||
| label: { | ||
| if textToSpeech { | ||
| Image(systemName: "speaker") | ||
| .accessibilityLabel(Text("Text to speech is enabled, press to disable text to speech.")) | ||
| } else { | ||
| Image(systemName: "speaker.slash") | ||
| .accessibilityLabel(Text("Text to speech is disabled, press to enable text to speech.")) | ||
| } | ||
| } | ||
| ) | ||
| } | ||
| ToolbarItem(placement: .primaryAction) { | ||
| Button( | ||
| action: { | ||
| multipleResourceInterpreter.resetChat() | ||
| }, | ||
| label: { | ||
| Image(systemName: "trash") | ||
| .accessibilityLabel(Text("Reset Chat")) | ||
| } | ||
| ) | ||
| .disabled(multipleResourceInterpreter.llm?.state.representation == .processing) | ||
| } | ||
| } | ||
|
|
||
|
|
||
| /// Creates a ``MultipleResourcesChatView`` displaying a Spezi `Chat` with all available FHIR resources via a Spezi LLM.. | ||
| /// | ||
| /// - Parameters: | ||
| /// - navigationTitle: The localized title displayed for purposes of navigation. | ||
| /// - textToSpeech: Indicates if the output of the LLM is converted to speech and outputted to the user. | ||
| public init( | ||
| navigationTitle: LocalizedStringResource, | ||
| textToSpeech: Binding<Bool> | ||
| ) { | ||
| self.navigationTitle = Text(navigationTitle) | ||
| self._textToSpeech = textToSpeech | ||
| } | ||
|
|
||
| /// Creates a ``MultipleResourcesChatView`` displaying a Spezi `Chat` with all available FHIR resources via a Spezi LLM.. | ||
| /// | ||
| /// - Parameters: | ||
| /// - navigationTitle: The title displayed for purposes of navigation. | ||
| /// - textToSpeech: Indicates if the output of the LLM is converted to speech and outputted to the user. | ||
| @_disfavoredOverload | ||
| public init<Title: StringProtocol>( | ||
| navigationTitle: Title, | ||
| textToSpeech: Binding<Bool> | ||
| ) { | ||
| self.navigationTitle = Text(verbatim: String(navigationTitle)) | ||
| self._textToSpeech = textToSpeech | ||
| } | ||
| } | ||
File renamed without changes.
Oops, something went wrong.