Skip to content

Commit

Permalink
Step 4 - updated readme.md explaining usage of image based messages.
Browse files Browse the repository at this point in the history 
Step 3: added test project to cover all possible config and methods. Improved logic by adding content item validation, removed magic strings by moving to catalogs etc.

Support for image_url content added for chat completion endpoint

Step-1: encapsulate "Content" property in ChatGptMessage in order to support complex type of content for image support.
  • Loading branch information
@zeecorleone
zeecorleone committed Jun 8, 2024
1 parent e299688 commit 7087c2f
Show file tree
Hide file tree
Showing 12 changed files with 511 additions and 29 deletions.
38 changes: 38 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,43 @@ Console.WriteLine(response);
await bot.AskStream(response => {
Console.WriteLine(response);
}, "What is the weather like today?", "conversation name");

// Set a system message
bot.SetConversationSystemMessage("conversation name", "You are a helpful assistant that provides clear and concise answers to questions.");
```
Some models can understand both text and images. To use this feature, you need to pass a list of content items that includes both text and images.

```csharp
// To stream responses with both image and text input, create a list of content items
var contentItems = new List<ChatGptMessageContentItem>();

// Each content item can be either "Text" type or "Image" type. Let's add text to inquire about the image
contentItems.Add(new ChatGptMessageContentItem()
{
Type = ChatGptMessageContentType.TEXT,
Text = "what is this image about?"
});

// Now, create another content item of "Image" type
var contentItemWithImage = new ChatGptMessageContentItem()
{
Type = ChatGptMessageContentType.IMAGE
};
// Set image by path
contentItemWithImage.SetImageFromFile("<path-to-file>");
// Or set image by Url
contentItemWithImage.SetImageFromUrl("https://path-to-image.com/image.png");
// Or set base64 image url
contentItemWithImage.SetImageFromUrl("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA.....");

contentItems.Add(contentItemWithImage);

// Only specific models understand image input. Therefore, if you are using 'gpt-3.5-turbo' or similar model, switch or override the model before calling 'AskStream'. More details about `ChatGptOptions` (config) are covered in the next section.
config.Model = ChatGptModels.GPT_4_Vision_Preview;

var response = await bot.AskStream(response => {
Console.WriteLine(response);
}, contentItems, "conversation name");
```

### ChatGPT Unofficial API
Expand Down Expand Up @@ -120,6 +157,7 @@ Console.WriteLine(response);
await bot.AskStream(response => {
Console.WriteLine(response);
}, "What is the weather like today?", "conversation name");

```

## Configuration options
Expand Down
19 changes: 17 additions & 2 deletions src/ChatGPT.Net.sln
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@

Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ChatGPT.Net", "ChatGPT.Net\ChatGPT.Net.csproj", "{EFFFF53D-219F-48E1-87D2-4ABA130CC37F}"
# Visual Studio Version 17
VisualStudioVersion = 17.10.34916.146
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ChatGPT.Net", "ChatGPT.Net\ChatGPT.Net.csproj", "{EFFFF53D-219F-48E1-87D2-4ABA130CC37F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ChatGPT.Console", "ChatGPT.Console\ChatGPT.Console.csproj", "{677A2D4D-2EDA-4C84-8456-3CDEE6BFB4E7}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ChatGPT.Console", "ChatGPT.Console\ChatGPT.Console.csproj", "{677A2D4D-2EDA-4C84-8456-3CDEE6BFB4E7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ChatGpt.Net.Tests", "ChatGpt.Net.Tests\ChatGpt.Net.Tests.csproj", "{570FD4CD-519A-4AFB-BB12-CB3041F4E601}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand All @@ -18,5 +23,15 @@ Global
{677A2D4D-2EDA-4C84-8456-3CDEE6BFB4E7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{677A2D4D-2EDA-4C84-8456-3CDEE6BFB4E7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{677A2D4D-2EDA-4C84-8456-3CDEE6BFB4E7}.Release|Any CPU.Build.0 = Release|Any CPU
{570FD4CD-519A-4AFB-BB12-CB3041F4E601}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{570FD4CD-519A-4AFB-BB12-CB3041F4E601}.Debug|Any CPU.Build.0 = Debug|Any CPU
{570FD4CD-519A-4AFB-BB12-CB3041F4E601}.Release|Any CPU.ActiveCfg = Release|Any CPU
{570FD4CD-519A-4AFB-BB12-CB3041F4E601}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {F8BC8082-4E36-4863-9783-6C98E093FBCB}
EndGlobalSection
EndGlobal
104 changes: 81 additions & 23 deletions src/ChatGPT.Net/ChatGPT.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,21 +39,19 @@ private async IAsyncEnumerable<string> StreamCompletion(Stream stream)
public void SetConversationSystemMessage(string conversationId, string message)
{
var conversation = GetConversation(conversationId);
conversation.Messages.Add(new ChatGptMessage
conversation.Messages.Add(new ChatGptMessage(message)
{
Role = "system",
Content = message
Role = ChatGptMessageRoles.SYSTEM,
});
}

public void ReplaceConversationSystemMessage(string conversationId, string message)
{
var conversation = GetConversation(conversationId);
conversation.Messages = conversation.Messages.Where(x => x.Role != "system").ToList();
conversation.Messages.Add(new ChatGptMessage
conversation.Messages.Add(new ChatGptMessage(message)
{
Role = "system",
Content = message
Role = ChatGptMessageRoles.SYSTEM,
});
}

Expand Down Expand Up @@ -133,10 +131,9 @@ public async Task<string> Ask(string prompt, string? conversationId = null)
{
var conversation = GetConversation(conversationId);

conversation.Messages.Add(new ChatGptMessage
conversation.Messages.Add(new ChatGptMessage(prompt)
{
Role = "user",
Content = prompt
Role = ChatGptMessageRoles.USER,
});

var reply = await SendMessage(new ChatGptRequest
Expand All @@ -154,12 +151,11 @@ public async Task<string> Ask(string prompt, string? conversationId = null)

conversation.Updated = DateTime.Now;

var response = reply.Choices.FirstOrDefault()?.Message.Content ?? "";
var response = reply.Choices.FirstOrDefault()?.Message.GetContent ?? "";

conversation.Messages.Add(new ChatGptMessage
conversation.Messages.Add(new ChatGptMessage(response)
{
Role = "assistant",
Content = response
Role = ChatGptMessageRoles.ASSISTANT,
});

return response;
Expand All @@ -169,10 +165,9 @@ public async Task<string> AskStream(Action<string> callback, string prompt, stri
{
var conversation = GetConversation(conversationId);

conversation.Messages.Add(new ChatGptMessage
conversation.Messages.Add(new ChatGptMessage(prompt)
{
Role = "user",
Content = prompt
Role = ChatGptMessageRoles.USER,
});

var reply = await SendMessage(new ChatGptRequest
Expand All @@ -195,7 +190,42 @@ public async Task<string> AskStream(Action<string> callback, string prompt, stri

conversation.Updated = DateTime.Now;

return reply.Choices.FirstOrDefault()?.Message.Content ?? "";
return reply.Choices.FirstOrDefault()?.Message.GetContent ?? "";
}

public async Task<string> AskStream(Action<string> callback, List<ChatGptMessageContentItem> promptContentItems, string? conversationId = null)
{
await ValidateContentItems(promptContentItems);

var conversation = GetConversation(conversationId);
var msg = new ChatGptMessage()
{
Role = ChatGptMessageRoles.USER,
};
msg.ContentItems.AddRange(promptContentItems);
conversation.Messages.Add(msg);

var reply = await SendMessage(new ChatGptRequest
{
Messages = conversation.Messages,
Model = Config.Model,
Stream = true,
Temperature = Config.Temperature,
TopP = Config.TopP,
FrequencyPenalty = Config.FrequencyPenalty,
PresencePenalty = Config.PresencePenalty,
Stop = Config.Stop,
MaxTokens = Config.MaxTokens,
}, response =>
{
var content = response.Choices.FirstOrDefault()?.Delta.Content;
if (content is null) return;
if (!string.IsNullOrWhiteSpace(content)) callback(content);
});

conversation.Updated = DateTime.Now;

return reply.Choices.FirstOrDefault()?.Message.GetContent ?? "";
}

public async Task<ChatGptResponse> SendMessage(ChatGptRequest requestBody, Action<ChatGptStreamChunkResponse>? callback = null)
Expand All @@ -209,7 +239,7 @@ public async Task<ChatGptResponse> SendMessage(ChatGptRequest requestBody, Actio
{
{"Authorization", $"Bearer {APIKey}" }
},
Content = new StringContent(JsonConvert.SerializeObject(requestBody))
Content = new StringContent(JsonConvert.SerializeObject(requestBody, new JsonSerializerSettings() { NullValueHandling = NullValueHandling.Ignore }))
{
Headers =
{
Expand Down Expand Up @@ -249,16 +279,13 @@ public async Task<ChatGptResponse> SendMessage(ChatGptRequest requestBody, Actio
return new ChatGptResponse
{
Id = reply?.Id ?? Guid.NewGuid().ToString(),
Model = reply?.Model ?? "gpt-3.5-turbo",
Model = reply?.Model ?? ChatGptModels.GPT_3_5_Turbo,
Created = reply?.Created ?? 0,
Choices = new List<Choice>
{
new()
{
Message = new ChatGptMessage
{
Content = concatMessages
}
Message = new ChatGptMessage(concatMessages)
}
}
};
Expand All @@ -269,4 +296,35 @@ public async Task<ChatGptResponse> SendMessage(ChatGptRequest requestBody, Actio
if(content.Error is not null) throw new Exception(content.Error.Message);
return content;
}

/// <summary>
/// Validates ContentItems array for text and image type of chat completion request
/// </summary>
/// <param name="promptContentItems"></param>
/// <returns></returns>
/// <exception cref="ArgumentException">Thrown when invalid configuration is detected</exception>
/// <exception cref="InvalidOperationException">Thrown when invalid content item is detected</exception>
private async Task ValidateContentItems(List<ChatGptMessageContentItem> promptContentItems)
{
if (promptContentItems is null || promptContentItems.Count == 0)
throw new ArgumentException("Invalid request. No message to send.");

if (promptContentItems.Any(x => x.Type == ChatGptMessageContentType.IMAGE)
&& Config.Model != ChatGptModels.GPT_4_Vision_Preview)
throw new ArgumentException("Invalid model. This model cannot understand images.", "ChatGptOptions.Model");

//Validate 'text' type of content items
var textTypeItems = promptContentItems.Where(x => x.Type == ChatGptMessageContentType.TEXT).ToList();
if (textTypeItems.Any(x => x.GetImageUrl?.Length > 0))
throw new InvalidOperationException("Invalid request. One of the content item has content type set as 'text' but image URL is provided. Only one should be set in single content item.");
if (textTypeItems.Any(x => string.IsNullOrWhiteSpace(x.Text)))
throw new InvalidOperationException("Invalid request. One of the content item has content type set as 'text' but 'text' value is not set for this content item.");

//validate 'image_url' type of content items.
var imageTypeItems = promptContentItems.Where(x => x.Type == ChatGptMessageContentType.IMAGE).ToList();
if (imageTypeItems.Any(x => x.Text is not null && x.Text.Length > 0))
throw new InvalidOperationException("Invalid operation. One of the content item has message content type set as 'image' but text value is provided. Only one should be set in single content item.");
if (imageTypeItems.Any(x => string.IsNullOrWhiteSpace(x.GetImageUrl)))
throw new InvalidOperationException("Invalid operation. One of the content item has content type set as 'image' but 'imageUrl' value is not set for this content item.");
}
}
Loading

0 comments on commit 7087c2f

Please sign in to comment.