Bot - add support for GPT-4 Vision

Marfusios · Dec 3, 2023 · f8b8f15 · f8b8f15
1 parent bbd41fe
commit f8b8f15
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 1 deletion.
diff --git a/apps/nostr-bot/NostrBot.Web/Configs/OpenAiConfig.cs b/apps/nostr-bot/NostrBot.Web/Configs/OpenAiConfig.cs
@@ -6,6 +6,7 @@ public class OpenAiConfig
  public string Organization { get; init; } = null!;
 
  public string Model { get; init; } = "gpt-3.5-turbo";
+ public bool ModelSupportsVision { get; init; }
  public string ImageModel { get; init; } = "dall-e-3";
  public int ImageCount { get; init; } = 1;
  public string ImageQuality { get; init; } = "hd";

diff --git a/apps/nostr-bot/NostrBot.Web/Logic/BotMind.cs b/apps/nostr-bot/NostrBot.Web/Logic/BotMind.cs
@@ -1,4 +1,5 @@
 using System.Reactive.Linq;
+using System.Text.RegularExpressions;
 using Microsoft.Extensions.Options;
 using Nostr.Client.Client;
 using NostrBot.Web.Configs;
@@ -35,6 +36,10 @@ public class BotMind : BackgroundService
  private readonly NostrPrivateKey _botPrivateKey;
  private readonly NostrPublicKey _botPublicKey;
 
+ private readonly Regex _imageUrlRegex = new Regex(
+ @"\b(https?:\/\/\S+(?:png|webp|jpe?g|gif)\S*)\b",
+ RegexOptions.Multiline | RegexOptions.IgnoreCase);
+
  private CancellationToken? _stoppingToken;
 
  public BotMind(IOptions<NostrConfig> nostrConfig, IOptions<BotConfig> config, IOptions<OpenAiConfig> openAiConfig,
@@ -264,7 +269,17 @@ private async Task<string> RequestAiReply(NostrEventResponse response, string co
  chatPrompts.AddRange(IncludeBotDescription());
  chatPrompts.AddRange(IncludeBotWhois());
  chatPrompts.AddRange(await IncludeHistory(contextId, secondaryContextId, response));
- chatPrompts.Add(new Message(Role.User, $"@{ToNpub(response.Event?.Pubkey)}: {userMessageProcessed}"));
+
+ var content = new List<Content>();
+ var textContent = new Content($"@{ToNpub(response.Event?.Pubkey)}: {userMessageProcessed}");
+ content.Add(textContent);
+
+ if (_openAiConfig.ModelSupportsVision)
+ {
+ IncludeImages(content, userMessageProcessed);
+ }
+
+ chatPrompts.Add(new Message(Role.User, content));
 
  CircuitBreaker(chatPrompts);
 
@@ -434,6 +449,18 @@ private async Task LoadAdditionalHistory(string? pubkey, string primaryContext,
 
  events.AddRange(await _storage.GetHistoryForContext(context, null));
  }
+
+ private void IncludeImages(List<Content> content, string? message)
+ {
+ var messageSafe = message ?? string.Empty;
+ var match = _imageUrlRegex.Match(messageSafe);
+ if (!match.Success)
+ return;
+
+ var imageUrl = match.Value;
+ var imageContent = new Content(new ImageUrl(imageUrl));
+ content.Add(imageContent);
+ }
 
  private int CountTextTokens(string text)
  {