Add translation from images to DeepL

Naamloos · Dec 11, 2023 · 0ab6392 · 0ab6392
1 parent 7448f9b
commit 0ab6392
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 17 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "ModCore/tessdata_fast"]
+	path = ModCore/tessdata_fast
+	url = https://github.com/tesseract-ocr/tessdata_fast
diff --git a/ModCore/ContextMenu/MessageContextMenu.cs b/ModCore/ContextMenu/MessageContextMenu.cs
@@ -13,6 +13,8 @@
 using System;
 using System.Reflection;
 using ModCore.Entities;
+using Tesseract;
+using DeepL.Model;
 
 namespace ModCore.ContextMenu
 {
@@ -48,7 +50,7 @@ public async Task YoinkAsync(ContextMenuContext ctx)
                 var match = matches.First();
                 DiscordMessage followup = null;
 
-                if(matches.Count() > 1)
+                if (matches.Count() > 1)
                 {
                     var message = new DiscordFollowupMessageBuilder()
                         .WithContent("Which emoji do you want to copy?")
@@ -61,7 +63,7 @@ public async Task YoinkAsync(ContextMenuContext ctx)
                     }
 
                     var selection = await ctx.MakeChoiceAsync<Match>(message, choices);
-                    if(selection.TimedOut)
+                    if (selection.TimedOut)
                     {
                         await ctx.EditFollowupAsync(selection.FollowupMessage.Id, new DiscordWebhookBuilder().WithContent("⚠️ Timed out selection."));
                         return;
@@ -86,7 +88,7 @@ public async Task YoinkAsync(ContextMenuContext ctx)
                     {
                         await ctx.Interaction.DeleteOriginalResponseAsync();
                         await ctx.Channel.SendMessageAsync(new DiscordMessageBuilder().WithContent(response).WithReply(ctx.TargetMessage.Id, false, false));
-                        
+
                         return;
                     }
                 }
@@ -187,41 +189,77 @@ private async Task<DiscordMessageSticker> stealieSticker(DiscordGuild guild, Dis
         [GuildOnly]
         public async Task TranslateAsync(ContextMenuContext ctx)
         {
-            if(string.IsNullOrEmpty(settings.DeepLToken))
+            if (string.IsNullOrEmpty(settings.DeepLToken))
             {
                 await ctx.CreateResponseAsync("No DeepL token configured! Notify the bot developers via /contact!", true);
                 return;
             }
 
+            await ctx.DeferAsync(false);
+
             var translate = ctx.TargetMessage.Content;
-            var translator = new Translator(settings.DeepLToken);
 
-            if(string.IsNullOrEmpty(translate))
+            var translateImage = "";
+
+            if (ctx.TargetMessage.Attachments.Count() > 0)
             {
-                await ctx.CreateResponseAsync("That message has no content!", true);
-                return;
+                try
+                {
+                    using var engine = new TesseractEngine(@"./tessdata_fast", "eng+jpn+rus+jpn_vert", EngineMode.Default);
+                    using var http = new HttpClient();
+
+                    var img = await http.GetAsync(ctx.TargetMessage.Attachments[0].Url);
+                    using var pix = Pix.LoadFromMemory(await img.Content.ReadAsByteArrayAsync());
+                    using var page = engine.Process(pix);
+
+                    translateImage = page.GetText();
+                }
+                catch (Exception ex)
+                {
+
+                }
             }
 
-            await ctx.DeferAsync(false);
+            if (string.IsNullOrEmpty(translate) && string.IsNullOrEmpty(translateImage))
+            {
+                await ctx.FollowUpAsync(new DiscordFollowupMessageBuilder().WithContent("Neither text or images with text were found on this message!"));
+                return;
+            }
 
+            using var translator = new Translator(settings.DeepLToken);
             try
             {
                 var assembly = Assembly.GetExecutingAssembly();
                 var resource = "ModCore.Assets.globe-showing-europe.gif";
                 var iconAsset = assembly.GetManifestResourceStream(resource);
 
-                var translation = await translator.TranslateTextAsync(translate, null, LanguageCode.EnglishAmerican);
-                await ctx.FollowUpAsync(new DiscordFollowupMessageBuilder()
-                    .AddEmbed(new DiscordEmbedBuilder()
-                        .WithDescription($"Translated using DeepL from language: {translation.DetectedSourceLanguageCode} to {LanguageCode.EnglishAmerican}.")
-                        .AddField("Original Text", translate)
-                        .AddField("Translated Text", translation.Text)
+                TextResult translation = string.IsNullOrEmpty(translate) ? null : await translator.TranslateTextAsync(translate, null, LanguageCode.EnglishAmerican);
+                TextResult imageTranslation = string.IsNullOrEmpty(translateImage)? null : await translator.TranslateTextAsync(translateImage, null, LanguageCode.EnglishAmerican);
+
+                var embed = new DiscordEmbedBuilder()
+                        .WithDescription($"Translated using [DeepL](https://www.deepl.com/translator).")
                         .WithColor(new DiscordColor("09a0e2"))
-                        .WithThumbnail("attachment://globe-showing-europe.gif"))
+                        .WithAuthor(name: "Translation", iconUrl: "attachment://globe-showing-europe.gif");
+
+                if(translation is not null)
+                {
+                    embed.AddField($"Original Message Text ({translation.DetectedSourceLanguageCode})", translate);
+                    embed.AddField($"Translated Message Text ({LanguageCode.EnglishAmerican})", translation.Text);
+                }
+
+                if (imageTranslation is not null)
+                {
+                    embed.AddField($"Original Image Text ({imageTranslation.DetectedSourceLanguageCode})", translateImage);
+                    embed.AddField($"Translated Image Text ({LanguageCode.EnglishAmerican})", imageTranslation.Text);
+                    embed.WithFooter("Please note that translations from images may not be super accurate due to the fact that text recognition is prone to mistakes.");
+                }
+
+                await ctx.FollowUpAsync(new DiscordFollowupMessageBuilder()
+                    .AddEmbed(embed)
                     .AddComponents(new DiscordLinkButtonComponent(ctx.TargetMessage.JumpLink.ToString(), "Jump to Original", emoji: new DiscordComponentEmoji("💭")))
                     .AddFile("globe-showing-europe.gif", iconAsset));
             }
-            catch(Exception ex)
+            catch (Exception ex)
             {
                 Console.WriteLine(ex);
             }

diff --git a/ModCore/ModCore.csproj b/ModCore/ModCore.csproj
@@ -18,6 +18,11 @@
   <ItemGroup>
     <EmbeddedResource Include="Assets\globe-showing-europe.gif" />
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="tessdata_fast\**">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Chronic-netstandard" Version="0.3.2.4" />
     <PackageReference Include="DeepL.net" Version="1.8.0" />
@@ -41,6 +46,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="System.Drawing.Common" Version="7.0.0" />
+    <PackageReference Include="Tesseract" Version="5.2.0" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="Migrations\" />

diff --git a/ModCore/tessdata_fast b/ModCore/tessdata_fast