Skip to content

Commit

Permalink
Add screen reader
Browse files Browse the repository at this point in the history
  • Loading branch information
Platonymous committed Mar 14, 2021
1 parent fa457d6 commit da70794
Show file tree
Hide file tree
Showing 7 changed files with 246 additions and 21 deletions.
5 changes: 4 additions & 1 deletion PelicanTTS/ModConfig.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Collections.Generic;
using StardewModdingAPI;
using System.Collections.Generic;

namespace PelicanTTS
{
Expand All @@ -21,6 +22,8 @@ public class ModConfig

public int Rate { get; set; } = 100;

public SButton ReadScreenKey { get; set; } = SButton.N;

public Dictionary<string, VoiceSetup> Voices { get; set; } = new Dictionary<string, VoiceSetup>();
}
}
10 changes: 5 additions & 5 deletions PelicanTTS/PelicanTTS.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<PropertyGroup>
<AssemblyName>PelicanTTS</AssemblyName>
<RootNamespace>PelicanTTS</RootNamespace>
<Version>1.8.4</Version>
<Version>1.9.0-alpha</Version>
<TargetFramework>net452</TargetFramework>
<Platforms>x86</Platforms>
<PlatformTarget>x86</PlatformTarget>
Expand All @@ -12,12 +12,12 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="AWSSDK.Core" Version="3.5.0-beta" />
<PackageReference Include="AWSSDK.Polly" Version="3.5.0-beta" />
<PackageReference Include="Pathoschild.Stardew.ModBuildConfig" Version="3.1.0" />
<PackageReference Include="AWSSDK.Core" Version="3.5.3.6" />
<PackageReference Include="AWSSDK.Polly" Version="3.5.1.40" />
<PackageReference Include="Pathoschild.Stardew.ModBuildConfig" Version="3.2.2" />
<PackageReference Include="Platonymous.Ogg2XNA" Version="1.1.0" />
</ItemGroup>

<Import Project="$(SolutionDir)\common.targets" />

</Project>
42 changes: 38 additions & 4 deletions PelicanTTS/PelicanTTSMod.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,19 @@ public class PelicanTTSMod : Mod
internal static IManifest _manifest;
internal static ITranslationHelper i18n => _helper.Translation;
internal static Dictionary<LocalizedContentManager.LanguageCode, List<string>> voices = new Dictionary<LocalizedContentManager.LanguageCode, List<string>>();
internal static List<string> neural = new List<string>();
internal static string currentName = "Abigail";
internal static Dictionary<object, int> currentIndex = new Dictionary<object, int>();
internal static int maxValues = 5;
internal static Screengraber screenGraber;


public override void Entry(IModHelper helper)
{
_helper = helper;
_manifest = ModManifest;
config = Helper.ReadConfig<ModConfig>();

screenGraber = new Screengraber();
Helper.Events.GameLoop.OneSecondUpdateTicked += GameLoop_OneSecondUpdateTicked;

helper.ConsoleCommands.Add("tts_update", "Updates new NPCs", (s,p) => setUpNPCConfig());
Expand All @@ -47,14 +49,22 @@ public override void Entry(IModHelper helper)
helper.Events.GameLoop.SaveLoaded += OnSaveLoaded;
helper.Events.GameLoop.ReturnedToTitle += OnReturnedToTitle;

helper.Events.Input.ButtonPressed += Input_ButtonPressed;

neural.AddRange(new string[]{
"Salli","Joanna","Ivy","Kendra","Kimberly","Kevin","Matthew","Justin","Joey",
"Lupe","Olivia","Camila",
"Amy","Emma", "Brian",
});

voices.Add(LocalizedContentManager.LanguageCode.de, (new List<string>()
{
"Marlene","Vicki","Hans"
}).OrderBy(v => v).ToList());

voices.Add(LocalizedContentManager.LanguageCode.en, (new List<string>()
{
"Brian","Amy","Joey","Emma","Nicole","Justin","Russell","Matthew","Kendra","Salli","Kimberly","Geraint","Ivy","Raveena","Aditi"
"Brian","Amy","Joey","Emma","Nicole","Olivia","Justin","Russell","Joanna","Matthew","Kevin", "Kendra","Salli","Kimberly","Geraint","Ivy","Raveena","Aditi"
}).OrderBy(v => v).ToList());

voices.Add(LocalizedContentManager.LanguageCode.es, (new List<string>()
Expand Down Expand Up @@ -111,7 +121,24 @@ public override void Entry(IModHelper helper)
{
"Zhiyu"
}).OrderBy(v => v).ToList());


/*voices.Add(LocalizedContentManager.LanguageCode.hu, (new List<string>()
{
"None"
}).OrderBy(v => v).ToList());
voices.Add(LocalizedContentManager.LanguageCode.th, (new List<string>()
{
"None"
}).OrderBy(v => v).ToList());*/

}

private void Input_ButtonPressed(object sender, ButtonPressedEventArgs e)
{
if (e.Button == config.ReadScreenKey && Game1.getMousePosition() is Point p)
screenGraber.read(p);
}

private void GameLoop_OneSecondUpdateTicked(object sender, OneSecondUpdateTickedEventArgs e)
Expand Down Expand Up @@ -171,6 +198,7 @@ public void setUpConfig()
config.ReadHudMessages = true;
config.ReadLetters = true;
config.ReadNonCharacterMessages = true;
config.ReadScreenKey = SButton.N;
//config.ReadChatMessages = true;

var npcs = Helper.Content.Load<Dictionary<string, string>>("Data//NPCDispositions", ContentSource.GameContent);
Expand All @@ -190,7 +218,9 @@ public void setUpConfig()
api.RegisterSimpleOption(ModManifest, "Read Non-Character Messages", "", () => config.ReadNonCharacterMessages, (s) => config.ReadNonCharacterMessages = s);
api.RegisterSimpleOption(ModManifest, "Read Letters", "", () => config.ReadLetters, (s) => config.ReadLetters = s);
api.RegisterSimpleOption(ModManifest, "Read Hud Messages", "", () => config.ReadHudMessages, (s) => config.ReadHudMessages = s);
// api.RegisterSimpleOption(ModManifest, "Read Chat Messages", "", () => config.ReadChatMessages, (s) => config.ReadChatMessages = s);
api.RegisterSimpleOption(ModManifest, "Read Screen Key", "", () => config.ReadScreenKey, (s) => config.ReadScreenKey = s);

// api.RegisterSimpleOption(ModManifest, "Read Chat Messages", "", () => config.ReadChatMessages, (s) => config.ReadChatMessages = s);

api.RegisterClampedOption(ModManifest, "Volume", "Set Volume", () =>
{
Expand Down Expand Up @@ -284,7 +314,11 @@ public void setUpConfig()
{
var dialogues = _helper.Content.Load<Dictionary<string, string>>(@"Characters/Dialogue/" + character, ContentSource.GameContent);
if (dialogues != null && dialogues.ContainsKey("Introduction"))
{
intro = dialogues["Introduction"].Split('^')[0].Split('#')[0].Replace("@", "");
if(intro.Length < 7)
intro = dialogues["Mon"].Split('^')[0].Split('#')[0].Replace("@", "");
}
}
SpeechHandlerPolly.configSay(character, value, intro, activeRate, mvs is MenuVoiceSetup ? mvs.Pitch : -1, activeVolume);
}
Expand Down
165 changes: 165 additions & 0 deletions PelicanTTS/Screengraber.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using Harmony;
using Microsoft.Xna.Framework;
using Microsoft.Xna.Framework.Graphics;
using StardewModdingAPI;
using StardewValley;
using StardewValley.BellsAndWhistles;
using StardewValley.Menus;

namespace PelicanTTS
{
internal class Screengraber
{
HarmonyInstance instance;
static bool shouldRead = false;
static Point target = Point.Zero;
static string capture = "";
static string capturedTitle = "";
static string capturedContent = "";
static bool readFullScreen = false;

public Screengraber()
{
instance = HarmonyInstance.Create("PelicanTTS.Screengrabber");
foreach (MethodInfo drawMethod in typeof(SpriteBatch).GetMethods().Where(m => m.GetParameters().Any(p => p.ParameterType == typeof(string) && p.Name == "text")))
if (drawMethod.GetParameters().FirstOrDefault(p => p.Name == "scale") is ParameterInfo pa)
if (pa.ParameterType == typeof(Vector2))
instance.Patch(drawMethod, new HarmonyMethod(AccessTools.Method(typeof(Screengraber), nameof(drawStringPatchN))));
else if (pa.ParameterType == typeof(float))
instance.Patch(drawMethod, new HarmonyMethod(AccessTools.Method(typeof(Screengraber), nameof(drawStringPatchF))));
else
instance.Patch(drawMethod, new HarmonyMethod(AccessTools.Method(typeof(Screengraber), nameof(drawStringPatchN))));
else
instance.Patch(drawMethod, new HarmonyMethod(AccessTools.Method(typeof(Screengraber), nameof(drawStringPatchN))));

instance.Patch(AccessTools.Method(typeof(SpriteText), nameof(SpriteText.drawString)), new HarmonyMethod(AccessTools.Method(typeof(Screengraber), nameof(drawFontPatch))));
instance.Patch(AccessTools.Method(typeof(IClickableMenu), nameof(IClickableMenu.drawToolTip)), new HarmonyMethod(AccessTools.Method(typeof(Screengraber), nameof(drawToolTip))));
if (typeof(IClickableMenu).GetMethods(BindingFlags.Static | BindingFlags.Public).Where(m => m.Name.Contains(nameof(IClickableMenu.drawHoverText)) && m.GetParameters().Any(p => p.ParameterType == typeof(string) && p.Name == "text")).FirstOrDefault() is MethodInfo hoverMethod)
instance.Patch(hoverMethod, new HarmonyMethod(AccessTools.Method(typeof(Screengraber), nameof(drawHoverText))));

if (Type.GetType("Pathoschild.Stardew.LookupAnything.DrawTextHelper, LookupAnything") is Type drawTextHelper)
{
foreach (MethodInfo laMethod in drawTextHelper.GetMethods(BindingFlags.Public | BindingFlags.Static))
if (laMethod.GetParameters().Any(p => p.ParameterType == typeof(string) && p.Name == "text"))
instance.Patch(laMethod, new HarmonyMethod(AccessTools.Method(typeof(Screengraber), nameof(drawTextBlock))));
else
instance.Patch(laMethod, new HarmonyMethod(AccessTools.Method(typeof(Screengraber), nameof(drawTextBlock2))));
}
}

public static void drawTextBlock(string text, Vector2 position, SpriteFont font, float scale)
{
drawStringPatchF(text, position, font, scale);
}

public static void drawTextBlock2(IEnumerable<object> text, Vector2 position, SpriteFont font, float scale)
{
drawStringPatchF(string.Join(" ",text.Select(t => t.GetType().GetProperty("Text").GetValue(t))), position, font, scale);
}

public static void drawToolTip(string hoverText, string hoverTitle)
{
capturedTitle = hoverTitle;
capturedContent = hoverText;
}
public static void drawHoverText(string text)
{
capturedContent = text;
}


public void read(Point position)
{
capture = null;
target = position;
capturedTitle = "";
capturedContent = "";
PelicanTTSMod._helper.Events.GameLoop.UpdateTicked += GameLoop_UpdateTicked;
}

public void readFull()
{
readFullScreen = true;
capture = null;
PelicanTTSMod._helper.Events.GameLoop.UpdateTicked += GameLoop_UpdateTicked;
}

private void GameLoop_UpdateTicked(object sender, StardewModdingAPI.Events.UpdateTickedEventArgs e)
{
PelicanTTSMod._helper.Events.GameLoop.UpdateTicked -= GameLoop_UpdateTicked;
PelicanTTSMod._helper.Events.Display.Rendering += Display_Rendering;
}

private void Display_Rendering(object sender, StardewModdingAPI.Events.RenderingEventArgs e)
{
shouldRead = true;
PelicanTTSMod._helper.Events.Display.Rendering -= Display_Rendering;
PelicanTTSMod._helper.Events.Display.Rendered += Display_Rendered;
}

private void Display_Rendered(object sender, StardewModdingAPI.Events.RenderedEventArgs e)
{
PelicanTTSMod._helper.Events.Display.Rendered -= Display_Rendered;
shouldRead = false;

if (capture == null || capture.Length == 0)
if (capturedTitle.Length > 0)
capture = capturedTitle + ": " + capturedContent;
else
capture = capturedContent;

if (capture != null && capture != "")
SpeechHandlerPolly.configSay("Default", PelicanTTSMod.config.Voices["Default"]?.Voice ?? "Salli", capture);
}

public static Rectangle TextBounds(string text, SpriteFont spriteFont, Vector2 scale, Vector2 position)
{
var p = spriteFont.MeasureString(text);
return new Rectangle((int)position.X, (int)position.Y, (int)(p.X * scale.X), (int)(p.Y * scale.Y));
}

public static void drawStringPatchN(string text, Vector2 position, SpriteFont spriteFont)
{
drawStringPatch(text, position, spriteFont, Vector2.One);
}


public static void drawStringPatchF(string text, Vector2 position, SpriteFont spriteFont, float scale)
{
drawStringPatch(text, position, spriteFont, new Vector2(scale, scale));
}

public static void drawStringPatchV(string text, Vector2 position, SpriteFont spriteFont, Vector2 scale)
{
drawStringPatch(text, position, spriteFont, scale);
}

public static void drawStringPatch(string text, Vector2 position, SpriteFont spriteFont, Vector2 scale)
{
if (shouldRead && (readFullScreen || TextBounds(text, spriteFont, scale, position).Contains(target)))
{
capture = text;
shouldRead = false;
}
}

public static void drawFontPatch(string s, int x, int y)
{
int width = SpriteText.getWidthOfString(s);
int height = SpriteText.getHeightOfString(s);

if (shouldRead && (readFullScreen || new Rectangle(x, y, width, height).Contains(target)))
{
capture = s;
shouldRead = false;
}
}

}
}
Loading

0 comments on commit da70794

Please sign in to comment.