Add experimental shaders

.unimportedrc.json

@@ -9,7 +9,10 @@
     "**/__tests__/**",
     "**/*.d.ts"
   ],
-  "moduleDirectory": ["node_modules", "src/"],
+  "moduleDirectory": [
+    "node_modules",
+    "src/"
+  ],
   "rootDir": "./src",
   "ignoreUnimported": [
     "src/Scenes/Game/Singing/Input/MicStrategies/Yin.ts",
@@ -22,6 +25,13 @@
     "src/utils/testUtils.ts"
   ],
   "ignoreUnused": [],
-  "ignoreUnresolved": ["utility-types"],
-  "entry": ["src/main.tsx", "src/videos/index.ts"]
-}
+  "ignoreUnresolved": [
+      "Scenes/Game/Singing/GameOverlay/Drawing/Shaders/shader.frag?raw",
+      "Scenes/Game/Singing/GameOverlay/Drawing/Shaders/shader.vert?raw",
+    "utility-types"
+  ],
+  "entry": [
+    "src/main.tsx",
+    "src/videos/index.ts"
+  ]
+}

package.json

@@ -14,6 +14,7 @@
     "@uriopass/nosleep.js": "^0.12.2",
     "aubiojs": "0.1.0",
     "core-js": "^3.33.0",
+    "curtainsjs": "^8.1.5",
     "date-fns": "^2.30.0",
     "howler": "^2.2.4",
     "latinize": "^0.5.0",
@@ -98,6 +99,7 @@
     "@storybook/testing-library": "^0.2.2",
     "@testing-library/react": "^14.0.0",
     "@testing-library/user-event": "^14.5.1",
+    "@types/curtainsjs": "^8.1.1",
     "@types/dom-screen-wake-lock": "^1.0.1",
     "@types/howler": "^2.2.9",
     "@types/latinize": "^0.2.16",

src/Scenes/Game/Singing/GameOverlay/Drawing/CanvasDrawing.ts

@@ -1,9 +1,12 @@
+import { captureException } from '@sentry/react';
 import events from 'GameEvents/GameEvents';
 import PlayersManager from 'Players/PlayersManager';
 import getNoteColor from 'Scenes/Game/Singing/GameOverlay/Drawing/Elements/utils/getNoteColor';
 import SungTriangle from 'Scenes/Game/Singing/GameOverlay/Drawing/Particles/SungTriangle';
+import { Shaders } from 'Scenes/Game/Singing/GameOverlay/Drawing/Shaders/Shaders';
 import { FPSCountSetting, GraphicSetting } from 'Scenes/Settings/SettingsState';
 import isNotesSection from 'Songs/utils/isNotesSection';
+import { getFirstNoteFromSection } from 'Songs/utils/notesSelectors';
 import { noDistanceNoteTypes } from 'consts';
 import { Note, NotesSection, PlayerNote } from 'interfaces';
 import { randomFloat } from 'utils/randomValue';
@@ -20,6 +23,7 @@ import RayParticle from './Particles/Ray';
 import VibratoParticle from './Particles/Vibrato';
 import calculateData, { BIG_NOTE_HEIGHT, DrawingData, NOTE_HEIGHT, pitchPadding } from './calculateData';
 
+const MAX_LOOKUP_RANGE = 20;
 function getPlayerNoteAtBeat(playerNotes: PlayerNote[], beat: number) {
   return playerNotes.find((note) => note.start <= beat && note.start + note.length >= beat);
 }
@@ -34,11 +38,21 @@ export default class CanvasDrawing {
     private verticalMargin: number = 0,
     private scaleFactor: number = 1,
   ) {}
+
+  private shaders: Shaders | null = null;
   public start = () => {
     events.sectionChange.subscribe(this.onSectionEnd);
     this.loop = true;
 
     this.drawFrame();
+
+    if (GraphicSetting.get() === 'high') {
+      try {
+        this.shaders = new Shaders(this.canvas);
+      } catch (e) {
+        captureException(e);
+      }
+    }
   };
 
   public pause = () => {
@@ -51,6 +65,13 @@ export default class CanvasDrawing {
     this.start();
   };
 
+  public end = () => {
+    this.loop = false;
+
+    this.shaders?.cleanup();
+    events.sectionChange.unsubscribe(this.onSectionEnd);
+  };
+
   public isPlaying = () => this.loop;
 
   public drawFrame = () => {
@@ -80,11 +101,6 @@ export default class CanvasDrawing {
     }
   };
 
-  public end = () => {
-    this.loop = false;
-    events.sectionChange.unsubscribe(this.onSectionEnd);
-  };
-
   private drawPlayer = (playerNumber: number, ctx: CanvasRenderingContext2D) => {
     const drawingData = this.getDrawingData(playerNumber);
     const { currentSection } = calculateData(drawingData);
@@ -94,6 +110,9 @@ export default class CanvasDrawing {
 
     this.drawNotesToSing(ctx, drawingData, displacements);
     this.drawSungNotes(ctx, drawingData, displacements);
+    if (GraphicSetting.get() === 'high') {
+      this.setDistortionForce(drawingData);
+    }
 
     if (GraphicSetting.get() === 'high') {
       this.drawFlare(ctx, drawingData, displacements);
@@ -102,6 +121,20 @@ export default class CanvasDrawing {
     false && debugPitches(ctx!, drawingData);
   };
 
+  private setDistortionForce = (drawingData: DrawingData) => {
+    const currentSection = drawingData.currentSection;
+    if (!isNotesSection(currentSection)) return;
+    const max = Math.max(getFirstNoteFromSection(currentSection).start, drawingData.currentBeat - MAX_LOOKUP_RANGE);
+
+    const sungNotes = drawingData.currentPlayerNotes
+      .filter((note) => note.note.start + note.note.length >= max)
+      .filter((note) => getPlayerNoteDistance(note) === 0)
+      .reduce((curr, note) => curr + note.length - Math.max(0, max - note.start), 0);
+
+    const forcePercent = Math.min(Math.pow(sungNotes / MAX_LOOKUP_RANGE, 3), 0.99);
+    this.shaders?.updatePlayerForce(drawingData.playerNumber, forcePercent);
+  };
+
   private drawNotesToSing = (
     ctx: CanvasRenderingContext2D,
     drawingData: DrawingData,
@@ -183,6 +216,8 @@ export default class CanvasDrawing {
           getNoteColor(ctx, drawingData.playerNumber, true, lastNote).fill,
         ),
       );
+
+      this.shaders?.updatePlayerCenter(drawingData.playerNumber, finalX, finalY);
     }
   };
 

src/Scenes/Game/Singing/GameOverlay/Drawing/Elements/ray.ts

@@ -8,7 +8,7 @@ const getColors = (color: string) => {
     const c = tinycolor(color);
     cachedColors[color] = [
       [c.setAlpha(0.8).toRgbString(), c.setAlpha(0.9).toRgbString(), c.setAlpha(1).toRgbString()],
-      ['rgba(255, 255, 255, 0)'],
+      [c.setAlpha(0).toRgbString()],
       [c.setAlpha(0.1).toRgbString(), c.setAlpha(0.3).toRgbString(), c.setAlpha(0.2).toRgbString()],
     ];
   }
@@ -26,6 +26,8 @@ export default function ray(
   color: string | any,
   alpha: number = 1,
 ) {
+  if (width <= 0) return;
+
   ctx.globalAlpha = 1;
   const gradient = ctx.createRadialGradient(x, y, 0, x, y, width / 2);
   const [cFirst, cLast, c0] = getColors(color);

src/Scenes/Game/Singing/GameOverlay/Drawing/Shaders/Shaders.ts

@@ -0,0 +1,78 @@
+import GameState from 'Scenes/Game/Singing/GameState/GameState';
+import { Curtains, Plane, PlaneParams } from 'curtainsjs';
+
+export class Shaders {
+  private curtains: Curtains;
+  private plane: Plane;
+  public constructor(private canvas: HTMLCanvasElement) {
+    this.curtains = new Curtains({
+      container: 'canvas',
+    });
+    const { gl } = this.curtains;
+    // https://stackoverflow.com/a/39354174
+    gl.enable(gl.BLEND);
+    gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
+
+    const planeElement = document.getElementById('plane')!;
+    const planeDimensions = planeElement.getBoundingClientRect();
+
+    const playerUniforms: Record<string, any> = GameState.getPlayers().reduce(
+      (uniforms, player) => ({
+        ...uniforms,
+        [`p${player.getNumber()}center`]: {
+          name: `uP${player.getNumber()}center`,
+          type: '2f',
+          value: [0, 0],
+        },
+        [`p${player.getNumber()}force`]: {
+          name: `uP${player.getNumber()}force`,
+          type: '1f',
+          value: 0,
+        },
+      }),
+      {
+        resolution: {
+          // resolution of our plane
+          name: 'uResolution',
+          type: '2f',
+          value: [planeDimensions.width, planeDimensions.height],
+        },
+        time: {
+          name: 'uTime', // uniform name that will be passed to our shaders
+          type: '1f', // this means our uniform is a float
+          value: 0,
+        },
+      },
+    );
+
+    // set our initial parameters (basic uniforms)
+    const params: PlaneParams = {
+      vertexShaderID: 'plane-vs', // our vertex shader ID
+      fragmentShaderID: 'plane-fs', // our fragment shader ID
+      uniforms: playerUniforms,
+    };
+
+    // create our plane using our curtains object, the HTML element and the parameters
+    this.plane = new Plane(this.curtains, planeElement, params);
+
+    this.plane.loadCanvas(this.canvas);
+
+    this.plane.onRender(() => {
+      // @ts-expect-error
+      this.plane!.uniforms.time.value++;
+    });
+  }
+
+  public updatePlayerCenter = (playerNumber: number, x: number, y: number) => {
+    this.plane.uniforms[`p${playerNumber}center`].value = [x / this.canvas.width, 1 - y / this.canvas.height];
+  };
+
+  public updatePlayerForce = (playerNumber: number, force: number) => {
+    this.plane.uniforms[`p${playerNumber}force`].value = force;
+  };
+
+  public cleanup = () => {
+    this.plane?.remove();
+    this.curtains?.dispose();
+  };
+}

src/Scenes/Game/Singing/GameOverlay/Drawing/Shaders/shader.frag

@@ -0,0 +1,55 @@
+precision mediump float;
+
+float MAX_FORCE = 300.0;
+// get our varyings
+varying vec3 vVertexPosition;
+varying vec2 vTextureCoord;
+// the uniform we declared inside our javascript
+uniform float uTime;
+uniform sampler2D planeTexture;
+uniform vec2 uResolution;
+
+uniform vec2 uP0center;
+uniform float uP0force;
+
+uniform vec2 uP1center;
+uniform float uP1force;
+
+float rnd(float n) {
+    return fract(sin(n) * 43758.5453);
+}
+float random (vec2 st) {
+    return rnd(dot(st.xy, vec2(12.9898, 78.233)));
+}
+
+void distort(vec2 center, float force, float zoom, float noise) {
+    float actualForce = force * MAX_FORCE;
+    vec2 centerAbs = center * uResolution;
+    vec2 diff = (gl_FragCoord.xy - centerAbs.xy);
+    float dist = length(diff);
+
+    vec4 newColor = texture2D(planeTexture, (centerAbs + (diff * zoom) + (noise * 50.0 * (1.0 - pow(zoom, 12.0)))) / uResolution);
+    gl_FragColor = newColor;
+
+}
+
+float calculateZoom(vec2 center, float force) {
+    float actualForce = force * MAX_FORCE;
+    vec2 centerAbs = center * uResolution;
+    vec2 diff = (gl_FragCoord.xy - centerAbs.xy);
+    float dist = length(diff);
+
+    return pow(smoothstep(0.0, actualForce, dist), 1.0/6.0);
+}
+
+void main() {
+    float p0zoom = calculateZoom(uP0center, uP0force);
+    float p1zoom = calculateZoom(uP1center, uP1force);
+    float noise = (random(gl_FragCoord.xy)) / 2.0 - 0.5;
+
+    if (p0zoom < p1zoom) {
+        distort(uP0center, uP0force, p0zoom, noise);
+    } else {
+        distort(uP1center, uP1force, p1zoom, noise);
+    }
+}

src/Scenes/Game/Singing/GameOverlay/Drawing/Shaders/shader.vert

@@ -0,0 +1,23 @@
+
+precision mediump float;
+
+// default mandatory variables
+attribute vec3 aVertexPosition;
+attribute vec2 aTextureCoord;
+
+uniform mat4 uMVMatrix;
+uniform mat4 uPMatrix;
+
+uniform mat4 planeTextureMatrix;
+
+// custom varyings
+varying vec3 vVertexPosition;
+varying vec2 vTextureCoord;
+
+void main() {
+    gl_Position = uPMatrix * uMVMatrix * vec4(aVertexPosition, 1.0);
+
+    // varyings
+    vVertexPosition = aVertexPosition;
+    vTextureCoord = (planeTextureMatrix * vec4(aTextureCoord, 0.0, 1.0)).xy;
+}