Merge pull request lavalink-devs#264 from Devoxin/filters

Working filter stuff

IMPLEMENTATION.md

@@ -6,6 +6,11 @@ The Java client has support for JDA, but can also be adapted to work with other
 * You must be able to send messages via a shard's mainWS connection.
 * You must be able to intercept voice server updates from mainWS on your shard connection.
 
+## Significant changes v3.3 -> v3.4
+* Added filters
+* The `error` string on the `TrackExceptionEvent` has been deprecated and replaced by 
+the `exception` object following the same structure as the `LOAD_FAILED` error on [`/loadtracks`](#rest-api)
+
 ## Significant changes v2.0 -> v3.0 
 * The response of `/loadtracks` has been completely changed (again since the initial v3.0 pre-release).
 * Lavalink v3.0 now reports its version as a handshake response header.
@@ -116,35 +121,56 @@ The position is in milliseconds.
 }
 ```
 
-#### Set player volume
+#### Using filters
 
-Volume may range from 0 to 1000. 100 is default.
+The `filters` op sets the filters. All the filters are optional, and leaving them out of this message will disable them.
 
-```json
-{
-    "op": "volume",
-    "guildId": "...",
-    "volume": 125
-}
-```
-
-#### Using the player equalizer
+Adding a filter can have adverse effects on performance. These filters force Lavaplayer to decode all audio to PCM,
+even if the input was already in the Opus format that Discord uses. This means decoding and encoding audio that would
+normally require very little processing. This is often the case with YouTube videos.
 
-There are 15 bands (0-14) that can be changed.
-`gain` is the multiplier for the given band. The default value is 0. Valid values range from -0.25 to 1.0,
-where -0.25 means the given band is completely muted, and 0.25 means it is doubled. Modifying the gain could
-also change the volume of the output.
+JSON comments are for illustration purposes only, and will not be accepted by the server.
 
-```json
+```yaml
 {
-    "op": "equalizer",
+    "op": "filters",
     "guildId": "...",
-    "bands": [
+
+    // Float value where 1.0 is 100%. Values >1.0 may cause clipping
+    "volume": 1.0,
+
+    // There are 15 bands (0-14) that can be changed.
+    // "gain" is the multiplier for the given band. The default value is 0. Valid values range from -0.25 to 1.0,
+    // where -0.25 means the given band is completely muted, and 0.25 means it is doubled. Modifying the gain could
+    // also change the volume of the output.
+    "equalizer": [
         {
             "band": 0,
             "gain": 0.2
         }
-    ]
+    ],
+
+    // Uses equalization to eliminate part of a band, usually targeting vocals.
+    karaoke: {
+        "level": 1.0,
+        "monoLevel": 1.0,
+        "filterBand": 220.0,
+        "filterWidth": 100.0
+    },
+
+    // Changes the speed, pitch, and rate. All default to 1.
+    timescale: {
+        "speed": 1.0,
+        "pitch": 1.0,
+        "rate": 1.0
+    },
+
+    // Uses amplification to create a shuddering effect, where the volume quickly oscillates.
+    // Example: https://en.wikipedia.org/wiki/File:Fuse_Electronics_Tremolo_MK-III_Quick_Demo.ogv
+    tremolo: {
+        frequency: 2.0, // 0 < x
+        depth: 0.5      // 0 < x ≤ 1
+    }
 }
 ```
 
@@ -246,9 +272,14 @@ private void handleEvent(JSONObject json) throws IOException {
             );
             break;
         case "TrackExceptionEvent":
+            JSONObject jsonEx = json.getJSONObject("exception");
             event = new TrackExceptionEvent(player,
                     LavalinkUtil.toAudioTrack(json.getString("track")),
-                    new RemoteTrackException(json.getString("error"))
+                    new FriendlyException(
+                        jsonEx.getString("message"),
+                        FriendlyException.Severity.valueOf(jsonEx.getString("severity")),
+                        new RuntimeException(jsonEx.getString("cause"))
+                    )
             );
             break;
         case "TrackStuckEvent":

LavalinkServer/build.gradle

@@ -41,13 +41,15 @@ dependencies {
     compile group: 'moe.kyokobot.koe', name: 'ext-udpqueue', version: koeVersion
     compile group: 'com.sedmelluq', name: 'lavaplayer', version: lavaplayerVersion
     compile group: 'com.sedmelluq', name: 'lavaplayer-ext-youtube-rotator', version: lavaplayerIpRotatorVersion
+    compile group: 'com.github.natanbc', name: 'lavadsp', version: lavaDspVersion
     compile group: 'org.jetbrains.kotlin', name: 'kotlin-reflect', version: kotlinVersion
 
     compile group: 'org.springframework', name: 'spring-websocket', version: springWebSocketVersion
     compile group: 'ch.qos.logback', name: 'logback-classic', version: logbackVersion
     compile group: 'io.sentry', name: 'sentry-logback', version: sentryLogbackVersion
     compile group: 'com.github.oshi', name: 'oshi-core', version: oshiVersion
     compile group: 'org.json', name: 'json', version: jsonOrgVersion
+    compile group: 'com.google.code.gson', name: 'gson', version: gsonVersion
     compile(group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: springBootVersion) {
         exclude group: 'org.springframework.boot', module: 'spring-boot-starter-tomcat'
     }

LavalinkServer/src/main/java/lavalink/server/config/SentryConfiguration.java

@@ -31,6 +31,7 @@ public SentryConfiguration(ServerConfig serverConfig, SentryConfigProperties sen
         boolean warnDeprecatedDsnConfig = false;
         if (dsn == null || dsn.isEmpty()) {
             //try deprecated config location
+            //noinspection deprecation
             dsn = serverConfig.getSentryDsn();
             warnDeprecatedDsnConfig = true;
         }

LavalinkServer/src/main/java/lavalink/server/io/HandshakeInterceptorImpl.kt

@@ -1,7 +1,6 @@
 package lavalink.server.io
 
 import lavalink.server.config.ServerConfig
-import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.http.HttpStatus
@@ -10,7 +9,6 @@ import org.springframework.http.server.ServerHttpResponse
 import org.springframework.stereotype.Controller
 import org.springframework.web.socket.WebSocketHandler
 import org.springframework.web.socket.server.HandshakeInterceptor
-import java.util.Objects
 
 @Controller
 class HandshakeInterceptorImpl @Autowired

LavalinkServer/src/main/java/lavalink/server/io/SocketServer.kt

@@ -34,7 +34,6 @@ import org.springframework.web.socket.CloseStatus
 import org.springframework.web.socket.TextMessage
 import org.springframework.web.socket.WebSocketSession
 import org.springframework.web.socket.handler.TextWebSocketHandler
-import java.util.*
 import java.util.concurrent.ConcurrentHashMap
 
 @Service
@@ -157,6 +156,7 @@ class SocketServer(
             "destroy"           -> handlers.destroy(context, json)
             "configureResuming" -> handlers.configureResuming(context, json)
             "equalizer"         -> handlers.equalizer(context, json)
+            "filters"           -> handlers.filters(context, json.getString("guildId"), message.payload)
             else                -> log.warn("Unexpected operation: " + json.getString("op"))
             // @formatter:on
         }

LavalinkServer/src/main/java/lavalink/server/io/WebSocketHandlers.kt

@@ -1,5 +1,7 @@
 package lavalink.server.io
 
+import lavalink.server.player.filters.Band
+import lavalink.server.player.filters.FilterChain
 import com.sedmelluq.discord.lavaplayer.track.TrackMarker
 import lavalink.server.player.TrackEndMarkerHandler
 import lavalink.server.util.Util
@@ -14,6 +16,9 @@ class WebSocketHandlers(private val contextMap: Map<String, SocketContext>) {
         private val log: Logger = LoggerFactory.getLogger(WebSocketHandlers::class.java)
     }
 
+    private var loggedVolumeDeprecationWarning = false
+    private var loggedEqualizerDeprecationWarning = false
+
     fun voiceUpdate(context: SocketContext, json: JSONObject) {
         val sessionId = json.getString("sessionId")
         val guildId = json.getLong("guildId")
@@ -45,7 +50,13 @@ class WebSocketHandlers(private val contextMap: Map<String, SocketContext>) {
 
         player.setPause(json.optBoolean("pause", false))
         if (json.has("volume")) {
-            player.setVolume(json.getInt("volume"))
+            if(!loggedVolumeDeprecationWarning) log.warn("The volume property in the play operation has been deprecated" +
+                    "and will be removed in v4. Please configure a filter instead. Note that the new filter takes a " +
+                    "float value with 1.0 being 100%")
+            loggedVolumeDeprecationWarning = true
+            val filters = player.filters ?: FilterChain()
+            filters.volume = json.getFloat("volume") / 100
+            player.filters = filters
         }
 
         if (json.has("endTime")) {
@@ -86,13 +97,25 @@ class WebSocketHandlers(private val contextMap: Map<String, SocketContext>) {
     }
 
     fun equalizer(context: SocketContext, json: JSONObject) {
+        if (!loggedEqualizerDeprecationWarning) log.warn("The 'equalizer' op has been deprecated in favour of the " +
+                "'filters' op. Please switch to use that one, as this op will get removed in v4.")
+        loggedEqualizerDeprecationWarning = true
+
         val player = context.getPlayer(json.getString("guildId"))
-        val bands = json.getJSONArray("bands")
 
-        for (i in 0 until bands.length()) {
-            val band = bands.getJSONObject(i)
-            player.setBandGain(band.getInt("band"), band.getFloat("gain"))
+        val list = mutableListOf<Band>()
+        json.getJSONArray("bands").forEach { b ->
+            val band = b as JSONObject
+            list.add(Band(band.getInt("band"), band.getFloat("gain")))
         }
+        val filters = player.filters ?: FilterChain()
+        filters.equalizer = list
+        player.filters = filters
+    }
+
+    fun filters(context: SocketContext, guildId: String, json: String) {
+        val player = context.getPlayer(guildId)
+        player.filters = FilterChain.parse(json)
     }
 
     fun destroy(context: SocketContext, json: JSONObject) {

LavalinkServer/src/main/java/lavalink/server/player/EventEmitter.java

@@ -94,6 +94,11 @@ public void onTrackException(AudioPlayer player, AudioTrack track, FriendlyExcep
         }
 
         out.put("error", exception.getMessage());
+        JSONObject exceptionJson = new JSONObject();
+        exceptionJson.put("message", exception.getMessage());
+        exceptionJson.put("severity", exception.severity.toString());
+        exceptionJson.put("cause", Util.getRootCause(exception).toString());
+        out.put("exception", exceptionJson);
 
         linkPlayer.getSocket().send(out);
     }

LavalinkServer/src/main/java/lavalink/server/player/Player.java

@@ -22,8 +22,6 @@
 
 package lavalink.server.player;
 
-import com.sedmelluq.discord.lavaplayer.filter.equalizer.Equalizer;
-import com.sedmelluq.discord.lavaplayer.filter.equalizer.EqualizerFactory;
 import com.sedmelluq.discord.lavaplayer.player.AudioPlayer;
 import com.sedmelluq.discord.lavaplayer.player.AudioPlayerManager;
 import com.sedmelluq.discord.lavaplayer.player.event.AudioEventAdapter;
@@ -33,6 +31,7 @@
 import io.netty.buffer.ByteBuf;
 import lavalink.server.io.SocketContext;
 import lavalink.server.io.SocketServer;
+import lavalink.server.player.filters.FilterChain;
 import lavalink.server.config.ServerConfig;
 import moe.kyokobot.koe.VoiceConnection;
 import moe.kyokobot.koe.media.OpusAudioFrameProvider;
@@ -54,9 +53,8 @@ public class Player extends AudioEventAdapter {
     private final AudioPlayer player;
     private final AudioLossCounter audioLossCounter = new AudioLossCounter();
     private AudioFrame lastFrame = null;
+    private FilterChain filters;
     private ScheduledFuture<?> myFuture = null;
-    private final EqualizerFactory equalizerFactory = new EqualizerFactory();
-    private boolean isEqualizerApplied = false;
 
     public Player(SocketContext socketContext, String guildId, AudioPlayerManager audioPlayerManager, ServerConfig serverConfig) {
         this.socketContext = socketContext;
@@ -97,33 +95,6 @@ public void setVolume(int volume) {
         player.setVolume(volume);
     }
 
-    public void setBandGain(int band, float gain) {
-        log.debug("Setting band {}'s gain to {}", band, gain);
-        equalizerFactory.setGain(band, gain);
-
-        if (gain == 0.0f) {
-            if (!isEqualizerApplied) {
-                return;
-            }
-
-            boolean shouldDisable = true;
-
-            for (int i = 0; i < Equalizer.BAND_COUNT; i++) {
-                if (equalizerFactory.getGain(i) != 0.0f) {
-                    shouldDisable = false;
-                }
-            }
-
-            if (shouldDisable) {
-                this.player.setFilterFactory(null);
-                this.isEqualizerApplied = false;
-            }
-        } else if (!this.isEqualizerApplied) {
-            this.player.setFilterFactory(equalizerFactory);
-            this.isEqualizerApplied = true;
-        }
-    }
-
     public JSONObject getState() {
         JSONObject json = new JSONObject();
 
@@ -203,4 +174,18 @@ public void retrieveOpusFrame(ByteBuf buf) {
         }
     }
 
+    @Nullable
+    public FilterChain getFilters() {
+        return filters;
+    }
+
+    public void setFilters(FilterChain filters) {
+        this.filters = filters;
+
+        if (filters.isEnabled()) {
+            player.setFilterFactory(filters);
+        } else {
+            player.setFilterFactory(null);
+        }
+    }
 }

LavalinkServer/src/main/java/lavalink/server/player/filters/FilterChain.kt

@@ -0,0 +1,50 @@
+package lavalink.server.player.filters
+
+import com.google.gson.Gson
+import com.sedmelluq.discord.lavaplayer.filter.*
+import com.sedmelluq.discord.lavaplayer.format.AudioDataFormat
+import com.sedmelluq.discord.lavaplayer.track.AudioTrack
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+class FilterChain : PcmFilterFactory {
+
+    companion object {
+        private val log: Logger = LoggerFactory.getLogger(FilterChain::class.java)
+        private val gson = Gson()
+        fun parse(json: String) = gson.fromJson(json, FilterChain::class.java)!!
+    }
+
+    var volume: Float? = null
+    var equalizer: List<Band>? = null
+    private val karaoke: KaraokeConfig? = null
+    private val timescale: TimescaleConfig? = null
+    private val tremolo: TremoloConfig? = null
+    private val vibrato: VibratoConfig? = null
+
+    private fun buildList() = listOfNotNull(
+            volume?.let { VolumeConfig(it) },
+            equalizer?.let { EqualizerConfig(it) },
+            karaoke,
+            timescale,
+            tremolo,
+            vibrato
+    )
+
+    val isEnabled get() = buildList().any { it.isEnabled }
+
+    override fun buildChain(track: AudioTrack?, format: AudioDataFormat, output: UniversalPcmAudioFilter): MutableList<AudioFilter> {
+        val enabledFilters = buildList().takeIf { it.isNotEmpty() }
+            ?: return mutableListOf()
+
+        val pipeline = mutableListOf<FloatPcmAudioFilter>()
+
+        for (filter in enabledFilters) {
+            val outputTo = pipeline.lastOrNull() ?: output
+            pipeline.add(filter.build(format, outputTo))
+        }
+
+        return pipeline.reversed().toMutableList() // Output last
+    }
+
+}