From a1b17ec29bea78d24b9a7ed7bf429147a2d62816 Mon Sep 17 00:00:00 2001 From: Niklas Fiekas Date: Tue, 28 Apr 2020 15:29:34 +0200 Subject: [PATCH] replace ipintel with ip2proxy --- COPYING.md | 1 - app/controllers/Mod.scala | 8 ++- conf/base.conf | 5 +- conf/routes | 2 +- modules/common/src/main/mon.scala | 5 +- modules/security/src/main/Env.scala | 8 +-- .../security/src/main/GarbageCollector.scala | 4 +- modules/security/src/main/Ip2Proxy.scala | 59 ++++++++++++++++++ modules/security/src/main/IpIntel.scala | 62 ------------------- modules/security/src/main/IpTrust.scala | 12 ++-- .../security/src/main/SecurityConfig.scala | 4 +- modules/security/src/main/Signup.scala | 4 +- ui/site/src/user-mod.js | 4 +- 13 files changed, 86 insertions(+), 92 deletions(-) create mode 100644 modules/security/src/main/Ip2Proxy.scala delete mode 100644 modules/security/src/main/IpIntel.scala diff --git a/COPYING.md b/COPYING.md index cb027997c4422..d9e814fa095d2 100644 --- a/COPYING.md +++ b/COPYING.md @@ -100,7 +100,6 @@ Lichess as deployed on https://lichess.org/ also uses these external services: * [stripe](https://stripe.com/) and [PayPal](https://www.paypal.com) for [Patron donations](https://lichess.org/patron) * [Spreadshirt](https://shop.spreadshirt.com/lichess-org) for the [Swag store](https://lichess.org/swag) * [check.torproject.org](https://check.torproject.org/exit-addresses) for a list or Tor exit nodes -* [getipintel.net](https://getipintel.net/) * [detectlanguage.com](https://detectlanguage.com/) * Fallback to [Google Fonts](https://fonts.google.com/) * [Google Cloud Messaging](https://developers.google.com/cloud-messaging/) and [OneSignal](https://onesignal.com/) for mobile notifications diff --git a/app/controllers/Mod.scala b/app/controllers/Mod.scala index fb1f7e7c56d88..4574a8980f1ed 100644 --- a/app/controllers/Mod.scala +++ b/app/controllers/Mod.scala @@ -12,7 +12,7 @@ import lila.chat.Chat import lila.common.{ EmailAddress, HTTPRequest, IpAddress } import lila.mod.UserSearch import lila.report.{ Suspect, Mod => AsMod } -import lila.security.{ FingerHash, IpIntel, Permission } +import lila.security.{ FingerHash, Ip2Proxy, Permission } import lila.user.{ User => UserModel, Title } import ornicar.scalalib.Zero import views._ @@ -268,8 +268,10 @@ final class Mod( def communicationPublic(username: String) = communications(username, false) def communicationPrivate(username: String) = communications(username, true) - def ipIntel(ip: String) = Secure(_.IpBan) { _ => _ => - env.security.ipIntel.failable(IpAddress(ip), IpIntel.Reason.UserMod).map { Ok(_) }.recover { + def ip2proxy(ip: String) = Secure(_.IpBan) { _ => _ => + env.security.ip2proxy.failable(IpAddress(ip), Ip2Proxy.Reason.UserMod).dmap { proxyType => + Ok(proxyType.value) + }.recover { case e: Exception => InternalServerError(e.getMessage) } } diff --git a/conf/base.conf b/conf/base.conf index 5b5bdef05d3c3..eafa99b511d32 100644 --- a/conf/base.conf +++ b/conf/base.conf @@ -223,9 +223,8 @@ security { } recaptcha = ${recaptcha} mailgun = ${mailgun} - ipintel { - url = "https://check.getipintel.net/check.php" - email = "" + ip2proxy { + url = "http://ip2proxy.lichess.ovh:1929" } lame_name_check = true } diff --git a/conf/routes b/conf/routes index 6c96e8d058c2f..f262568b8dc76 100644 --- a/conf/routes +++ b/conf/routes @@ -394,7 +394,7 @@ GET /mod/table controllers.Mod.table POST /mod/:username/refreshUserAssess controllers.Mod.refreshUserAssess(username: String) POST /mod/:username/email controllers.Mod.setEmail(username: String) POST /mod/:username/notify-slack controllers.Mod.notifySlack(username: String) -GET /mod/ip-intel controllers.Mod.ipIntel(ip: String) +GET /mod/ip2proxy controllers.Mod.ip2proxy(ip: String) GET /mod/leaderboard controllers.Mod.gamify GET /mod/leaderboard/:period controllers.Mod.gamifyPeriod(period: String) GET /mod/search controllers.Mod.search diff --git a/modules/common/src/main/mon.scala b/modules/common/src/main/mon.scala index 4c664bf82424f..69d6d11eb1c99 100644 --- a/modules/common/src/main/mon.scala +++ b/modules/common/src/main/mon.scala @@ -298,9 +298,8 @@ object mon { val prints = gauge("security.firewall.prints").withoutTags } object proxy { - def ipintel(reason: String) = counter("security.proxy.ipintel").withTag("reason", reason) - val request = future("security.proxy.time") - val percent = histogram("security.proxy.percent").withoutTags + def reason(reason: String) = counter("security.proxy.reason").withTag("reason", reason) + val request = future("security.proxy.time") } def rateLimit(key: String) = counter("security.rateLimit.count").withTag("key", key) def concurrencyLimit(key: String) = counter("security.concurrencyLimit.count").withTag("key", key) diff --git a/modules/security/src/main/Env.scala b/modules/security/src/main/Env.scala index 89fd0b43cc82b..98e3d47776c98 100644 --- a/modules/security/src/main/Env.scala +++ b/modules/security/src/main/Env.scala @@ -7,7 +7,7 @@ import play.api.libs.ws.WSClient import scala.concurrent.duration._ import lila.common.config._ -import lila.common.{ Bus, EmailAddress, Strings } +import lila.common.{ Bus, Strings } import lila.memo.SettingStore.Strings._ import lila.oauth.OAuthServer import lila.user.{ Authenticator, UserRepo } @@ -53,9 +53,9 @@ final class Env( lazy val store = new Store(db(config.collection.security), net.ip) - lazy val ipIntel = { - def mk = (url: String, email: EmailAddress) => wire[IpIntel] - mk(config.ipIntelUrl, config.ipIntelEmail) + lazy val ip2proxy = { + def mk = (url: String) => wire[Ip2Proxy] + mk(config.ip2ProxyUrl) } lazy val ugcArmedSetting = settingStore[Boolean]( diff --git a/modules/security/src/main/GarbageCollector.scala b/modules/security/src/main/GarbageCollector.scala index c9a44c9d4b49f..e219afde1ff0a 100644 --- a/modules/security/src/main/GarbageCollector.scala +++ b/modules/security/src/main/GarbageCollector.scala @@ -53,7 +53,7 @@ final class GarbageCollector( case ApplyData(user, ip, email, req) => for { spy <- userSpy(user) - ipSusp <- ipTrust.isSuspicious(ip, IpIntel.Reason.GarbageCollector) + ipSusp <- ipTrust.isSuspicious(ip, Ip2Proxy.Reason.GarbageCollector) } yield { val printOpt = spy.prints.headOption logger.debug(s"apply ${data.user.username} print=${printOpt}") @@ -67,7 +67,7 @@ final class GarbageCollector( badOtherAccounts(spy.otherUsers.map(_.user)) ?? { others => logger.debug(s"other ${data.user.username} others=${others.map(_.username)}") lila.common.Future - .exists(spy.ips)(ipTrust.isSuspicious(_, IpIntel.Reason.GarbageCollector)) + .exists(spy.ips)(ipTrust.isSuspicious(_, Ip2Proxy.Reason.GarbageCollector)) .map { _ ?? collect( user, diff --git a/modules/security/src/main/Ip2Proxy.scala b/modules/security/src/main/Ip2Proxy.scala new file mode 100644 index 0000000000000..ca6261c734275 --- /dev/null +++ b/modules/security/src/main/Ip2Proxy.scala @@ -0,0 +1,59 @@ +package lila.security + +import play.api.libs.ws.WSClient +import scala.concurrent.duration._ +import com.github.blemale.scaffeine.AsyncCache + +import lila.common.IpAddress + +final class Ip2Proxy( + ws: WSClient, + cacheApi: lila.memo.CacheApi, + checkUrl: String +)(implicit ec: scala.concurrent.ExecutionContext) { + + import Ip2Proxy._ + + def apply(ip: IpAddress, reason: Reason): Fu[ProxyType] = failable(ip, reason).nevermind(ProxyType("-")) + + def failable(ip: IpAddress, reason: Reason): Fu[ProxyType] = + if (isBlacklisted(ip)) fuccess(ProxyType("BLK")) + else cache.getFuture(ip, get(reason)) + + private val cache: AsyncCache[IpAddress, ProxyType] = cacheApi.scaffeine + .expireAfterWrite(1 days) + .buildAsync + + private def get(reason: Reason)(ip: IpAddress): Fu[ProxyType] = { + lila.mon.security.proxy.reason(reason.toString).increment() + ws.url(checkUrl) + .addQueryStringParameters("ip" -> ip.value) + .get() + .map { body => + ProxyType((body.json \ "proxy_type").asOpt[String] getOrElse "-") + } + .monSuccess(_.security.proxy.request) + } +} + +object Ip2Proxy { + + case class ProxyType(value: String) extends AnyVal { + def isProxy = value != "-" + } + + sealed trait Reason + object Reason { + case object GarbageCollector extends Reason + case object UserMod extends Reason + case object Signup extends Reason + } + + // Proxies ip2proxy does not detect + private val blackList = List( + "5.121.", + "5.122." + ) + + def isBlacklisted(ip: IpAddress): Boolean = blackList.exists(ip.value.startsWith) +} diff --git a/modules/security/src/main/IpIntel.scala b/modules/security/src/main/IpIntel.scala deleted file mode 100644 index db83afd129df1..0000000000000 --- a/modules/security/src/main/IpIntel.scala +++ /dev/null @@ -1,62 +0,0 @@ -package lila.security - -import play.api.libs.ws.WSClient -import scala.concurrent.duration._ -import com.github.blemale.scaffeine.AsyncCache - -import lila.common.{ EmailAddress, IpAddress } - -final class IpIntel( - ws: WSClient, - cacheApi: lila.memo.CacheApi, - checkUrl: String, - contactEmail: EmailAddress -)(implicit ec: scala.concurrent.ExecutionContext) { - - def apply(ip: IpAddress, reason: IpIntel.Reason): Fu[Int] = failable(ip, reason).nevermind - - def failable(ip: IpAddress, reason: IpIntel.Reason): Fu[Int] = - if (IpIntel isBlacklisted ip) fuccess(90) - else if (contactEmail.value.isEmpty) fuccess(0) - else cache.getFuture(ip, get(reason)) - - private val cache: AsyncCache[IpAddress, Int] = cacheApi.scaffeine - .expireAfterWrite(7 days) - .buildAsync - - private def get(reason: IpIntel.Reason)(ip: IpAddress): Fu[Int] = { - lila.mon.security.proxy.ipintel(reason.toString).increment() - val url = s"$checkUrl?ip=$ip&contact=${contactEmail.value}" - ws.url(url) - .get() - .dmap(_.body) - .flatMap { str => - str.toFloatOption.fold[Fu[Int]](fufail(s"Invalid ratio ${str.take(140)}")) { ratio => - if (ratio < 0) fufail(s"IpIntel error $ratio on $url") - else fuccess((ratio * 100).toInt) - } - } - .monSuccess(_.security.proxy.request) - .addEffect { percent => - lila.mon.security.proxy.percent.record(percent max 0) - } - } -} - -object IpIntel { - - sealed trait Reason - object Reason { - case object GarbageCollector extends Reason - case object UserMod extends Reason - case object Signup extends Reason - } - - // Proxies ipintel doesn't detect - private val blackList = List( - "5.121.", - "5.122." - ) - - def isBlacklisted(ip: IpAddress): Boolean = blackList.exists(ip.value.startsWith) -} diff --git a/modules/security/src/main/IpTrust.scala b/modules/security/src/main/IpTrust.scala index ed8b2a73bdded..0e54b86250b68 100644 --- a/modules/security/src/main/IpTrust.scala +++ b/modules/security/src/main/IpTrust.scala @@ -2,23 +2,23 @@ package lila.security import lila.common.IpAddress -final class IpTrust(intelApi: IpIntel, geoApi: GeoIP, torApi: Tor, firewallApi: Firewall) { +final class IpTrust(proxyApi: Ip2Proxy, geoApi: GeoIP, torApi: Tor, firewallApi: Firewall) { - def isSuspicious(ip: IpAddress, reason: IpIntel.Reason): Fu[Boolean] = - if (IpIntel isBlacklisted ip) fuTrue + def isSuspicious(ip: IpAddress, reason: Ip2Proxy.Reason): Fu[Boolean] = + if (Ip2Proxy isBlacklisted ip) fuTrue else if (firewallApi blocksIp ip) fuTrue else if (torApi isExitNode ip) fuTrue else { val location = geoApi orUnknown ip if (location == Location.unknown || location == Location.tor) fuTrue else if (isUndetectedProxy(location)) fuTrue - else intelApi(ip, reason).dmap { 75 < _ } + else proxyApi(ip, reason).dmap(_.isProxy) } - def isSuspicious(ipData: UserSpy.IPData, reason: IpIntel.Reason): Fu[Boolean] = + def isSuspicious(ipData: UserSpy.IPData, reason: Ip2Proxy.Reason): Fu[Boolean] = isSuspicious(ipData.ip.value, reason) - /* lichess blacklist of proxies that ipintel doesn't know about */ + /* lichess blacklist of proxies that ip2proxy doesn't know about */ private def isUndetectedProxy(location: Location): Boolean = location.shortCountry == "Iran" || location.shortCountry == "United Arab Emirates" || diff --git a/modules/security/src/main/SecurityConfig.scala b/modules/security/src/main/SecurityConfig.scala index 0ec9e4ce8c28f..8557091a93e8e 100644 --- a/modules/security/src/main/SecurityConfig.scala +++ b/modules/security/src/main/SecurityConfig.scala @@ -5,7 +5,6 @@ import io.methvin.play.autoconfig._ import scala.concurrent.duration.FiniteDuration import lila.common.config._ -import lila.common.EmailAddress import SecurityConfig._ @@ -24,8 +23,7 @@ final private class SecurityConfig( @ConfigName("check_mail_api") val checkMail: CheckMail, val recaptcha: Recaptcha.Config, val mailgun: Mailgun.Config, - @ConfigName("ipintel.url") val ipIntelUrl: String, - @ConfigName("ipintel.email") val ipIntelEmail: EmailAddress, + @ConfigName("ip2proxy.url") val ip2ProxyUrl: String, @ConfigName("lame_name_check") val lameNameCheck: LameNameCheck ) diff --git a/modules/security/src/main/Signup.scala b/modules/security/src/main/Signup.scala index ef8fdaf82ef17..234988c315749 100644 --- a/modules/security/src/main/Signup.scala +++ b/modules/security/src/main/Signup.scala @@ -46,7 +46,7 @@ final class Signup( store.recentByPrintExists(fp) flatMap { printFound => if (printFound) fuccess(YesBecausePrintExists) else - ipTrust.isSuspicious(ip, IpIntel.Reason.Signup).map { + ipTrust.isSuspicious(ip, Ip2Proxy.Reason.Signup).map { case true => YesBecauseIpSusp case _ => Nope } @@ -172,7 +172,7 @@ final class Signup( s"fp: ${fingerPrint} mustConfirm: $mustConfirm fp: ${fingerPrint.??(_.value)} api: ${apiVersion.??(_.value)}" ) val ip = HTTPRequest lastRemoteAddress req - ipTrust.isSuspicious(ip, IpIntel.Reason.Signup) foreach { susp => + ipTrust.isSuspicious(ip, Ip2Proxy.Reason.Signup) foreach { susp => slack.signup(user, email, ip, fingerPrint.flatMap(_.hash).map(_.value), apiVersion, susp) } } diff --git a/ui/site/src/user-mod.js b/ui/site/src/user-mod.js index 2c5bfedafc4ad..c69454ec18270 100644 --- a/ui/site/src/user-mod.js +++ b/ui/site/src/user-mod.js @@ -76,9 +76,9 @@ function userMod($zone) { var $li = $(this); $(this).one('mouseover', function() { $.ajax({ - url: '/mod/ip-intel?ip=' + $(this).find('.address ip').text(), + url: '/mod/ip2proxy?ip=' + $(this).find('.address ip').text(), success: function(res) { - $li.append($('' + res + '% proxy')); + $li.append($('').text(res)); } }); });