From 84bbc454b0f56616e30464bf2e021595753eae5e Mon Sep 17 00:00:00 2001 From: Thibault Duplessis Date: Mon, 31 Aug 2015 22:32:59 +0200 Subject: [PATCH] get rid of elasticsearch for good --- modules/forum/src/main/PostApi.scala | 2 +- modules/forumSearch/src/main/Env.scala | 2 +- modules/forumSearch/src/main/Indexer.scala | 116 +++++++++--------- modules/forumSearch/src/main/Query.scala | 66 +++++----- modules/gameSearch/src/main/Env.scala | 5 +- modules/gameSearch/src/main/Indexer.scala | 78 ++++++------ modules/gameSearch/src/main/Query.scala | 72 ++++++----- modules/gameSearch/src/main/Sorting.scala | 11 +- modules/search/src/main/ESClient.scala | 83 +------------ modules/search/src/main/ElasticSearch.scala | 4 - modules/search/src/main/Env.scala | 27 +--- .../search/src/main/PaginatorBuilder.scala | 18 +-- modules/search/src/main/Query.scala | 6 - modules/search/src/main/Range.scala | 12 +- modules/search/src/main/actorApi.scala | 11 +- modules/search/src/main/model.scala | 24 ---- modules/teamSearch/src/main/Env.scala | 2 +- modules/teamSearch/src/main/Indexer.scala | 88 +++++++------ modules/teamSearch/src/main/Query.scala | 34 +++-- project/Build.scala | 13 +- project/Dependencies.scala | 1 - 21 files changed, 252 insertions(+), 423 deletions(-) delete mode 100644 modules/search/src/main/model.scala diff --git a/modules/forum/src/main/PostApi.scala b/modules/forum/src/main/PostApi.scala index b67005467fbce..fbdc8adcf62e2 100644 --- a/modules/forum/src/main/PostApi.scala +++ b/modules/forum/src/main/PostApi.scala @@ -97,7 +97,7 @@ final class PostApi( } yield PostView(post, topic, categ, lastPageOf(topic)) } flatten - def viewsFromIds(postIds: List[String]): Fu[List[PostView]] = + def viewsFromIds(postIds: Seq[String]): Fu[List[PostView]] = $find.byOrderedIds[Post](postIds) flatMap views def view(post: Post): Fu[Option[PostView]] = diff --git a/modules/forumSearch/src/main/Env.scala b/modules/forumSearch/src/main/Env.scala index c4c0b674e79dd..b2900c5ad19c1 100644 --- a/modules/forumSearch/src/main/Env.scala +++ b/modules/forumSearch/src/main/Env.scala @@ -40,7 +40,7 @@ final class Env( private lazy val paginatorBuilder = new lila.search.PaginatorBuilder( indexer = indexer, maxPerPage = PaginatorMaxPerPage, - converter = res => postApi viewsFromIds res.hitIds + converter = postApi.viewsFromIds _ ) } diff --git a/modules/forumSearch/src/main/Indexer.scala b/modules/forumSearch/src/main/Indexer.scala index dcab02f18bb44..252ada41d3882 100644 --- a/modules/forumSearch/src/main/Indexer.scala +++ b/modules/forumSearch/src/main/Indexer.scala @@ -2,10 +2,6 @@ package lila.forumSearch import akka.actor._ import akka.pattern.pipe -import com.sksamuel.elastic4s -import elastic4s.SimpleAnalyzer -import elastic4s.ElasticDsl._ -import elastic4s.mappings.FieldType._ import lila.forum.actorApi._ import lila.forum.{ Post, PostLiteView, PostApi } @@ -22,67 +18,67 @@ private[forumSearch] final class Indexer( def receive = { - case Search(definition) => client search definition pipeTo sender - case Count(definition) => client count definition pipeTo sender +// case Search(definition) => client search definition pipeTo sender +// case Count(definition) => client count definition pipeTo sender - case InsertPost(post) => postApi liteView post foreach { - _ foreach { view => - client store store(view) - } - } +// case InsertPost(post) => postApi liteView post foreach { +// _ foreach { view => +// client store store(view) +// } +// } - case RemovePost(id) => client.deleteById(id, indexType) +// case RemovePost(id) => client.deleteById(id, indexType) - case RemoveTopic(id) => client.deleteByQuery(s"${Fields.topicId}:$id", indexType) +// case RemoveTopic(id) => client.deleteByQuery(s"${Fields.topicId}:$id", indexType) case Reset => - client.createType(indexName, typeName) - try { - client put { - put mapping indexName / typeName as Seq( - Fields.body typed StringType boost 2, - Fields.topic typed StringType boost 4, - Fields.author typed StringType index "not_analyzed", - Fields.topicId typed StringType, - Fields.staff typed BooleanType, - Fields.troll typed BooleanType, - Fields.date typed DateType - ) - } - import scala.concurrent.Await - import scala.concurrent.duration._ - import play.api.libs.json.Json - import lila.db.api._ - import lila.forum.tube.postTube - Await.result( - $enumerate.bulk[Option[Post]]($query[Post](Json.obj()), 200) { postOptions => - (postApi liteViews postOptions.flatten) flatMap { views => - client bulk { - bulk { - (views map store): _* - } - } - } - }, 20 minutes) - sender ! (()) - } - catch { - case e: Exception => - println(e) - sender ! Status.Failure(e) - } +// client.createType(indexName, typeName) +// try { +// client put { +// put mapping indexName / typeName as Seq( +// Fields.body typed StringType boost 2, +// Fields.topic typed StringType boost 4, +// Fields.author typed StringType index "not_analyzed", +// Fields.topicId typed StringType, +// Fields.staff typed BooleanType, +// Fields.troll typed BooleanType, +// Fields.date typed DateType +// ) +// } +// import scala.concurrent.Await +// import scala.concurrent.duration._ +// import play.api.libs.json.Json +// import lila.db.api._ +// import lila.forum.tube.postTube +// Await.result( +// $enumerate.bulk[Option[Post]]($query[Post](Json.obj()), 200) { postOptions => +// (postApi liteViews postOptions.flatten) flatMap { views => +// client bulk { +// bulk { +// (views map store): _* +// } +// } +// } +// }, 20 minutes) +// sender ! (()) +// } +// catch { +// case e: Exception => +// println(e) +// sender ! Status.Failure(e) +// } } - private def store(view: PostLiteView) = - index into indexType fields { - List( - Fields.body -> view.post.text.take(10000), - Fields.topic -> view.topic.name, - Fields.author -> ~(view.post.userId orElse view.post.author map (_.toLowerCase)), - Fields.topicId -> view.topic.id, - Fields.staff -> view.post.isStaff, - Fields.troll -> view.post.troll, - Fields.date -> view.post.createdAt.getDate - ): _* - } id view.post.id + // private def store(view: PostLiteView) = + // index into indexType fields { + // List( + // Fields.body -> view.post.text.take(10000), + // Fields.topic -> view.topic.name, + // Fields.author -> ~(view.post.userId orElse view.post.author map (_.toLowerCase)), + // Fields.topicId -> view.topic.id, + // Fields.staff -> view.post.isStaff, + // Fields.troll -> view.post.troll, + // Fields.date -> view.post.createdAt.getDate + // ): _* + // } id view.post.id } diff --git a/modules/forumSearch/src/main/Query.scala b/modules/forumSearch/src/main/Query.scala index 4b4bb24e87c78..eef0f85d5fbd3 100644 --- a/modules/forumSearch/src/main/Query.scala +++ b/modules/forumSearch/src/main/Query.scala @@ -1,9 +1,5 @@ package lila.forumSearch -import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s.QueryDefinition -import org.elasticsearch.search.sort.SortOrder - import lila.search.ElasticSearch private[forumSearch] final class Query private ( @@ -12,37 +8,37 @@ private[forumSearch] final class Query private ( staff: Boolean, troll: Boolean) extends lila.search.Query { - def searchDef(from: Int = 0, size: Int = 10) = - search in indexType query makeQuery sort ( - field sort Fields.date order SortOrder.DESC - ) start from size size - - def countDef = count from indexType query makeQuery - - private def queryTerms = terms filterNot (_ startsWith "user:") - private def userSearch = terms find (_ startsWith "user:") map { _ drop 5 } - - private lazy val makeQuery = filteredQuery query { - queryTerms match { - case Nil => all - case terms => must { - terms.map { term => - multiMatchQuery(term) fields (Query.searchableFields: _*) - }: _* - } - } - } filter { - List( - userSearch map { termFilter(Fields.author, _) }, - !staff option termFilter(Fields.staff, false), - !troll option termFilter(Fields.troll, false) - ).flatten match { - case Nil => matchAllFilter - case filters => must { - filters: _* - } - } - } + // def searchDef(from: Int = 0, size: Int = 10) = + // search in indexType query makeQuery sort ( + // field sort Fields.date order SortOrder.DESC + // ) start from size size + + // def countDef = count from indexType query makeQuery + + // private def queryTerms = terms filterNot (_ startsWith "user:") + // private def userSearch = terms find (_ startsWith "user:") map { _ drop 5 } + + // private lazy val makeQuery = filteredQuery query { + // queryTerms match { + // case Nil => all + // case terms => must { + // terms.map { term => + // multiMatchQuery(term) fields (Query.searchableFields: _*) + // }: _* + // } + // } + // } filter { + // List( + // userSearch map { termFilter(Fields.author, _) }, + // !staff option termFilter(Fields.staff, false), + // !troll option termFilter(Fields.troll, false) + // ).flatten match { + // case Nil => matchAllFilter + // case filters => must { + // filters: _* + // } + // } + // } } object Query { diff --git a/modules/gameSearch/src/main/Env.scala b/modules/gameSearch/src/main/Env.scala index 3aa9e3cecad92..21b0be4990c5d 100644 --- a/modules/gameSearch/src/main/Env.scala +++ b/modules/gameSearch/src/main/Env.scala @@ -22,10 +22,13 @@ final class Env( typeName = TypeName )), name = IndexerName) + private def converter(ids: Seq[String]) = + $find.byOrderedIds[lila.game.Game](ids) + lazy val paginator = new lila.search.PaginatorBuilder( indexer = indexer, maxPerPage = PaginatorMaxPerPage, - converter = res => $find.byOrderedIds[lila.game.Game](res.hitIds)) + converter = converter _) lazy val forms = new DataForm diff --git a/modules/gameSearch/src/main/Indexer.scala b/modules/gameSearch/src/main/Indexer.scala index ddb0bad0a7900..169ff57fdbe32 100644 --- a/modules/gameSearch/src/main/Indexer.scala +++ b/modules/gameSearch/src/main/Indexer.scala @@ -2,8 +2,6 @@ package lila.gameSearch import akka.actor._ import akka.pattern.pipe -import com.sksamuel.elastic4s.ElasticDsl.{ RichFuture => _, _ } -import com.sksamuel.elastic4s.mappings.FieldType._ import lila.game.actorApi.{ InsertGame, FinishGame } import lila.game.GameRepo @@ -19,16 +17,16 @@ private[gameSearch] final class Indexer( def receive = { - case Search(definition) => client search definition pipeTo sender - case Count(definition) => client count definition pipeTo sender + // case Search(definition) => sender ! Nil + // case Count(definition) => sender ! 0 - case FinishGame(game, _, _) => self ! InsertGame(game) + // case FinishGame(game, _, _) => self ! InsertGame(game) - case InsertGame(game) => if (storable(game)) { - GameRepo isAnalysed game.id foreach { analysed => - client store store(indexName, game, analysed) - } - } + // case InsertGame(game) => if (storable(game)) { + // GameRepo isAnalysed game.id foreach { analysed => + // client store store(indexName, game, analysed) + // } + // } case Reset => sys error "Game search reset disabled" @@ -98,35 +96,35 @@ private[gameSearch] final class Indexer( // }.await } - private def storable(game: lila.game.Game) = - (game.finished || game.imported) && game.playedTurns > 4 + // private def storable(game: lila.game.Game) = + // (game.finished || game.imported) && game.playedTurns > 4 - private def store(inIndex: String, game: lila.game.Game, hasAnalyse: Boolean) = { - import Fields._ - index into s"$inIndex/$typeName" fields { - List( - status -> (game.status match { - case s if s.is(_.Timeout) => chess.Status.Resign - case s if s.is(_.NoStart) => chess.Status.Resign - case s => game.status - }).id.some, - turns -> math.ceil(game.turns.toFloat / 2).some, - rated -> game.rated.some, - variant -> game.variant.id.some, - uids -> game.userIds.toArray.some.filterNot(_.isEmpty), - winner -> (game.winner flatMap (_.userId)), - winnerColor -> game.winner.fold(3)(_.color.fold(1, 2)).some, - averageRating -> game.averageUsersRating, - ai -> game.aiLevel, - date -> (ElasticSearch.Date.formatter print game.createdAt).some, - duration -> game.estimateTotalTime.some, - opening -> (game.opening map (_.code.toLowerCase)), - analysed -> hasAnalyse.some, - whiteUser -> game.whitePlayer.userId, - blackUser -> game.blackPlayer.userId - ).collect { - case (key, Some(value)) => key -> value - }: _* - } id game.id - } + // private def store(inIndex: String, game: lila.game.Game, hasAnalyse: Boolean) = { + // import Fields._ + // index into s"$inIndex/$typeName" fields { + // List( + // status -> (game.status match { + // case s if s.is(_.Timeout) => chess.Status.Resign + // case s if s.is(_.NoStart) => chess.Status.Resign + // case s => game.status + // }).id.some, + // turns -> math.ceil(game.turns.toFloat / 2).some, + // rated -> game.rated.some, + // variant -> game.variant.id.some, + // uids -> game.userIds.toArray.some.filterNot(_.isEmpty), + // winner -> (game.winner flatMap (_.userId)), + // winnerColor -> game.winner.fold(3)(_.color.fold(1, 2)).some, + // averageRating -> game.averageUsersRating, + // ai -> game.aiLevel, + // date -> (ElasticSearch.Date.formatter print game.createdAt).some, + // duration -> game.estimateTotalTime.some, + // opening -> (game.opening map (_.code.toLowerCase)), + // analysed -> hasAnalyse.some, + // whiteUser -> game.whitePlayer.userId, + // blackUser -> game.blackPlayer.userId + // ).collect { + // case (key, Some(value)) => key -> value + // }: _* + // } id game.id + // } } diff --git a/modules/gameSearch/src/main/Query.scala b/modules/gameSearch/src/main/Query.scala index cabe8f02d52f6..ea2dc98ba6d5c 100644 --- a/modules/gameSearch/src/main/Query.scala +++ b/modules/gameSearch/src/main/Query.scala @@ -1,8 +1,6 @@ package lila.gameSearch import chess.{ Mode, Status, Openings } -import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s.QueryDefinition import org.joda.time.DateTime import lila.rating.RatingRange @@ -47,45 +45,45 @@ case class Query( date.nonEmpty || duration.nonEmpty - def searchDef(from: Int = 0, size: Int = 10) = - search in indexType query makeQuery sort sorting.definition start from size size - - def countDef = count from indexType query makeQuery - - private lazy val makeQuery = filteredQuery query matchall filter { - List( - usernames map { termFilter(Fields.uids, _) }, - toFilters(winner, Fields.winner), - toFilters(winnerColor, Fields.winnerColor), - turns filters Fields.turns, - averageRating filters Fields.averageRating, - duration map (60 *) filters Fields.duration, - date map ElasticSearch.Date.formatter.print filters Fields.date, - hasAiFilters, - (hasAi | true).fold(aiLevel filters Fields.ai, Nil), - toFilters(variant, Fields.variant), - toFilters(rated, Fields.rated), - toFilters(opening, Fields.opening), - toFilters(status, Fields.status), - toFilters(analysed, Fields.analysed), - toFilters(whiteUser, Fields.whiteUser), - toFilters(blackUser, Fields.blackUser) - ).flatten match { - case Nil => matchAllFilter - case filters => must(filters: _*) - } - } + // def searchDef(from: Int = 0, size: Int = 10) = + // search in indexType query makeQuery sort sorting.definition start from size size + + // def countDef = count from indexType query makeQuery + + // private lazy val makeQuery = filteredQuery query matchall filter { + // List( + // usernames map { termFilter(Fields.uids, _) }, + // toFilters(winner, Fields.winner), + // toFilters(winnerColor, Fields.winnerColor), + // turns filters Fields.turns, + // averageRating filters Fields.averageRating, + // duration map (60 *) filters Fields.duration, + // date map ElasticSearch.Date.formatter.print filters Fields.date, + // hasAiFilters, + // (hasAi | true).fold(aiLevel filters Fields.ai, Nil), + // toFilters(variant, Fields.variant), + // toFilters(rated, Fields.rated), + // toFilters(opening, Fields.opening), + // toFilters(status, Fields.status), + // toFilters(analysed, Fields.analysed), + // toFilters(whiteUser, Fields.whiteUser), + // toFilters(blackUser, Fields.blackUser) + // ).flatten match { + // case Nil => matchAllFilter + // case filters => must(filters: _*) + // } + // } def usernames = List(user1, user2).flatten - private def hasAiFilters = hasAi.toList map { a => - a.fold(existsFilter(Fields.ai), missingFilter(Fields.ai)) - } + // private def hasAiFilters = hasAi.toList map { a => + // a.fold(existsFilter(Fields.ai), missingFilter(Fields.ai)) + // } - private def toFilters(query: Option[_], name: String) = query.toList map { - case s: String => termFilter(name, s.toLowerCase) - case x => termFilter(name, x) - } + // private def toFilters(query: Option[_], name: String) = query.toList map { + // case s: String => termFilter(name, s.toLowerCase) + // case x => termFilter(name, x) + // } } object Query { diff --git a/modules/gameSearch/src/main/Sorting.scala b/modules/gameSearch/src/main/Sorting.scala index eb548f631d6ca..5c934738e80ff 100644 --- a/modules/gameSearch/src/main/Sorting.scala +++ b/modules/gameSearch/src/main/Sorting.scala @@ -1,13 +1,12 @@ package lila.gameSearch -import com.sksamuel.elastic4s.ElasticDsl._ -import org.elasticsearch.search.sort.SortOrder +import lila.db.api.SortOrder case class Sorting(f: String, order: String) { - def definition = - field sort (Sorting.fieldKeys contains f).fold(f, Sorting.default.f) order - (order.toLowerCase == "asc").fold(SortOrder.ASC, SortOrder.DESC) + // def definition = + // field sort (Sorting.fieldKeys contains f).fold(f, Sorting.default.f) order + // (order.toLowerCase == "asc").fold(SortOrder.Ascending, SortOrder.Descending) } object Sorting { @@ -19,7 +18,7 @@ object Sorting { def fieldKeys = fields map (_._1) - val orders = List(SortOrder.DESC, SortOrder.ASC) map { s => s.toString -> s.toString } + val orders = List(SortOrder.Descending, SortOrder.Ascending) map { s => s.toString -> s.toString } val default = Sorting(Fields.date, "desc") } diff --git a/modules/search/src/main/ESClient.scala b/modules/search/src/main/ESClient.scala index f59f5dc024767..da364b1cf88f1 100644 --- a/modules/search/src/main/ESClient.scala +++ b/modules/search/src/main/ESClient.scala @@ -1,88 +1,9 @@ package lila.search -import scala.concurrent.Future - -import com.sksamuel.elastic4s.ElasticDsl -import com.sksamuel.elastic4s.ElasticDsl.{ RichFuture => _, _ } -import com.sksamuel.elastic4s.Executable -import com.sksamuel.elastic4s.mappings.{ PutMappingDefinition } -import com.sksamuel.elastic4s.{ ElasticClient, CountDefinition, SearchDefinition, IndexDefinition, BulkDefinition } - -sealed trait ESClient { - - def search(d: SearchDefinition): Fu[SearchResponse] - def count(d: CountDefinition): Fu[CountResponse] - - def store(d: IndexDefinition): Funit - def deleteById(id: String, indexType: String): Funit - def deleteByQuery(query: String, indexType: String): Funit - def bulk(d: BulkDefinition): Funit - - def put(d: PutMappingDefinition): Funit - - // synchronously create index and type, ignoring errors (already existing) - def createType(indexName: String, typeName: String): Unit +trait ESClient { } object ESClient { - def make(client: Option[ElasticClient]): ESClient = client match { - case None => new StubESClient - case Some(c) => new RealESClient(c) - } -} - -final class StubESClient extends ESClient { - - def search(d: SearchDefinition) = fuccess(SearchResponse.stub) - def count(d: CountDefinition) = fuccess(CountResponse.stub) - - def store(d: IndexDefinition) = funit - def deleteById(id: String, indexType: String) = funit - def deleteByQuery(query: String, indexType: String) = funit - def bulk(d: BulkDefinition) = funit - - def put(d: PutMappingDefinition) = funit - - def createType(indexName: String, typeName: String) { - } -} - -final class RealESClient(client: ElasticClient) extends ESClient { - - def search(d: SearchDefinition) = client execute d map SearchResponse.apply - def count(d: CountDefinition) = client execute d map CountResponse.apply - - def store(d: IndexDefinition) = client execute d void - def deleteById(id: String, indexType: String) = client execute { - ElasticDsl.delete id id from indexType - } void - def deleteByQuery(query: String, indexType: String) = client execute { - ElasticDsl.delete from indexType where query - } void - def bulk(d: BulkDefinition) = client execute d void - - def put(d: PutMappingDefinition) = client execute d void - - def createType(indexName: String, typeName: String) { - try { - import scala.concurrent.Await - import scala.concurrent.duration._ - Await.result(client execute { - create index indexName - }, 10.seconds) - } - catch { - case e: Exception => // println("create type: " + e) - } - // client.sync execute { - // delete from indexName -> typeName where matchall - // } - import org.elasticsearch.index.query.QueryBuilders._ - client.java.prepareDeleteByQuery(indexName) - .setTypes(typeName) - .setQuery(matchAllQuery) - .execute() - .actionGet() - } + def make: ESClient = new ESClient {} } diff --git a/modules/search/src/main/ElasticSearch.scala b/modules/search/src/main/ElasticSearch.scala index 2dd8460cf2678..53ff53f8d866e 100644 --- a/modules/search/src/main/ElasticSearch.scala +++ b/modules/search/src/main/ElasticSearch.scala @@ -2,12 +2,8 @@ package lila.search import play.api.libs.json._ -import com.sksamuel.elastic4s.ElasticClient -import com.sksamuel.elastic4s.ElasticDsl._ - object ElasticSearch { - def decomposeTextQuery(text: String): List[String] = text.trim.toLowerCase.replace("+", " ").split(" ").toList diff --git a/modules/search/src/main/Env.scala b/modules/search/src/main/Env.scala index c96ad801401a1..74f15ebe68f9d 100644 --- a/modules/search/src/main/Env.scala +++ b/modules/search/src/main/Env.scala @@ -5,39 +5,14 @@ import scala.concurrent.Future import scala.util.{ Success, Failure } import akka.actor.ActorSystem -import com.sksamuel.elastic4s.ElasticClient import com.typesafe.config.Config -import org.elasticsearch.common.settings.ImmutableSettings final class Env( config: Config, system: ActorSystem, scheduler: lila.common.Scheduler) { - private val Enabled = config getBoolean "enabled" - private val ESHost = config getString "es.host" - private val ESPort = config getInt "es.port" - private val ESCluster = config getString "es.cluster" - private val IndexesToOptimize = config getStringList "indexes_to_optimize" - private val IndexerMaxAttempts = 10 - - lazy val underlyingClient: Option[ElasticClient] = Enabled option { - val settings = ImmutableSettings.settingsBuilder() - .put("cluster.name", ESCluster).build() - ElasticClient.remote(settings, ESHost -> ESPort) - } - - lazy val client = ESClient make underlyingClient - - underlyingClient foreach { c => - import scala.concurrent.duration._ - import com.sksamuel.elastic4s.ElasticDsl._ - scheduler.effect(1 hour, "search: optimize index") { - c execute { - optimize index IndexesToOptimize - } - } - } + lazy val client = ESClient.make } object Env { diff --git a/modules/search/src/main/PaginatorBuilder.scala b/modules/search/src/main/PaginatorBuilder.scala index 93f8c63985470..f3e5975612114 100644 --- a/modules/search/src/main/PaginatorBuilder.scala +++ b/modules/search/src/main/PaginatorBuilder.scala @@ -1,8 +1,6 @@ package lila.search import akka.actor.ActorRef -import akka.pattern.ask -import com.sksamuel.elastic4s.ElasticDsl._ import lila.common.paginator._ import makeTimeout.large @@ -10,27 +8,19 @@ import makeTimeout.large final class PaginatorBuilder[A]( indexer: ActorRef, maxPerPage: Int, - converter: SearchResponse => Fu[List[A]]) { + converter: Seq[String] => Fu[List[A]]) { def apply(query: Query, page: Int): Fu[Paginator[A]] = Paginator( adapter = new ESAdapter(query), currentPage = page, maxPerPage = maxPerPage) - def ids(query: Query, max: Int): Fu[List[String]] = - indexer ? actorApi.Search(query.searchDef(0, max)) map { - case res: SearchResponse => res.hitIds - } + def ids(query: Query, max: Int): Fu[List[String]] = fuccess(Nil) private final class ESAdapter(query: Query) extends AdapterLike[A] { - def nbResults = indexer ? actorApi.Count(query.countDef) map { - case res: CountResponse => res.count - } + def nbResults = fuccess(0) - def slice(offset: Int, length: Int) = - indexer ? actorApi.Search(query.searchDef(offset, length)) flatMap { - case res: SearchResponse => converter(res) - } + def slice(offset: Int, length: Int) = fuccess(Nil) } } diff --git a/modules/search/src/main/Query.scala b/modules/search/src/main/Query.scala index 5ed2ba5725831..f485749408c5c 100644 --- a/modules/search/src/main/Query.scala +++ b/modules/search/src/main/Query.scala @@ -1,10 +1,4 @@ package lila.search -import com.sksamuel.elastic4s.{ CountDefinition, SearchDefinition } - trait Query { - - def searchDef(from: Int = 0, size: Int = 10): SearchDefinition - - def countDef: CountDefinition } diff --git a/modules/search/src/main/Range.scala b/modules/search/src/main/Range.scala index 96ac8f0f9acd2..9729a8b6f6d40 100644 --- a/modules/search/src/main/Range.scala +++ b/modules/search/src/main/Range.scala @@ -1,14 +1,12 @@ package lila.search -import com.sksamuel.elastic4s.ElasticDsl._ - final class Range[A] private (val a: Option[A], val b: Option[A]) { - def filters(name: String) = a.fold(b.toList map { bb => rangeFilter(name) lte bb.toString }) { aa => - b.fold(List(rangeFilter(name) gte aa.toString)) { bb => - List(rangeFilter(name) gte aa.toString lte bb.toString) - } - } + // def filters(name: String) = a.fold(b.toList map { bb => rangeFilter(name) lte bb.toString }) { aa => + // b.fold(List(rangeFilter(name) gte aa.toString)) { bb => + // List(rangeFilter(name) gte aa.toString lte bb.toString) + // } + // } def map[B](f: A => B) = new Range(a map f, b map f) diff --git a/modules/search/src/main/actorApi.scala b/modules/search/src/main/actorApi.scala index 9fcd5b23eb35a..4b0b15bed91c4 100644 --- a/modules/search/src/main/actorApi.scala +++ b/modules/search/src/main/actorApi.scala @@ -1,13 +1,10 @@ package lila.search package actorApi -import com.sksamuel.elastic4s.{ CountDefinition, SearchDefinition } -import org.elasticsearch.action.search.{ SearchResponse => ESSR } - case object Reset -case class Search(definition: SearchDefinition) -case class SearchResponse(res: ESSR) +// case class Search(definition: SearchDefinition) +// case class SearchResponse(res: ESSR) -case class Count(definition: CountDefinition) -case class CountResponse(res: Int) +// case class Count(definition: CountDefinition) +// case class CountResponse(res: Int) diff --git a/modules/search/src/main/model.scala b/modules/search/src/main/model.scala deleted file mode 100644 index 3903de117fe41..0000000000000 --- a/modules/search/src/main/model.scala +++ /dev/null @@ -1,24 +0,0 @@ -package lila.search - -import org.elasticsearch.action.count.{ CountResponse => CR } -import org.elasticsearch.action.search.{ SearchResponse => SR } - -case class SearchResponse(hitIds: List[String]) - -object SearchResponse { - - val stub = SearchResponse(Nil) - - def apply[A](res: SR): SearchResponse = - SearchResponse(res.getHits.hits.toList map (_.id)) -} - -case class CountResponse(count: Int) - -object CountResponse { - - val stub = CountResponse(0) - - def apply[A](res: CR): CountResponse = - CountResponse(res.getCount.toInt) -} diff --git a/modules/teamSearch/src/main/Env.scala b/modules/teamSearch/src/main/Env.scala index f85243df5506d..2865aded22939 100644 --- a/modules/teamSearch/src/main/Env.scala +++ b/modules/teamSearch/src/main/Env.scala @@ -39,7 +39,7 @@ final class Env( private lazy val paginatorBuilder = new lila.search.PaginatorBuilder( indexer = indexer, maxPerPage = PaginatorMaxPerPage, - converter = res => $find.byOrderedIds[lila.team.Team](res.hitIds) + converter = $find.byOrderedIds[lila.team.Team] _ ) } diff --git a/modules/teamSearch/src/main/Indexer.scala b/modules/teamSearch/src/main/Indexer.scala index 03fcaea4d2b79..fc630859fe9a1 100644 --- a/modules/teamSearch/src/main/Indexer.scala +++ b/modules/teamSearch/src/main/Indexer.scala @@ -2,8 +2,6 @@ package lila.teamSearch import akka.actor._ import akka.pattern.pipe -import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s.mappings.FieldType._ import lila.search.ESClient import lila.search.actorApi._ @@ -19,53 +17,53 @@ private[teamSearch] final class Indexer( def receive = { - case Search(definition) => client search definition pipeTo sender - case Count(definition) => client count definition pipeTo sender +// case Search(definition) => client search definition pipeTo sender +// case Count(definition) => client count definition pipeTo sender - case InsertTeam(team) => client store store(team) +// case InsertTeam(team) => client store store(team) - case RemoveTeam(id) => client.deleteById(id, indexType) +// case RemoveTeam(id) => client.deleteById(id, indexType) case Reset => - client.createType(indexName, typeName) - try { - client put { - put mapping indexName/typeName as Seq( - Fields.name typed StringType boost 3, - Fields.description typed StringType boost 2, - Fields.location typed StringType, - Fields.nbMembers typed ShortType - ) - } - import scala.concurrent.Await - import scala.concurrent.duration._ - import play.api.libs.json.Json - import lila.db.api._ - import lila.team.tube.teamTube - Await.result( - $enumerate.bulk[Option[Team]]($query[Team](Json.obj("enabled" -> true)), 100) { teamOptions => - client bulk { - bulk { - (teamOptions.flatten map store): _* - } - } - }, 20 minutes) - sender ! (()) - } - catch { - case e: Exception => - println(e) - sender ! Status.Failure(e) - } + // client.createType(indexName, typeName) + // try { + // client put { + // put mapping indexName/typeName as Seq( + // Fields.name typed StringType boost 3, + // Fields.description typed StringType boost 2, + // Fields.location typed StringType, + // Fields.nbMembers typed ShortType + // ) + // } + // import scala.concurrent.Await + // import scala.concurrent.duration._ + // import play.api.libs.json.Json + // import lila.db.api._ + // import lila.team.tube.teamTube + // Await.result( + // $enumerate.bulk[Option[Team]]($query[Team](Json.obj("enabled" -> true)), 100) { teamOptions => + // client bulk { + // bulk { + // (teamOptions.flatten map store): _* + // } + // } + // }, 20 minutes) + // sender ! (()) + // } + // catch { + // case e: Exception => + // println(e) + // sender ! Status.Failure(e) + // } } - private def store(team: Team) = - index into indexType fields { - List( - Fields.name -> team.name, - Fields.description -> team.description.take(10000), - Fields.location -> team.location, - Fields.nbMembers -> team.nbMembers - ): _* - } id team.id + // private def store(team: Team) = + // index into indexType fields { + // List( + // Fields.name -> team.name, + // Fields.description -> team.description.take(10000), + // Fields.location -> team.location, + // Fields.nbMembers -> team.nbMembers + // ): _* + // } id team.id } diff --git a/modules/teamSearch/src/main/Query.scala b/modules/teamSearch/src/main/Query.scala index 72e2e806c29c0..370829713624b 100644 --- a/modules/teamSearch/src/main/Query.scala +++ b/modules/teamSearch/src/main/Query.scala @@ -1,30 +1,26 @@ package lila.teamSearch -import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s.QueryDefinition -import org.elasticsearch.search.sort.SortOrder - import lila.search.ElasticSearch private[teamSearch] final class Query private ( indexType: String, terms: List[String]) extends lila.search.Query { - def searchDef(from: Int = 0, size: Int = 10) = - search in indexType query makeQuery sort ( - field sort Fields.nbMembers order SortOrder.DESC - ) start from size size - - def countDef = count from indexType query makeQuery - - private def makeQuery = terms match { - case Nil => all - case terms => must { - terms.map { term => - multiMatchQuery(term) fields (Query.searchableFields: _*) - }: _* - } - } +// def searchDef(from: Int = 0, size: Int = 10) = +// search in indexType query makeQuery sort ( +// field sort Fields.nbMembers order SortOrder.DESC +// ) start from size size + +// def countDef = count from indexType query makeQuery + +// private def makeQuery = terms match { +// case Nil => all +// case terms => must { +// terms.map { term => +// multiMatchQuery(term) fields (Query.searchableFields: _*) +// }: _* +// } +// } } object Query { diff --git a/project/Build.scala b/project/Build.scala index 6718ad65d7ac4..92ebdfa93407a 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -26,7 +26,7 @@ object ApplicationBuild extends Build { // offline := true, libraryDependencies ++= Seq( scalaz, scalalib, hasher, config, apache, - jgit, elastic4s, findbugs, RM, PRM, + jgit, findbugs, RM, PRM, spray.caching, maxmind, prismic), TwirlKeys.templateImports ++= Seq( "lila.game.{ Game, Player, Pov }", @@ -59,8 +59,7 @@ object ApplicationBuild extends Build { lazy val api = project("api", moduleCPDeps) .settings( libraryDependencies ++= provided( - play.api, hasher, config, apache, jgit, - elastic4s, findbugs, RM) + play.api, hasher, config, apache, jgit, findbugs, RM) ) aggregate (moduleRefs: _*) lazy val puzzle = project("puzzle", Seq( @@ -130,7 +129,7 @@ object ApplicationBuild extends Build { ) lazy val search = project("search", Seq(common, hub)).settings( - libraryDependencies ++= provided(play.api, elastic4s) + libraryDependencies ++= provided(play.api) ) lazy val chat = project("chat", Seq(common, db, user, security, i18n)).settings( @@ -159,7 +158,7 @@ object ApplicationBuild extends Build { lazy val gameSearch = project("gameSearch", Seq(common, hub, chess, search, game)).settings( libraryDependencies ++= provided( - play.api, RM, PRM, elastic4s) + play.api, RM, PRM) ) lazy val tv = project("tv", Seq(common, db, hub, socket, game, user, chess)).settings( @@ -246,7 +245,7 @@ object ApplicationBuild extends Build { lazy val forumSearch = project("forumSearch", Seq(common, hub, forum, search)).settings( libraryDependencies ++= provided( - play.api, RM, PRM, elastic4s) + play.api, RM, PRM) ) lazy val team = project("team", Seq(common, memo, db, user, forum, security, hub)).settings( @@ -256,7 +255,7 @@ object ApplicationBuild extends Build { lazy val teamSearch = project("teamSearch", Seq(common, hub, team, search)).settings( libraryDependencies ++= provided( - play.api, RM, PRM, elastic4s) + play.api, RM, PRM) ) lazy val i18n = project("i18n", Seq(common, db, user, hub)).settings( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index deafc8a67603b..54b0f4370812c 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -35,7 +35,6 @@ object Dependencies { val hasher = "com.roundeights" %% "hasher" % "1.0.0" val jgit = "org.eclipse.jgit" % "org.eclipse.jgit" % "3.2.0.201312181205-r" val jodaTime = "joda-time" % "joda-time" % "2.8.1" - val elastic4s = "com.sksamuel.elastic4s" %% "elastic4s-core" % "1.6.4" val RM = "org.reactivemongo" %% "reactivemongo" % "0.11.6" val PRM = "org.reactivemongo" %% "play2-reactivemongo" % "0.11.6.play24" val maxmind = "com.sanoma.cda" %% "maxmind-geoip2-scala" % "1.2.3-THIB"