Skip to content

Commit

Permalink
get rid of elasticsearch for good
Browse files Browse the repository at this point in the history
  • Loading branch information
@ornicar
ornicar committed Aug 31, 2015
1 parent b498991 commit 84bbc45
Show file tree
Hide file tree
Showing 21 changed files with 252 additions and 423 deletions.
2 changes: 1 addition & 1 deletion modules/forum/src/main/PostApi.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ final class PostApi(
} yield PostView(post, topic, categ, lastPageOf(topic))
} flatten

def viewsFromIds(postIds: List[String]): Fu[List[PostView]] =
def viewsFromIds(postIds: Seq[String]): Fu[List[PostView]] =
$find.byOrderedIds[Post](postIds) flatMap views

def view(post: Post): Fu[Option[PostView]] =
Expand Down
2 changes: 1 addition & 1 deletion modules/forumSearch/src/main/Env.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ final class Env(
private lazy val paginatorBuilder = new lila.search.PaginatorBuilder(
indexer = indexer,
maxPerPage = PaginatorMaxPerPage,
converter = res => postApi viewsFromIds res.hitIds
converter = postApi.viewsFromIds _
)
}

Expand Down
116 changes: 56 additions & 60 deletions modules/forumSearch/src/main/Indexer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,6 @@ package lila.forumSearch

import akka.actor._
import akka.pattern.pipe
import com.sksamuel.elastic4s
import elastic4s.SimpleAnalyzer
import elastic4s.ElasticDsl._
import elastic4s.mappings.FieldType._

import lila.forum.actorApi._
import lila.forum.{ Post, PostLiteView, PostApi }
Expand All @@ -22,67 +18,67 @@ private[forumSearch] final class Indexer(

def receive = {

case Search(definition) => client search definition pipeTo sender
case Count(definition) => client count definition pipeTo sender
// case Search(definition) => client search definition pipeTo sender
// case Count(definition) => client count definition pipeTo sender

case InsertPost(post) => postApi liteView post foreach {
_ foreach { view =>
client store store(view)
}
}
// case InsertPost(post) => postApi liteView post foreach {
// _ foreach { view =>
// client store store(view)
// }
// }

case RemovePost(id) => client.deleteById(id, indexType)
// case RemovePost(id) => client.deleteById(id, indexType)

case RemoveTopic(id) => client.deleteByQuery(s"${Fields.topicId}:$id", indexType)
// case RemoveTopic(id) => client.deleteByQuery(s"${Fields.topicId}:$id", indexType)

case Reset =>
client.createType(indexName, typeName)
try {
client put {
put mapping indexName / typeName as Seq(
Fields.body typed StringType boost 2,
Fields.topic typed StringType boost 4,
Fields.author typed StringType index "not_analyzed",
Fields.topicId typed StringType,
Fields.staff typed BooleanType,
Fields.troll typed BooleanType,
Fields.date typed DateType
)
}
import scala.concurrent.Await
import scala.concurrent.duration._
import play.api.libs.json.Json
import lila.db.api._
import lila.forum.tube.postTube
Await.result(
$enumerate.bulk[Option[Post]]($query[Post](Json.obj()), 200) { postOptions =>
(postApi liteViews postOptions.flatten) flatMap { views =>
client bulk {
bulk {
(views map store): _*
}
}
}
}, 20 minutes)
sender ! (())
}
catch {
case e: Exception =>
println(e)
sender ! Status.Failure(e)
}
// client.createType(indexName, typeName)
// try {
// client put {
// put mapping indexName / typeName as Seq(
// Fields.body typed StringType boost 2,
// Fields.topic typed StringType boost 4,
// Fields.author typed StringType index "not_analyzed",
// Fields.topicId typed StringType,
// Fields.staff typed BooleanType,
// Fields.troll typed BooleanType,
// Fields.date typed DateType
// )
// }
// import scala.concurrent.Await
// import scala.concurrent.duration._
// import play.api.libs.json.Json
// import lila.db.api._
// import lila.forum.tube.postTube
// Await.result(
// $enumerate.bulk[Option[Post]]($query[Post](Json.obj()), 200) { postOptions =>
// (postApi liteViews postOptions.flatten) flatMap { views =>
// client bulk {
// bulk {
// (views map store): _*
// }
// }
// }
// }, 20 minutes)
// sender ! (())
// }
// catch {
// case e: Exception =>
// println(e)
// sender ! Status.Failure(e)
// }
}

private def store(view: PostLiteView) =
index into indexType fields {
List(
Fields.body -> view.post.text.take(10000),
Fields.topic -> view.topic.name,
Fields.author -> ~(view.post.userId orElse view.post.author map (_.toLowerCase)),
Fields.topicId -> view.topic.id,
Fields.staff -> view.post.isStaff,
Fields.troll -> view.post.troll,
Fields.date -> view.post.createdAt.getDate
): _*
} id view.post.id
// private def store(view: PostLiteView) =
// index into indexType fields {
// List(
// Fields.body -> view.post.text.take(10000),
// Fields.topic -> view.topic.name,
// Fields.author -> ~(view.post.userId orElse view.post.author map (_.toLowerCase)),
// Fields.topicId -> view.topic.id,
// Fields.staff -> view.post.isStaff,
// Fields.troll -> view.post.troll,
// Fields.date -> view.post.createdAt.getDate
// ): _*
// } id view.post.id
}
66 changes: 31 additions & 35 deletions modules/forumSearch/src/main/Query.scala
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
package lila.forumSearch

import com.sksamuel.elastic4s.ElasticDsl._
import com.sksamuel.elastic4s.QueryDefinition
import org.elasticsearch.search.sort.SortOrder

import lila.search.ElasticSearch

private[forumSearch] final class Query private (
Expand All @@ -12,37 +8,37 @@ private[forumSearch] final class Query private (
staff: Boolean,
troll: Boolean) extends lila.search.Query {

def searchDef(from: Int = 0, size: Int = 10) =
search in indexType query makeQuery sort (
field sort Fields.date order SortOrder.DESC
) start from size size

def countDef = count from indexType query makeQuery

private def queryTerms = terms filterNot (_ startsWith "user:")
private def userSearch = terms find (_ startsWith "user:") map { _ drop 5 }

private lazy val makeQuery = filteredQuery query {
queryTerms match {
case Nil => all
case terms => must {
terms.map { term =>
multiMatchQuery(term) fields (Query.searchableFields: _*)
}: _*
}
}
} filter {
List(
userSearch map { termFilter(Fields.author, _) },
!staff option termFilter(Fields.staff, false),
!troll option termFilter(Fields.troll, false)
).flatten match {
case Nil => matchAllFilter
case filters => must {
filters: _*
}
}
}
// def searchDef(from: Int = 0, size: Int = 10) =
// search in indexType query makeQuery sort (
// field sort Fields.date order SortOrder.DESC
// ) start from size size

// def countDef = count from indexType query makeQuery

// private def queryTerms = terms filterNot (_ startsWith "user:")
// private def userSearch = terms find (_ startsWith "user:") map { _ drop 5 }

// private lazy val makeQuery = filteredQuery query {
// queryTerms match {
// case Nil => all
// case terms => must {
// terms.map { term =>
// multiMatchQuery(term) fields (Query.searchableFields: _*)
// }: _*
// }
// }
// } filter {
// List(
// userSearch map { termFilter(Fields.author, _) },
// !staff option termFilter(Fields.staff, false),
// !troll option termFilter(Fields.troll, false)
// ).flatten match {
// case Nil => matchAllFilter
// case filters => must {
// filters: _*
// }
// }
// }
}

object Query {
Expand Down
5 changes: 4 additions & 1 deletion modules/gameSearch/src/main/Env.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,13 @@ final class Env(
typeName = TypeName
)), name = IndexerName)

private def converter(ids: Seq[String]) =
$find.byOrderedIds[lila.game.Game](ids)

lazy val paginator = new lila.search.PaginatorBuilder(
indexer = indexer,
maxPerPage = PaginatorMaxPerPage,
converter = res => $find.byOrderedIds[lila.game.Game](res.hitIds))
converter = converter _)

lazy val forms = new DataForm

Expand Down
78 changes: 38 additions & 40 deletions modules/gameSearch/src/main/Indexer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package lila.gameSearch

import akka.actor._
import akka.pattern.pipe
import com.sksamuel.elastic4s.ElasticDsl.{ RichFuture => _, _ }
import com.sksamuel.elastic4s.mappings.FieldType._

import lila.game.actorApi.{ InsertGame, FinishGame }
import lila.game.GameRepo
Expand All @@ -19,16 +17,16 @@ private[gameSearch] final class Indexer(

def receive = {

case Search(definition) => client search definition pipeTo sender
case Count(definition) => client count definition pipeTo sender
// case Search(definition) => sender ! Nil
// case Count(definition) => sender ! 0

case FinishGame(game, _, _) => self ! InsertGame(game)
// case FinishGame(game, _, _) => self ! InsertGame(game)

case InsertGame(game) => if (storable(game)) {
GameRepo isAnalysed game.id foreach { analysed =>
client store store(indexName, game, analysed)
}
}
// case InsertGame(game) => if (storable(game)) {
// GameRepo isAnalysed game.id foreach { analysed =>
// client store store(indexName, game, analysed)
// }
// }

case Reset =>
sys error "Game search reset disabled"
Expand Down Expand Up @@ -98,35 +96,35 @@ private[gameSearch] final class Indexer(
// }.await
}

private def storable(game: lila.game.Game) =
(game.finished || game.imported) && game.playedTurns > 4
// private def storable(game: lila.game.Game) =
// (game.finished || game.imported) && game.playedTurns > 4

private def store(inIndex: String, game: lila.game.Game, hasAnalyse: Boolean) = {
import Fields._
index into s"$inIndex/$typeName" fields {
List(
status -> (game.status match {
case s if s.is(_.Timeout) => chess.Status.Resign
case s if s.is(_.NoStart) => chess.Status.Resign
case s => game.status
}).id.some,
turns -> math.ceil(game.turns.toFloat / 2).some,
rated -> game.rated.some,
variant -> game.variant.id.some,
uids -> game.userIds.toArray.some.filterNot(_.isEmpty),
winner -> (game.winner flatMap (_.userId)),
winnerColor -> game.winner.fold(3)(_.color.fold(1, 2)).some,
averageRating -> game.averageUsersRating,
ai -> game.aiLevel,
date -> (ElasticSearch.Date.formatter print game.createdAt).some,
duration -> game.estimateTotalTime.some,
opening -> (game.opening map (_.code.toLowerCase)),
analysed -> hasAnalyse.some,
whiteUser -> game.whitePlayer.userId,
blackUser -> game.blackPlayer.userId
).collect {
case (key, Some(value)) => key -> value
}: _*
} id game.id
}
// private def store(inIndex: String, game: lila.game.Game, hasAnalyse: Boolean) = {
// import Fields._
// index into s"$inIndex/$typeName" fields {
// List(
// status -> (game.status match {
// case s if s.is(_.Timeout) => chess.Status.Resign
// case s if s.is(_.NoStart) => chess.Status.Resign
// case s => game.status
// }).id.some,
// turns -> math.ceil(game.turns.toFloat / 2).some,
// rated -> game.rated.some,
// variant -> game.variant.id.some,
// uids -> game.userIds.toArray.some.filterNot(_.isEmpty),
// winner -> (game.winner flatMap (_.userId)),
// winnerColor -> game.winner.fold(3)(_.color.fold(1, 2)).some,
// averageRating -> game.averageUsersRating,
// ai -> game.aiLevel,
// date -> (ElasticSearch.Date.formatter print game.createdAt).some,
// duration -> game.estimateTotalTime.some,
// opening -> (game.opening map (_.code.toLowerCase)),
// analysed -> hasAnalyse.some,
// whiteUser -> game.whitePlayer.userId,
// blackUser -> game.blackPlayer.userId
// ).collect {
// case (key, Some(value)) => key -> value
// }: _*
// } id game.id
// }
}
Loading

0 comments on commit 84bbc45

Please sign in to comment.