Skip to content

Commit

Permalink
Implement public rooms APIs (#185)
Browse files Browse the repository at this point in the history
* Move events contents to common

* Basic database structure

* Complete database update

* Support visibility update and retrieval

* Add HTTP methods for visibility update and retrieval

* Add the database for the new component

* Add a listener for the new component

* Fix attribute update statements

* Create public rooms component

* Fix failing test

* Add roomserver consumer

* Fix a bug in aliases creation

* Add a check on type

* Implement public rooms directory

* Use auth API for visibility update

* Support filtering

* Add component to monolith

* Various fixes

* Fix computation of next public rooms batch

* Retrieve state events from the roomserver query API + avoid dupes on join

* Split update of string or boolean attribute in two separate functions

* Use event type to detect duplicate joins

* Improve the joined members counter computation

* Use event.RoomID()
  • Loading branch information
babolivier authored and NegativeMjark committed Aug 22, 2017
1 parent fc86821 commit b15ce90
Show file tree
Hide file tree
Showing 22 changed files with 1,098 additions and 36 deletions.
2 changes: 2 additions & 0 deletions dendrite-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ database:
room_server: "postgres://dendrite:itsasecret@localhost/dendrite_roomserver?sslmode=disable"
server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable"
federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable"
public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable"

# The TCP host:port pairs to bind the internal HTTP APIs to.
# These shouldn't be exposed to the public internet.
Expand All @@ -82,3 +83,4 @@ listen:
federation_api: "localhost:7772"
sync_api: "localhost:7773"
media_api: "localhost:7774"
public_rooms_api: "localhost:7775"
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
Expand Down Expand Up @@ -271,7 +272,7 @@ func buildMembershipEvents(
StateKey: &userID,
}

content := events.MemberContent{
content := common.MemberContent{
Membership: "join",
}

Expand Down
14 changes: 0 additions & 14 deletions src/github.com/matrix-org/dendrite/clientapi/routing/routing.go
Original file line number Diff line number Diff line change
Expand Up @@ -265,20 +265,6 @@ func Setup(
}),
)

r0mux.Handle("/publicRooms",
common.MakeAPI("public_rooms", func(req *http.Request) util.JSONResponse {
// TODO: Return a list of public rooms
return util.JSONResponse{
Code: 200,
JSON: struct {
Chunk []struct{} `json:"chunk"`
Start string `json:"start"`
End string `json:"end"`
}{[]struct{}{}, "", ""},
}
}),
)

unstableMux.Handle("/thirdparty/protocols",
common.MakeAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse {
// TODO: Return the third party protcols
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
"github.com/matrix-org/dendrite/clientapi/events"
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
Expand Down Expand Up @@ -132,7 +132,7 @@ func createRoom(req *http.Request, device *authtypes.Device,
return httputil.LogThenError(req, err)
}

membershipContent := events.MemberContent{
membershipContent := common.MemberContent{
Membership: "join",
DisplayName: profile.DisplayName,
AvatarURL: profile.AvatarURL,
Expand All @@ -159,16 +159,16 @@ func createRoom(req *http.Request, device *authtypes.Device,
// harder to reason about, hence sticking to a strict static ordering.
// TODO: Synapse has txn/token ID on each event. Do we need to do this here?
eventsToMake := []fledglingEvent{
{"m.room.create", "", events.CreateContent{Creator: userID}},
{"m.room.create", "", common.CreateContent{Creator: userID}},
{"m.room.member", userID, membershipContent},
{"m.room.power_levels", "", events.InitialPowerLevelsContent(userID)},
{"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)},
// TODO: m.room.canonical_alias
{"m.room.join_rules", "", events.JoinRulesContent{"public"}}, // FIXME: Allow this to be changed
{"m.room.history_visibility", "", events.HistoryVisibilityContent{"joined"}}, // FIXME: Allow this to be changed
{"m.room.guest_access", "", events.GuestAccessContent{"can_join"}}, // FIXME: Allow this to be changed
{"m.room.join_rules", "", common.JoinRulesContent{"public"}}, // FIXME: Allow this to be changed
{"m.room.history_visibility", "", common.HistoryVisibilityContent{"joined"}}, // FIXME: Allow this to be changed
{"m.room.guest_access", "", common.GuestAccessContent{"can_join"}}, // FIXME: Allow this to be changed
// TODO: Other initial state items
{"m.room.name", "", events.NameContent{r.Name}}, // FIXME: Only send the name event if a name is supplied, to avoid sending a false room name removal event
{"m.room.topic", "", events.TopicContent{r.Topic}},
{"m.room.name", "", common.NameContent{r.Name}}, // FIXME: Only send the name event if a name is supplied, to avoid sending a false room name removal event
{"m.room.topic", "", common.TopicContent{r.Topic}},
// TODO: invite events
// TODO: 3pid invite events
// TODO: m.room.aliases
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/httputil"
"github.com/matrix-org/dendrite/clientapi/jsonerror"
"github.com/matrix-org/dendrite/clientapi/producers"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib"
Expand Down Expand Up @@ -69,7 +70,7 @@ func SendMembership(
membership = "leave"
}

content := events.MemberContent{
content := common.MemberContent{
Membership: membership,
DisplayName: profile.DisplayName,
AvatarURL: profile.AvatarURL,
Expand Down
33 changes: 24 additions & 9 deletions src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,13 @@ Arguments:
`

var (
syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'")
clientAPIURL = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'")
mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'")
bindAddress = flag.String("bind-address", ":8008", "The listening port for the proxy.")
certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS")
syncServerURL = flag.String("sync-api-server-url", "", "The base URL of the listening 'dendrite-sync-api-server' process. E.g. 'http://localhost:4200'")
clientAPIURL = flag.String("client-api-server-url", "", "The base URL of the listening 'dendrite-client-api-server' process. E.g. 'http://localhost:4321'")
mediaAPIURL = flag.String("media-api-server-url", "", "The base URL of the listening 'dendrite-media-api-server' process. E.g. 'http://localhost:7779'")
publicRoomsAPIURL = flag.String("public-rooms-api-server-url", "", "The base URL of the listening 'dendrite-public-rooms-api-server' process. E.g. 'http://localhost:7775'")
bindAddress = flag.String("bind-address", ":8008", "The listening port for the proxy.")
certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS")
keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS")
)

func makeProxy(targetURL string) (*httputil.ReverseProxy, error) {
Expand Down Expand Up @@ -122,6 +123,12 @@ func main() {
os.Exit(1)
}

if *publicRoomsAPIURL == "" {
flag.Usage()
fmt.Fprintln(os.Stderr, "no --public-rooms-api-server-url specified.")
os.Exit(1)
}

syncProxy, err := makeProxy(*syncServerURL)
if err != nil {
panic(err)
Expand All @@ -134,8 +141,14 @@ func main() {
if err != nil {
panic(err)
}
publicRoomsProxy, err := makeProxy(*publicRoomsAPIURL)
if err != nil {
panic(err)
}

http.Handle("/_matrix/client/r0/sync", syncProxy)
http.Handle("/_matrix/client/r0/directory/list/", publicRoomsProxy)
http.Handle("/_matrix/client/r0/publicRooms", publicRoomsProxy)
http.Handle("/_matrix/media/v1/", mediaProxy)
http.Handle("/", clientProxy)

Expand All @@ -146,9 +159,11 @@ func main() {
}

fmt.Println("Proxying requests to:")
fmt.Println(" /_matrix/client/r0/sync => ", *syncServerURL+"/api/_matrix/client/r0/sync")
fmt.Println(" /_matrix/media/v1 => ", *mediaAPIURL+"/api/_matrix/media/v1")
fmt.Println(" /* => ", *clientAPIURL+"/api/*")
fmt.Println(" /_matrix/client/r0/sync => ", *syncServerURL+"/api/_matrix/client/r0/sync")
fmt.Println(" /_matrix/client/r0/directory/list => ", *publicRoomsAPIURL+"/_matrix/client/r0/directory/list")
fmt.Println(" /_matrix/client/r0/publicRooms => ", *publicRoomsAPIURL+"/_matrix/media/client/r0/publicRooms")
fmt.Println(" /_matrix/media/v1 => ", *mediaAPIURL+"/api/_matrix/media/v1")
fmt.Println(" /* => ", *clientAPIURL+"/api/*")
fmt.Println("Listening on ", *bindAddress)
if *certFile != "" && *keyFile != "" {
panic(srv.ListenAndServeTLS(*certFile, *keyFile))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ import (
"github.com/matrix-org/dendrite/federationsender/queue"
federationsender_storage "github.com/matrix-org/dendrite/federationsender/storage"

publicroomsapi_consumers "github.com/matrix-org/dendrite/publicroomsapi/consumers"
publicroomsapi_routing "github.com/matrix-org/dendrite/publicroomsapi/routing"
publicroomsapi_storage "github.com/matrix-org/dendrite/publicroomsapi/storage"

log "github.com/Sirupsen/logrus"
sarama "gopkg.in/Shopify/sarama.v1"
)
Expand Down Expand Up @@ -119,6 +123,7 @@ type monolith struct {
mediaAPIDB *mediaapi_storage.Database
syncAPIDB *syncapi_storage.SyncServerDatabase
federationSenderDB *federationsender_storage.Database
publicRoomsAPIDB *publicroomsapi_storage.PublicRoomsServerDatabase

federation *gomatrixserverlib.FederationClient
keyRing gomatrixserverlib.KeyRing
Expand Down Expand Up @@ -171,6 +176,10 @@ func (m *monolith) setupDatabases() {
if err != nil {
log.Panicf("startup: failed to create federation sender database with data source %s : %s", m.cfg.Database.FederationSender, err)
}
m.publicRoomsAPIDB, err = publicroomsapi_storage.NewPublicRoomsServerDatabase(string(m.cfg.Database.PublicRoomsAPI))
if err != nil {
log.Panicf("startup: failed to setup public rooms api database with data source %s : %s", m.cfg.Database.PublicRoomsAPI, err)
}
}

func (m *monolith) setupFederation() {
Expand Down Expand Up @@ -290,6 +299,13 @@ func (m *monolith) setupConsumers() {
log.Panicf("startup: failed to start client API server consumer: %s", err)
}

publicRoomsAPIConsumer := publicroomsapi_consumers.NewOutputRoomEvent(
m.cfg, m.kafkaConsumer(), m.publicRoomsAPIDB, m.queryAPI,
)
if err = publicRoomsAPIConsumer.Start(); err != nil {
log.Panicf("startup: failed to start room server consumer: %s", err)
}

federationSenderQueues := queue.NewOutgoingQueues(m.cfg.Matrix.ServerName, m.federation)

federationSenderRoomConsumer := federationsender_consumers.NewOutputRoomEvent(
Expand Down Expand Up @@ -318,4 +334,6 @@ func (m *monolith) setupAPIs() {
federationapi_routing.Setup(
m.api, *m.cfg, m.queryAPI, m.roomServerProducer, m.keyRing, m.federation,
)

publicroomsapi_routing.Setup(m.api, m.deviceDB, m.publicRoomsAPIDB)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"flag"
"net/http"
"os"

"github.com/gorilla/mux"
"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
"github.com/matrix-org/dendrite/common"
"github.com/matrix-org/dendrite/common/config"
"github.com/matrix-org/dendrite/publicroomsapi/consumers"
"github.com/matrix-org/dendrite/publicroomsapi/routing"
"github.com/matrix-org/dendrite/publicroomsapi/storage"
"github.com/matrix-org/dendrite/roomserver/api"

log "github.com/Sirupsen/logrus"
sarama "gopkg.in/Shopify/sarama.v1"
)

var configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.")

func main() {
common.SetupLogging(os.Getenv("LOG_DIR"))

flag.Parse()

if *configPath == "" {
log.Fatal("--config must be supplied")
}
cfg, err := config.Load(*configPath)
if err != nil {
log.Fatalf("Invalid config file: %s", err)
}

queryAPI := api.NewRoomserverQueryAPIHTTP(cfg.RoomServerURL(), nil)

db, err := storage.NewPublicRoomsServerDatabase(string(cfg.Database.PublicRoomsAPI))
if err != nil {
log.Panicf("startup: failed to create public rooms server database with data source %s : %s", cfg.Database.PublicRoomsAPI, err)
}

deviceDB, err := devices.NewDatabase(string(cfg.Database.Device), cfg.Matrix.ServerName)
if err != nil {
log.Panicf("startup: failed to create device database with data source %s : %s", cfg.Database.Device, err)
}

kafkaConsumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil)
if err != nil {
log.WithFields(log.Fields{
log.ErrorKey: err,
"addresses": cfg.Kafka.Addresses,
}).Panic("Failed to setup kafka consumers")
}

roomConsumer := consumers.NewOutputRoomEvent(cfg, kafkaConsumer, db, queryAPI)
if err != nil {
log.Panicf("startup: failed to create room server consumer: %s", err)
}
if err = roomConsumer.Start(); err != nil {
log.Panicf("startup: failed to start room server consumer: %s", err)
}

log.Info("Starting public rooms server on ", cfg.Listen.PublicRoomsAPI)

api := mux.NewRouter()
routing.Setup(api, deviceDB, db)
common.SetupHTTPAPI(http.DefaultServeMux, api)

log.Fatal(http.ListenAndServe(string(cfg.Listen.PublicRoomsAPI), nil))
}
4 changes: 4 additions & 0 deletions src/github.com/matrix-org/dendrite/common/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,9 @@ type Dendrite struct {
// The FederationSender database stores information used by the FederationSender
// It is only accessed by the FederationSender.
FederationSender DataSource `yaml:"federation_sender"`
// The PublicRoomsAPI database stores information used to compute the public
// room directory. It is only accessed by the PublicRoomsAPI server.
PublicRoomsAPI DataSource `yaml:"public_rooms_api"`
} `yaml:"database"`

// The internal addresses the components will listen on.
Expand All @@ -144,6 +147,7 @@ type Dendrite struct {
SyncAPI Address `yaml:"sync_api"`
RoomServer Address `yaml:"room_server"`
FederationSender Address `yaml:"federation_sender"`
PublicRoomsAPI Address `yaml:"public_rooms_api"`
} `yaml:"listen"`
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package events
package common

// CreateContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-create
type CreateContent struct {
Expand Down Expand Up @@ -90,3 +90,29 @@ func InitialPowerLevelsContent(roomCreator string) PowerLevelContent {
Users: map[string]int{roomCreator: 100},
}
}

// AliasesContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-aliases
type AliasesContent struct {
Aliases []string `json:"aliases"`
}

// CanonicalAliasContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-canonical-alias
type CanonicalAliasContent struct {
Alias string `json:"alias"`
}

// AvatarContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-avatar
type AvatarContent struct {
Info ImageInfo `json:"info,omitempty"`
URL string `json:"url"`
ThumbnailURL string `json:"thumbnail_url,omitempty"`
ThumbnailInfo ImageInfo `json:"thumbnail_info,omitempty"`
}

// ImageInfo implements the ImageInfo structure from http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-avatar
type ImageInfo struct {
Mimetype string `json:"mimetype"`
Height int64 `json:"h"`
Width int64 `json:"w"`
Size int64 `json:"size"`
}
2 changes: 2 additions & 0 deletions src/github.com/matrix-org/dendrite/common/test/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,14 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con
cfg.Database.RoomServer = config.DataSource(database)
cfg.Database.ServerKey = config.DataSource(database)
cfg.Database.SyncAPI = config.DataSource(database)
cfg.Database.PublicRoomsAPI = config.DataSource(database)

cfg.Listen.ClientAPI = assignAddress()
cfg.Listen.FederationAPI = assignAddress()
cfg.Listen.MediaAPI = assignAddress()
cfg.Listen.RoomServer = assignAddress()
cfg.Listen.SyncAPI = assignAddress()
cfg.Listen.PublicRoomsAPI = assignAddress()

return &cfg, port, nil
}
Expand Down
1 change: 1 addition & 0 deletions src/github.com/matrix-org/dendrite/common/test/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) {
"--sync-api-server-url", "http://" + string(cfg.Listen.SyncAPI),
"--client-api-server-url", "http://" + string(cfg.Listen.ClientAPI),
"--media-api-server-url", "http://" + string(cfg.Listen.MediaAPI),
"--public-rooms-api-server-url", "http://" + string(cfg.Listen.PublicRoomsAPI),
"--tls-cert", "server.crt",
"--tls-key", "server.key",
}
Expand Down
5 changes: 5 additions & 0 deletions src/github.com/matrix-org/dendrite/publicroomsapi/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Public rooms API

This server is responsible for serving requests hitting `/publicRooms` and `/directory/list/room/{roomID}` as per:

https://matrix.org/docs/spec/client_server/r0.2.0.html#listing-rooms
Loading

0 comments on commit b15ce90

Please sign in to comment.