Skip to content

Commit

Permalink
encode all emojis on message to simplify unbox (keybase#23343)
Browse files Browse the repository at this point in the history
* bring over good stuff

* works
  • Loading branch information
mmaxim authored Mar 28, 2020
1 parent b1d0c13 commit 66cb473
Show file tree
Hide file tree
Showing 10 changed files with 293 additions and 158 deletions.
52 changes: 15 additions & 37 deletions go/chat/boxer.go
Original file line number Diff line number Diff line change
Expand Up @@ -690,12 +690,6 @@ func (b *Boxer) unboxV1(ctx context.Context, boxed chat1.MessageBoxed,
// Get at mention usernames
atMentions, atMentionUsernames, maybeRes, chanMention, channelNameMentions :=
b.getAtMentionInfo(ctx, clientHeader.Conv.Tlfid, clientHeader.Conv.TopicType, conv, body)
// Get emojis
var emojis []chat1.HarvestedEmoji
emojis, ierr = b.getEmojis(ctx, conv, body, clientHeader.Conv.TopicType)
if ierr != nil {
return nil, ierr
}

ierr = b.compareHeadersMBV1(ctx, boxed.ClientHeader, clientHeader)
if ierr != nil {
Expand All @@ -721,7 +715,7 @@ func (b *Boxer) unboxV1(ctx context.Context, boxed chat1.MessageBoxed,
ChannelNameMentions: channelNameMentions,
MaybeMentions: maybeRes,
BotUsername: b.getBotInfoLocal(ctx, clientHeader.BotUID),
Emojis: emojis,
Emojis: b.getEmojis(ctx, clientHeader.Conv.TopicType, body),
}, nil
}

Expand Down Expand Up @@ -990,12 +984,6 @@ func (b *Boxer) unboxV2orV3orV4(ctx context.Context, boxed chat1.MessageBoxed,
// Get at mention usernames
atMentions, atMentionUsernames, maybeRes, chanMention, channelNameMentions :=
b.getAtMentionInfo(ctx, clientHeader.Conv.Tlfid, clientHeader.Conv.TopicType, conv, body)
// Get emojis
var emojis []chat1.HarvestedEmoji
emojis, ierr = b.getEmojis(ctx, conv, body, clientHeader.Conv.TopicType)
if ierr != nil {
return nil, ierr
}

clientHeader.HasPairwiseMacs = len(boxed.ClientHeader.PairwiseMacs) > 0

Expand All @@ -1018,7 +1006,7 @@ func (b *Boxer) unboxV2orV3orV4(ctx context.Context, boxed chat1.MessageBoxed,
ChannelNameMentions: channelNameMentions,
MaybeMentions: maybeRes,
BotUsername: b.getBotInfoLocal(ctx, clientHeader.BotUID),
Emojis: emojis,
Emojis: b.getEmojis(ctx, clientHeader.Conv.TopicType, body),
}, nil
}

Expand Down Expand Up @@ -1309,38 +1297,28 @@ func (b *Boxer) getBotInfoLocal(ctx context.Context, uid *gregor1.UID) string {
return username
}

func (b *Boxer) getEmojis(ctx context.Context, conv types.UnboxConversationInfo, body chat1.MessageBody,
topicType chat1.TopicType) ([]chat1.HarvestedEmoji, types.UnboxingError) {
func (b *Boxer) getEmojis(ctx context.Context, topicType chat1.TopicType, body chat1.MessageBody) (res []chat1.HarvestedEmoji) {
if topicType != chat1.TopicType_CHAT {
return nil, nil
return nil
}
uid := gregor1.UID(b.G().GetEnv().GetUID().ToBytes())
typ, err := body.MessageType()
if err != nil {
return nil, nil
return nil
}
makeList := func(m map[string]chat1.HarvestedEmoji) (l []chat1.HarvestedEmoji) {
l = make([]chat1.HarvestedEmoji, 0, len(m))
for _, e := range m {
l = append(l, e)
}
return l
}
switch typ {
case chat1.MessageType_TEXT:
text := body.Text()
emojis, err := b.G().EmojiSource.Harvest(ctx, text.Body, uid, conv.GetConvID(), text.Emojis,
types.EmojiSourceHarvestModeInbound)
if err != nil {
return nil, NewTransientUnboxingError(err)
}
b.Debug(ctx, "getEmojis: found %d emojis (text)", len(emojis))
return emojis, nil
return makeList(body.Text().Emojis)
case chat1.MessageType_REACTION:
reaction := body.Reaction()
emojis, err := b.G().EmojiSource.Harvest(ctx, reaction.Body, uid, conv.GetConvID(), reaction.Emojis,
types.EmojiSourceHarvestModeInbound)
if err != nil {
return nil, NewTransientUnboxingError(err)
}
b.Debug(ctx, "getEmojis: found %d emojis (reaction)", len(emojis))
return emojis, nil
default:
return makeList(body.Reaction().Emojis)
}
return nil, nil
return res
}

func (b *Boxer) getAtMentionInfo(ctx context.Context, tlfID chat1.TLFID, topicType chat1.TopicType,
Expand Down
60 changes: 32 additions & 28 deletions go/chat/emojisource.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/keybase/client/go/protocol/chat1"
"github.com/keybase/client/go/protocol/gregor1"
"github.com/keybase/client/go/protocol/keybase1"
"github.com/kyokomi/emoji"
)

type DevConvEmojiSource struct {
Expand Down Expand Up @@ -81,11 +82,30 @@ func (s *DevConvEmojiSource) addAdvanced(ctx context.Context, uid gregor1.UID, c
return res, storage.Put(ctx, uid, convID, topicName, stored)
}

func (s *DevConvEmojiSource) isStockEmoji(alias string) bool {
_, ok := emoji.CodeMap()[":"+alias+":"]
if !ok {
_, ok = emoji.CodeMap()[":"+strings.ReplaceAll(alias, "-", "_")+":"]
}
return ok
}

func (s *DevConvEmojiSource) validateAlias(alias string) (string, error) {
alias = strings.ReplaceAll(alias, ":", "") // drop any colons from alias
if strings.Contains(alias, "#") {
return alias, errors.New("invalid character in emoji alias")
}
if s.isStockEmoji(alias) {
return alias, errors.New("cannot use existing stock emoji alias")
}
return alias, nil
}

func (s *DevConvEmojiSource) Add(ctx context.Context, uid gregor1.UID, convID chat1.ConversationID,
alias, filename string) (res chat1.EmojiRemoteSource, err error) {
defer s.Trace(ctx, func() error { return err }, "Add")()
if strings.Contains(alias, "#") {
return res, errors.New("invalid character in emoji alias")
if alias, err = s.validateAlias(alias); err != nil {
return res, err
}
storage := s.makeStorage(chat1.TopicType_EMOJI)
return s.addAdvanced(ctx, uid, convID, alias, filename, nil, storage)
Expand All @@ -94,8 +114,8 @@ func (s *DevConvEmojiSource) Add(ctx context.Context, uid gregor1.UID, convID ch
func (s *DevConvEmojiSource) AddAlias(ctx context.Context, uid gregor1.UID, convID chat1.ConversationID,
newAlias, existingAlias string) (res chat1.EmojiRemoteSource, err error) {
defer s.Trace(ctx, func() error { return err }, "AddAlias")()
if strings.Contains(newAlias, "#") {
return res, errors.New("invalid character in emoji alias")
if newAlias, err = s.validateAlias(newAlias); err != nil {
return res, err
}
var stored chat1.EmojiStorage
storage := s.makeStorage(chat1.TopicType_EMOJI)
Expand Down Expand Up @@ -482,8 +502,7 @@ func (s *DevConvEmojiSource) syncCrossTeam(ctx context.Context, uid gregor1.UID,
}

func (s *DevConvEmojiSource) Harvest(ctx context.Context, body string, uid gregor1.UID,
convID chat1.ConversationID, crossTeams map[string]chat1.HarvestedEmoji,
mode types.EmojiSourceHarvestMode) (res []chat1.HarvestedEmoji, err error) {
convID chat1.ConversationID) (res []chat1.HarvestedEmoji, err error) {
if globals.IsEmojiHarvesterCtx(ctx) {
s.Debug(ctx, "Harvest: in an existing harvest context, bailing")
return nil, nil
Expand All @@ -493,7 +512,7 @@ func (s *DevConvEmojiSource) Harvest(ctx context.Context, body string, uid grego
return nil, nil
}
ctx = globals.CtxMakeEmojiHarvester(ctx)
defer s.Trace(ctx, func() error { return err }, "Harvest: mode: %v", mode)()
defer s.Trace(ctx, func() error { return err }, "Harvest")()
s.Debug(ctx, "Harvest: %d matches found", len(matches))
emojis, _, err := s.getNoSet(ctx, uid, &convID, chat1.EmojiFetchOpts{
GetCreationInfo: false,
Expand All @@ -503,7 +522,7 @@ func (s *DevConvEmojiSource) Harvest(ctx context.Context, body string, uid grego
if err != nil {
return res, err
}
if len(emojis.Emojis) == 0 && len(crossTeams) == 0 {
if len(emojis.Emojis) == 0 {
return nil, nil
}
groupMap := make(map[string]chat1.Emoji)
Expand All @@ -512,24 +531,13 @@ func (s *DevConvEmojiSource) Harvest(ctx context.Context, body string, uid grego
groupMap[emoji.Alias] = emoji
}
}
crossTeamMap := make(map[string]chat1.HarvestedEmoji)
aliasMap := make(map[string]chat1.Emoji)
switch mode {
case types.EmojiSourceHarvestModeInbound:
for _, emoji := range crossTeams {
crossTeamMap[emoji.Alias] = emoji
}
case types.EmojiSourceHarvestModeOutbound:
s.getLock.Lock()
for alias, emoji := range s.aliasLookup {
aliasMap[alias] = emoji
}
s.getLock.Unlock()
default:
return nil, errors.New("unknown harvest mode")
s.getLock.Lock()
for alias, emoji := range s.aliasLookup {
aliasMap[alias] = emoji
}
s.Debug(ctx, "Harvest: num emojis: conv: %d crossTeam: %d alias: %d", len(groupMap), len(crossTeamMap),
len(aliasMap))
s.getLock.Unlock()
s.Debug(ctx, "Harvest: num emojis: conv: %d alias: %d", len(groupMap), len(aliasMap))
for _, match := range matches {
// try group map first
if emoji, ok := groupMap[match.name]; ok {
Expand All @@ -549,10 +557,6 @@ func (s *DevConvEmojiSource) Harvest(ctx context.Context, body string, uid grego
}
}
res = append(res, resEmoji)
} else if emoji, ok := crossTeamMap[match.name]; ok {
// then known cross teams
emoji.IsCrossTeam = true
res = append(res, emoji)
} else if emoji, ok := aliasMap[match.name]; ok {
// then any aliases we know about from the last Get call
newEmoji, err := s.syncCrossTeam(ctx, uid, chat1.HarvestedEmoji{
Expand Down
13 changes: 11 additions & 2 deletions go/chat/localizer.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,17 @@ func (b *blockingLocalizer) Localize(ctx context.Context, uid gregor1.UID, inbox
for index, c := range convs {
indexMap[c.ConvIDStr] = index
}
for ar := range b.localizeCb {
res[indexMap[ar.ConvLocal.GetConvID().ConvIDStr()]] = ar.ConvLocal
doneCb := make(chan struct{})
go func() {
for ar := range b.localizeCb {
res[indexMap[ar.ConvLocal.GetConvID().ConvIDStr()]] = ar.ConvLocal
}
close(doneCb)
}()
select {
case <-doneCb:
case <-ctx.Done():
return res, ctx.Err()
}
return res, nil
}
Expand Down
16 changes: 7 additions & 9 deletions go/chat/sender.go
Original file line number Diff line number Diff line change
Expand Up @@ -569,14 +569,14 @@ func (s *BlockingSender) handleReplyTo(ctx context.Context, uid gregor1.UID, con
return msg, nil
}

func (s *BlockingSender) handleCrossTeamEmojis(ctx context.Context, uid gregor1.UID,
func (s *BlockingSender) handleEmojis(ctx context.Context, uid gregor1.UID,
convID chat1.ConversationID, msg chat1.MessagePlaintext, topicType chat1.TopicType) (chat1.MessagePlaintext, error) {
if topicType != chat1.TopicType_CHAT {
return msg, nil
}
typ, err := msg.MessageBody.MessageType()
if err != nil {
s.Debug(ctx, "handleCrossTeamEmojis: failed to get body type: %s", err)
s.Debug(ctx, "handleEmojis: failed to get body type: %s", err)
return msg, nil
}
var body string
Expand All @@ -588,20 +588,18 @@ func (s *BlockingSender) handleCrossTeamEmojis(ctx context.Context, uid gregor1.
default:
return msg, nil
}
ct := make(map[string]chat1.HarvestedEmoji)
emojis, err := s.G().EmojiSource.Harvest(ctx, body, uid, convID, ct, types.EmojiSourceHarvestModeOutbound)
emojis, err := s.G().EmojiSource.Harvest(ctx, body, uid, convID)
if err != nil {
return msg, err
}
ct := make(map[string]chat1.HarvestedEmoji)
for _, emoji := range emojis {
if emoji.IsCrossTeam {
ct[emoji.Alias] = emoji
}
ct[emoji.Alias] = emoji
}
if len(ct) == 0 {
return msg, nil
}
s.Debug(ctx, "handleCrossTeamEmojis: found %d cross team emojis", len(ct))
s.Debug(ctx, "handleEmojis: found %d emojis", len(ct))
switch typ {
case chat1.MessageType_TEXT:
newBody := msg.MessageBody.Text().DeepCopy()
Expand Down Expand Up @@ -840,7 +838,7 @@ func (s *BlockingSender) Prepare(ctx context.Context, plaintext chat1.MessagePla
}

// Handle cross team emoji
if msg, err = s.handleCrossTeamEmojis(ctx, uid, convID, msg, conv.Info.Triple.TopicType); err != nil {
if msg, err = s.handleEmojis(ctx, uid, convID, msg, conv.Info.Triple.TopicType); err != nil {
s.Debug(ctx, "Prepare: error processing cross team emoji: %s", err)
return res, err
}
Expand Down
3 changes: 1 addition & 2 deletions go/chat/types/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -619,8 +619,7 @@ type EmojiSource interface {
Remove(ctx context.Context, uid gregor1.UID, convID chat1.ConversationID, alias string) error
Get(ctx context.Context, uid gregor1.UID, convID *chat1.ConversationID, opts chat1.EmojiFetchOpts) (chat1.UserEmojis, error)
Decorate(ctx context.Context, body string, convID chat1.ConversationID, emojis []chat1.HarvestedEmoji) string
Harvest(ctx context.Context, body string, uid gregor1.UID, convID chat1.ConversationID,
crossTeams map[string]chat1.HarvestedEmoji, mode EmojiSourceHarvestMode) ([]chat1.HarvestedEmoji, error)
Harvest(ctx context.Context, body string, uid gregor1.UID, convID chat1.ConversationID) ([]chat1.HarvestedEmoji, error)
}

type ServerConnection interface {
Expand Down
10 changes: 1 addition & 9 deletions go/chat/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -382,13 +382,6 @@ type ParticipantResult struct {
Err error
}

type EmojiSourceHarvestMode int

const (
EmojiSourceHarvestModeOutbound EmojiSourceHarvestMode = iota
EmojiSourceHarvestModeInbound
)

type DummyAttachmentFetcher struct{}

var _ AttachmentFetcher = (*DummyAttachmentFetcher)(nil)
Expand Down Expand Up @@ -821,7 +814,6 @@ func (DummyEmojiSource) Decorate(ctx context.Context, body string, convID chat1.
return body
}
func (DummyEmojiSource) Harvest(ctx context.Context, body string, uid gregor1.UID,
convID chat1.ConversationID, crossTeams map[string]chat1.HarvestedEmoji,
mode EmojiSourceHarvestMode) (res []chat1.HarvestedEmoji, err error) {
convID chat1.ConversationID) (res []chat1.HarvestedEmoji, err error) {
return res, err
}
2 changes: 1 addition & 1 deletion go/vendor/github.com/kyokomi/emoji/README.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 66cb473

Please sign in to comment.