From 8a9c4501fcb0d145aaeed4dd69c39b6a9120cf5c Mon Sep 17 00:00:00 2001 From: James Houlahan Date: Fri, 5 Aug 2022 12:53:29 +0200 Subject: [PATCH] feat(GODT-1611): Handle optional charset in SEARCH command The IMAP SEARCH command supports an optional charset. If this is provided, search terms should be decoded before being compared with search targets. In addition, RFC3501 says that search targets should be decoded before comparison, but this is out of scope for this PR, as it requires parsing and decoding of messages and addresses. --- internal/backend/mailbox_search.go | 175 ++++++++++++------- internal/parser/proto/imap.pb.go | 10 +- internal/parser/proto/imap.proto | 2 +- internal/parser/tests/parser/parser_test.cpp | 30 ++-- internal/response/item_badcharset.go | 11 ++ internal/session/handle_search.go | 19 +- tests/connection_test.go | 15 ++ tests/search_test.go | 56 +++++- 8 files changed, 223 insertions(+), 95 deletions(-) create mode 100644 internal/response/item_badcharset.go diff --git a/internal/backend/mailbox_search.go b/internal/backend/mailbox_search.go index 6c7907ff..0a44efc9 100644 --- a/internal/backend/mailbox_search.go +++ b/internal/backend/mailbox_search.go @@ -11,10 +11,11 @@ import ( "github.com/ProtonMail/gluon/internal/parser/proto" "github.com/ProtonMail/gluon/rfc822" "github.com/bradenaw/juniper/xslices" + "golang.org/x/text/encoding" ) -func (m *Mailbox) Search(ctx context.Context, keys []*proto.SearchKey) ([]int, error) { - messages, err := doSearch(ctx, m, m.snap.getAllMessages(), keys) +func (m *Mailbox) Search(ctx context.Context, keys []*proto.SearchKey, decoder *encoding.Decoder) ([]int, error) { + messages, err := doSearch(ctx, m, m.snap.getAllMessages(), keys, decoder) if err != nil { return nil, err } @@ -28,9 +29,9 @@ func (m *Mailbox) Search(ctx context.Context, keys []*proto.SearchKey) ([]int, e }), nil } -func doSearch(ctx context.Context, m *Mailbox, candidates []*snapMsg, keys []*proto.SearchKey) ([]*snapMsg, error) { +func doSearch(ctx context.Context, m *Mailbox, candidates []*snapMsg, keys []*proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { for _, key := range keys { - filtered, err := m.matchSearchKey(ctx, candidates, key) + filtered, err := m.matchSearchKey(ctx, candidates, key, decoder) if err != nil { return nil, err } @@ -41,40 +42,40 @@ func doSearch(ctx context.Context, m *Mailbox, candidates []*snapMsg, keys []*pr return candidates, nil } -func (m *Mailbox) matchSearchKey(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKey(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { switch key.Keyword { case proto.SearchKeyword_SearchKWAll: - return m.matchSearchKeyAll(ctx, candidates, key) + return m.matchSearchKeyAll(ctx, candidates) case proto.SearchKeyword_SearchKWAnswered: - return m.matchSearchKeyAnswered(ctx, candidates, key) + return m.matchSearchKeyAnswered(ctx, candidates) case proto.SearchKeyword_SearchKWBcc: - return m.matchSearchKeyBcc(ctx, candidates, key) + return m.matchSearchKeyBcc(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWBefore: return m.matchSearchKeyBefore(ctx, candidates, key) case proto.SearchKeyword_SearchKWBody: - return m.matchSearchKeyBody(ctx, candidates, key) + return m.matchSearchKeyBody(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWCc: - return m.matchSearchKeyCc(ctx, candidates, key) + return m.matchSearchKeyCc(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWDeleted: - return m.matchSearchKeyDeleted(ctx, candidates, key) + return m.matchSearchKeyDeleted(ctx, candidates) case proto.SearchKeyword_SearchKWDraft: - return m.matchSearchKeyDraft(ctx, candidates, key) + return m.matchSearchKeyDraft(ctx, candidates) case proto.SearchKeyword_SearchKWFlagged: - return m.matchSearchKeyFlagged(ctx, candidates, key) + return m.matchSearchKeyFlagged(ctx, candidates) case proto.SearchKeyword_SearchKWFrom: - return m.matchSearchKeyFrom(ctx, candidates, key) + return m.matchSearchKeyFrom(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWHeader: - return m.matchSearchKeyHeader(ctx, candidates, key) + return m.matchSearchKeyHeader(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWKeyword: return m.matchSearchKeyKeyword(ctx, candidates, key) @@ -83,25 +84,25 @@ func (m *Mailbox) matchSearchKey(ctx context.Context, candidates []*snapMsg, key return m.matchSearchKeyLarger(ctx, candidates, key) case proto.SearchKeyword_SearchKWNew: - return m.matchSearchKeyNew(ctx, candidates, key) + return m.matchSearchKeyNew(ctx, candidates) case proto.SearchKeyword_SearchKWNot: - return m.matchSearchKeyNot(ctx, candidates, key) + return m.matchSearchKeyNot(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWOld: - return m.matchSearchKeyOld(ctx, candidates, key) + return m.matchSearchKeyOld(ctx, candidates) case proto.SearchKeyword_SearchKWOn: return m.matchSearchKeyOn(ctx, candidates, key) case proto.SearchKeyword_SearchKWOr: - return m.matchSearchKeyOr(ctx, candidates, key) + return m.matchSearchKeyOr(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWRecent: - return m.matchSearchKeyRecent(ctx, candidates, key) + return m.matchSearchKeyRecent(ctx, candidates) case proto.SearchKeyword_SearchKWSeen: - return m.matchSearchKeySeen(ctx, candidates, key) + return m.matchSearchKeySeen(ctx, candidates) case proto.SearchKeyword_SearchKWSentBefore: return m.matchSearchKeySentBefore(ctx, candidates, key) @@ -119,57 +120,57 @@ func (m *Mailbox) matchSearchKey(ctx context.Context, candidates []*snapMsg, key return m.matchSearchKeySmaller(ctx, candidates, key) case proto.SearchKeyword_SearchKWSubject: - return m.matchSearchKeySubject(ctx, candidates, key) + return m.matchSearchKeySubject(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWText: - return m.matchSearchKeyText(ctx, candidates, key) + return m.matchSearchKeyText(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWTo: - return m.matchSearchKeyTo(ctx, candidates, key) + return m.matchSearchKeyTo(ctx, candidates, key, decoder) case proto.SearchKeyword_SearchKWUID: return m.matchSearchKeyUID(ctx, candidates, key) case proto.SearchKeyword_SearchKWUnanswered: - return m.matchSearchKeyUnanswered(ctx, candidates, key) + return m.matchSearchKeyUnanswered(ctx, candidates) case proto.SearchKeyword_SearchKWUndeleted: - return m.matchSearchKeyUndeleted(ctx, candidates, key) + return m.matchSearchKeyUndeleted(ctx, candidates) case proto.SearchKeyword_SearchKWUndraft: - return m.matchSearchKeyUndraft(ctx, candidates, key) + return m.matchSearchKeyUndraft(ctx, candidates) case proto.SearchKeyword_SearchKWUnflagged: - return m.matchSearchKeyUnflagged(ctx, candidates, key) + return m.matchSearchKeyUnflagged(ctx, candidates) case proto.SearchKeyword_SearchKWUnkeyword: return m.matchSearchKeyUnkeyword(ctx, candidates, key) case proto.SearchKeyword_SearchKWUnseen: - return m.matchSearchKeyUnseen(ctx, candidates, key) + return m.matchSearchKeyUnseen(ctx, candidates) case proto.SearchKeyword_SearchKWSeqSet: return m.matchSearchKeySeqSet(ctx, candidates, key) case proto.SearchKeyword_SearchKWList: - return m.matchSearchKeyList(ctx, candidates, key) + return m.matchSearchKeyList(ctx, candidates, key, decoder) default: panic("bad keyword") } } -func (m *Mailbox) matchSearchKeyAll(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyAll(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return candidates, nil } -func (m *Mailbox) matchSearchKeyAnswered(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyAnswered(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return message.flags.Contains(imap.FlagAnswered), nil }) } -func (m *Mailbox) matchSearchKeyBcc(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyBcc(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { literal, err := m.state.getLiteral(message.ID) if err != nil { @@ -181,7 +182,12 @@ func (m *Mailbox) matchSearchKeyBcc(ctx context.Context, candidates []*snapMsg, return false, err } - return strings.Contains(strings.ToLower(value), strings.ToLower(key.GetText())), nil + b, err := decoder.Bytes(key.GetText()) + if err != nil { + return false, err + } + + return strings.Contains(strings.ToLower(value), strings.ToLower(string(b))), nil }) } @@ -201,7 +207,7 @@ func (m *Mailbox) matchSearchKeyBefore(ctx context.Context, candidates []*snapMs }) } -func (m *Mailbox) matchSearchKeyBody(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyBody(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { literal, err := m.state.getLiteral(message.ID) if err != nil { @@ -213,11 +219,16 @@ func (m *Mailbox) matchSearchKeyBody(ctx context.Context, candidates []*snapMsg, return false, err } - return bytes.Contains([]byte(strings.ToLower(string(section.Body()))), []byte(strings.ToLower(key.GetText()))), nil + b, err := decoder.Bytes(key.GetText()) + if err != nil { + return false, err + } + + return bytes.Contains(bytes.ToLower(section.Body()), bytes.ToLower(b)), nil }) } -func (m *Mailbox) matchSearchKeyCc(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyCc(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { literal, err := m.state.getLiteral(message.ID) if err != nil { @@ -229,29 +240,34 @@ func (m *Mailbox) matchSearchKeyCc(ctx context.Context, candidates []*snapMsg, k return false, err } - return strings.Contains(strings.ToLower(value), strings.ToLower(key.GetText())), nil + b, err := decoder.Bytes(key.GetText()) + if err != nil { + return false, err + } + + return strings.Contains(strings.ToLower(value), strings.ToLower(string(b))), nil }) } -func (m *Mailbox) matchSearchKeyDeleted(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyDeleted(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return message.flags.Contains(imap.FlagDeleted), nil }) } -func (m *Mailbox) matchSearchKeyDraft(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyDraft(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return message.flags.Contains(imap.FlagDraft), nil }) } -func (m *Mailbox) matchSearchKeyFlagged(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyFlagged(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return message.flags.Contains(imap.FlagFlagged), nil }) } -func (m *Mailbox) matchSearchKeyFrom(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyFrom(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { literal, err := m.state.getLiteral(message.ID) if err != nil { @@ -263,11 +279,16 @@ func (m *Mailbox) matchSearchKeyFrom(ctx context.Context, candidates []*snapMsg, return false, err } - return strings.Contains(strings.ToLower(value), strings.ToLower(key.GetText())), nil + b, err := decoder.Bytes(key.GetText()) + if err != nil { + return false, err + } + + return strings.Contains(strings.ToLower(value), strings.ToLower(string(b))), nil }) } -func (m *Mailbox) matchSearchKeyHeader(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyHeader(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { literal, err := m.state.getLiteral(message.ID) if err != nil { @@ -279,7 +300,12 @@ func (m *Mailbox) matchSearchKeyHeader(ctx context.Context, candidates []*snapMs return false, err } - return strings.Contains(strings.ToLower(value), strings.ToLower(key.GetText())), nil + b, err := decoder.Bytes(key.GetText()) + if err != nil { + return false, err + } + + return strings.Contains(strings.ToLower(value), strings.ToLower(string(b))), nil }) } @@ -300,14 +326,14 @@ func (m *Mailbox) matchSearchKeyLarger(ctx context.Context, candidates []*snapMs }) } -func (m *Mailbox) matchSearchKeyNew(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyNew(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return message.flags.Contains(imap.FlagRecent) && !message.flags.Contains(imap.FlagSeen), nil }) } -func (m *Mailbox) matchSearchKeyNot(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { - left, err := m.matchSearchKey(ctx, candidates, key.GetLeftOp()) +func (m *Mailbox) matchSearchKeyNot(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { + left, err := m.matchSearchKey(ctx, candidates, key.GetLeftOp(), decoder) if err != nil { return nil, err } @@ -317,7 +343,7 @@ func (m *Mailbox) matchSearchKeyNot(ctx context.Context, candidates []*snapMsg, }) } -func (m *Mailbox) matchSearchKeyOld(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyOld(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return !message.flags.Contains(imap.FlagRecent), nil }) @@ -339,13 +365,13 @@ func (m *Mailbox) matchSearchKeyOn(ctx context.Context, candidates []*snapMsg, k }) } -func (m *Mailbox) matchSearchKeyOr(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { - left, err := m.matchSearchKey(ctx, candidates, key.GetLeftOp()) +func (m *Mailbox) matchSearchKeyOr(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { + left, err := m.matchSearchKey(ctx, candidates, key.GetLeftOp(), decoder) if err != nil { return nil, err } - right, err := m.matchSearchKey(ctx, candidates, key.GetRightOp()) + right, err := m.matchSearchKey(ctx, candidates, key.GetRightOp(), decoder) if err != nil { return nil, err } @@ -358,13 +384,13 @@ func (m *Mailbox) matchSearchKeyOr(ctx context.Context, candidates []*snapMsg, k }) } -func (m *Mailbox) matchSearchKeyRecent(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyRecent(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return message.flags.Contains(imap.FlagRecent), nil }) } -func (m *Mailbox) matchSearchKeySeen(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeySeen(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return message.flags.Contains(imap.FlagSeen), nil }) @@ -484,7 +510,7 @@ func (m *Mailbox) matchSearchKeySmaller(ctx context.Context, candidates []*snapM }) } -func (m *Mailbox) matchSearchKeySubject(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeySubject(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { literal, err := m.state.getLiteral(message.ID) if err != nil { @@ -496,22 +522,32 @@ func (m *Mailbox) matchSearchKeySubject(ctx context.Context, candidates []*snapM return false, err } - return strings.Contains(strings.ToLower(value), strings.ToLower(key.GetText())), nil + b, err := decoder.Bytes(key.GetText()) + if err != nil { + return false, err + } + + return strings.Contains(strings.ToLower(value), strings.ToLower(string(b))), nil }) } -func (m *Mailbox) matchSearchKeyText(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyText(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { literal, err := m.state.getLiteral(message.ID) if err != nil { return false, err } - return bytes.Contains([]byte(strings.ToLower(string(literal))), []byte(strings.ToLower(key.GetText()))), nil + b, err := decoder.Bytes(key.GetText()) + if err != nil { + return false, err + } + + return bytes.Contains(bytes.ToLower(literal), bytes.ToLower(b)), nil }) } -func (m *Mailbox) matchSearchKeyTo(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyTo(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { literal, err := m.state.getLiteral(message.ID) if err != nil { @@ -523,7 +559,12 @@ func (m *Mailbox) matchSearchKeyTo(ctx context.Context, candidates []*snapMsg, k return false, err } - return strings.Contains(strings.ToLower(value), strings.ToLower(key.GetText())), nil + b, err := decoder.Bytes(key.GetText()) + if err != nil { + return false, err + } + + return strings.Contains(strings.ToLower(value), strings.ToLower(string(b))), nil }) } @@ -538,25 +579,25 @@ func (m *Mailbox) matchSearchKeyUID(ctx context.Context, candidates []*snapMsg, }) } -func (m *Mailbox) matchSearchKeyUnanswered(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyUnanswered(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return !message.flags.Contains(imap.FlagAnswered), nil }) } -func (m *Mailbox) matchSearchKeyUndeleted(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyUndeleted(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return !message.flags.Contains(imap.FlagDeleted), nil }) } -func (m *Mailbox) matchSearchKeyUndraft(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyUndraft(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return !message.flags.Contains(imap.FlagDraft), nil }) } -func (m *Mailbox) matchSearchKeyUnflagged(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyUnflagged(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return !message.flags.Contains(imap.FlagFlagged), nil }) @@ -568,7 +609,7 @@ func (m *Mailbox) matchSearchKeyUnkeyword(ctx context.Context, candidates []*sna }) } -func (m *Mailbox) matchSearchKeyUnseen(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { +func (m *Mailbox) matchSearchKeyUnseen(ctx context.Context, candidates []*snapMsg) ([]*snapMsg, error) { return filter(candidates, func(message *snapMsg) (bool, error) { return !message.flags.Contains(imap.FlagSeen), nil }) @@ -585,8 +626,8 @@ func (m *Mailbox) matchSearchKeySeqSet(ctx context.Context, candidates []*snapMs }) } -func (m *Mailbox) matchSearchKeyList(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey) ([]*snapMsg, error) { - return doSearch(ctx, m, candidates, key.GetChildren()) +func (m *Mailbox) matchSearchKeyList(ctx context.Context, candidates []*snapMsg, key *proto.SearchKey, decoder *encoding.Decoder) ([]*snapMsg, error) { + return doSearch(ctx, m, candidates, key.GetChildren(), decoder) } func filter(candidates []*snapMsg, wantMessage func(*snapMsg) (bool, error)) ([]*snapMsg, error) { diff --git a/internal/parser/proto/imap.pb.go b/internal/parser/proto/imap.pb.go index 09f5efeb..2783d349 100644 --- a/internal/parser/proto/imap.pb.go +++ b/internal/parser/proto/imap.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.28.0 -// protoc v3.21.1 +// protoc v3.21.4 // source: imap.proto package proto @@ -2261,7 +2261,7 @@ type SearchKey struct { unknownFields protoimpl.UnknownFields Keyword SearchKeyword `protobuf:"varint,1,opt,name=keyword,proto3,enum=proto.SearchKeyword" json:"keyword,omitempty"` - Text string `protobuf:"bytes,2,opt,name=text,proto3" json:"text,omitempty"` + Text []byte `protobuf:"bytes,2,opt,name=text,proto3" json:"text,omitempty"` Date string `protobuf:"bytes,3,opt,name=date,proto3" json:"date,omitempty"` Flag string `protobuf:"bytes,4,opt,name=flag,proto3" json:"flag,omitempty"` Field string `protobuf:"bytes,5,opt,name=field,proto3" json:"field,omitempty"` @@ -2311,11 +2311,11 @@ func (x *SearchKey) GetKeyword() SearchKeyword { return SearchKeyword_SearchKWAll } -func (x *SearchKey) GetText() string { +func (x *SearchKey) GetText() []byte { if x != nil { return x.Text } - return "" + return nil } func (x *SearchKey) GetDate() string { @@ -3617,7 +3617,7 @@ var file_imap_proto_rawDesc = []byte{ 0x07, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x4b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x77, 0x6f, 0x72, 0x64, 0x12, 0x12, 0x0a, - 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, + 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x6c, 0x61, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x6c, 0x61, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x69, 0x65, diff --git a/internal/parser/proto/imap.proto b/internal/parser/proto/imap.proto index 96e57bf5..a9c1536b 100644 --- a/internal/parser/proto/imap.proto +++ b/internal/parser/proto/imap.proto @@ -190,7 +190,7 @@ message Search { message SearchKey { SearchKeyword keyword = 1; - string text = 2; + bytes text = 2; string date = 3; string flag = 4; string field = 5; diff --git a/internal/parser/tests/parser/parser_test.cpp b/internal/parser/tests/parser/parser_test.cpp index d7e02cd6..dea2c4ed 100644 --- a/internal/parser/tests/parser/parser_test.cpp +++ b/internal/parser/tests/parser/parser_test.cpp @@ -377,7 +377,7 @@ TEST_F(ParserTest, SearchNotFrom) { "flag": "", "keyword": "SearchKWFrom", "size": 0, - "text": "Smith" + "text": "U21pdGg=" }, "size": 0, "text": "" @@ -407,7 +407,7 @@ TEST_F(ParserTest, SearchOrFrom) { "flag": "", "keyword": "SearchKWFrom", "size": 0, - "text": "Smith" + "text": "U21pdGg=" }, "rightOp": { "children": [], @@ -416,7 +416,7 @@ TEST_F(ParserTest, SearchOrFrom) { "flag": "", "keyword": "SearchKWFrom", "size": 0, - "text": "Bob" + "text": "Qm9i" }, "size": 0, "text": "" @@ -440,7 +440,7 @@ TEST_F(ParserTest, SearchBcc) { "flag": "", "keyword": "SearchKWBcc", "size": 0, - "text": "mail@example.com" + "text": "bWFpbEBleGFtcGxlLmNvbQ==" } ] } @@ -585,7 +585,7 @@ TEST_F(ParserTest, SearchBody) { "flag": "", "keyword": "SearchKWBody", "size": 0, - "text": "some body" + "text": "c29tZSBib2R5" } ] } @@ -605,7 +605,7 @@ TEST_F(ParserTest, SearchCc) { "flag": "", "keyword": "SearchKWCc", "size": 0, - "text": "mail@example.com" + "text": "bWFpbEBleGFtcGxlLmNvbQ==" } ] } @@ -626,7 +626,7 @@ TEST_F(ParserTest, SearchFrom) { "flag": "", "keyword": "SearchKWFrom", "size": 0, - "text": "mail@example.com" + "text": "bWFpbEBleGFtcGxlLmNvbQ==" } ] } @@ -688,7 +688,7 @@ TEST_F(ParserTest, SearchSubject) { "flag": "", "keyword": "SearchKWSubject", "size": 0, - "text": "some subject" + "text": "c29tZSBzdWJqZWN0" } ] } @@ -708,7 +708,7 @@ TEST_F(ParserTest, SearchText) { "flag": "", "keyword": "SearchKWText", "size": 0, - "text": "some text" + "text": "c29tZSB0ZXh0" } ] } @@ -729,7 +729,7 @@ TEST_F(ParserTest, SearchTo) { "flag": "", "keyword": "SearchKWTo", "size": 0, - "text": "mail@example.com" + "text": "bWFpbEBleGFtcGxlLmNvbQ==" } ] } @@ -790,7 +790,7 @@ TEST_F(ParserTest, SearchHeader) { "flag": "", "keyword": "SearchKWHeader", "size": 0, - "text": "string" + "text": "c3RyaW5n" } ] } @@ -898,7 +898,7 @@ TEST_F(ParserTest, SearchTextWithCharset) { "flag": "", "keyword": "SearchKWText", "size": 0, - "text": "some text" + "text": "c29tZSB0ZXh0" } ] } @@ -921,7 +921,7 @@ TEST_F(ParserTest, SearchChildren) { "flag": "", "keyword": "SearchKWText", "size": 0, - "text": "some text" + "text": "c29tZSB0ZXh0" }, { "children": [], @@ -930,7 +930,7 @@ TEST_F(ParserTest, SearchChildren) { "flag": "", "keyword": "SearchKWText", "size": 0, - "text": "some other text" + "text": "c29tZSBvdGhlciB0ZXh0" } ], "date": "", @@ -962,7 +962,7 @@ TEST_F(ParserTest, SearchLiteralText) { "flag": "", "keyword": "SearchKWText", "size": 0, - "text": "hello!" + "text": "aGVsbG8h" } ] } diff --git a/internal/response/item_badcharset.go b/internal/response/item_badcharset.go new file mode 100644 index 00000000..03b3f3c6 --- /dev/null +++ b/internal/response/item_badcharset.go @@ -0,0 +1,11 @@ +package response + +type itemBadCharset struct{} + +func ItemBadCharset() *itemBadCharset { + return &itemBadCharset{} +} + +func (c *itemBadCharset) String() string { + return "BADCHARSET" +} diff --git a/internal/session/handle_search.go b/internal/session/handle_search.go index ad6d05d0..c3c003d8 100644 --- a/internal/session/handle_search.go +++ b/internal/session/handle_search.go @@ -6,10 +6,27 @@ import ( "github.com/ProtonMail/gluon/internal/backend" "github.com/ProtonMail/gluon/internal/parser/proto" "github.com/ProtonMail/gluon/internal/response" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/ianaindex" ) func (s *Session) handleSearch(ctx context.Context, tag string, cmd *proto.Search, mailbox *backend.Mailbox, ch chan response.Response) error { - seq, err := mailbox.Search(ctx, cmd.GetKeys()) + var decoder *encoding.Decoder + + switch charset := cmd.GetOptionalCharset().(type) { + case *proto.Search_Charset: + encoding, err := ianaindex.IANA.Encoding(charset.Charset) + if err != nil { + return response.No(tag).WithItems(response.ItemBadCharset()) + } + + decoder = encoding.NewDecoder() + + default: + decoder = encoding.Nop.NewDecoder() + } + + seq, err := mailbox.Search(ctx, cmd.GetKeys(), decoder) if err != nil { return err } diff --git a/tests/connection_test.go b/tests/connection_test.go index 4a80642e..918f9b44 100644 --- a/tests/connection_test.go +++ b/tests/connection_test.go @@ -69,6 +69,14 @@ func (s *testConnection) C(value string) *testConnection { return s } +func (s *testConnection) Cb(b []byte) *testConnection { + n, err := s.conn.Write(append(b, []byte("\r\n")...)) + require.NoError(s.tb, err) + require.Greater(s.tb, n, 0) + + return s +} + func (s *testConnection) Cf(format string, a ...any) *testConnection { return s.C(fmt.Sprintf(format, a...)) } @@ -119,6 +127,13 @@ func (s *testConnection) Sxe(want ...string) *testConnection { return s } +// Continue is a shortcut for a server continuation request. +func (s *testConnection) Continue() *testConnection { + s.Sx("\\+") + + return s +} + // OK is a shortcut that we eventually get a tagged OK response of some kind. func (s *testConnection) OK(tag string, items ...string) { want := tag + " OK" diff --git a/tests/search_test.go b/tests/search_test.go index 62bcee39..f7f5995d 100644 --- a/tests/search_test.go +++ b/tests/search_test.go @@ -4,21 +4,51 @@ import ( "fmt" "testing" "time" + + "golang.org/x/text/encoding/htmlindex" ) -func TestSearchCharSet(t *testing.T) { +func TestSearchCharSetUTF8(t *testing.T) { + runOneToOneTestWithAuth(t, defaultServerOptions(t), func(c *testConnection, s *testSession) { + c.C(`tag select inbox`).OK("tag") + + // Encode "ééé" as UTF-8. + b := enc("ééé", "UTF-8") + + // Append a message with that as the body. + c.doAppend("inbox", "To: 1@pm.me\r\n\r\nééé").expect("OK") + + // Search for it with UTF-8 encoding. + c.Cf(`TAG SEARCH CHARSET UTF-8 BODY {%v}`, len(b)).Continue().Cb(b).S("* SEARCH 1").OK("TAG") + }) +} + +func TestSearchCharSetISO88591(t *testing.T) { + runOneToOneTestWithAuth(t, defaultServerOptions(t), func(c *testConnection, s *testSession) { + c.C(`tag select inbox`).OK("tag") + + // Encode "ééé" as ISO-8859-1. + b := enc("ééé", "ISO-8859-1") + + // Append a message with that as the body. + c.doAppend("inbox", "To: 1@pm.me\r\n\r\nééé").expect("OK") + + // Search for it with ISO-8859-1 encoding. + c.Cf(`TAG SEARCH CHARSET ISO-8859-1 BODY {%v}`, len(b)).Continue().Cb(b).S("* SEARCH 1").OK("TAG") + }) +} + +func TestSearchCharSetASCII(t *testing.T) { runOneToOneTestWithData(t, defaultServerOptions(t), func(c *testConnection, s *testSession, mbox, mboxID string) { - c.C("A001 search CHARSET UTF-8 TEXT foo") + c.C("A001 search CHARSET US-ASCII TEXT foo") c.S("* SEARCH 75") c.OK("A001") }) } -// TODO: GOMSRV-184 . -func _TestSearchCharSetInvalid(t *testing.T) { +func TestSearchCharSetInvalid(t *testing.T) { runOneToOneTestWithData(t, defaultServerOptions(t), func(c *testConnection, s *testSession, mbox, mboxID string) { - c.C("A001 search CHARSET invalid-charset TEXT foo") - c.NO("A001") + c.C("A001 search CHARSET invalid-charset TEXT foo").NO("A001", "BADCHARSET") }) } @@ -673,3 +703,17 @@ func TestSearchList(t *testing.T) { c.OK("A004") }) } + +func enc(text, encoding string) []byte { + enc, err := htmlindex.Get(encoding) + if err != nil { + panic(err) + } + + b, err := enc.NewEncoder().Bytes([]byte(text)) + if err != nil { + panic(err) + } + + return b +}