diff --git a/lib/net/imap.rb b/lib/net/imap.rb index b2a35c9e..e08634cc 100644 --- a/lib/net/imap.rb +++ b/lib/net/imap.rb @@ -534,6 +534,11 @@ module Net # See FetchData#emailid and FetchData#emailid. # - Updates #status with support for the +MAILBOXID+ status attribute. # + # ==== RFC9394: +PARTIAL+ + # - Updates #search, #uid_search with the +PARTIAL+ return option which adds + # ESearchResult#partial return data. + # - Updates #uid_fetch with the +partial+ modifier. + # # == References # # [{IMAP4rev1}[https://www.rfc-editor.org/rfc/rfc3501.html]]:: @@ -701,6 +706,11 @@ module Net # Gondwana, B., Ed., "IMAP Extension for Object Identifiers", # RFC 8474, DOI 10.17487/RFC8474, September 2018, # . + # [PARTIAL[https://www.rfc-editor.org/info/rfc9394]]:: + # Melnikov, A., Achuthan, A., Nagulakonda, V., and L. Alves, + # "IMAP PARTIAL Extension for Paged SEARCH and FETCH", RFC 9394, + # DOI 10.17487/RFC9394, June 2023, + # . # # === IANA registries # * {IMAP Capabilities}[http://www.iana.org/assignments/imap4-capabilities] @@ -1971,8 +1981,9 @@ def uid_expunge(uid_set) # the server to return an ESearchResult instead of a SearchResult, but some # servers disobey this requirement. Requires an extended search # capability, such as +ESEARCH+ or +IMAP4rev2+. - # See {"Argument translation"}[rdoc-ref:#search@Argument+translation] - # and {"Return options"}[rdoc-ref:#search@Return+options], below. + # See {"Argument translation"}[rdoc-ref:#search@Argument+translation] and + # {"Supported return options"}[rdoc-ref:#search@Supported+return+options], + # below. # # +charset+ is the name of the {registered character # set}[https://www.iana.org/assignments/character-sets/character-sets.xhtml] @@ -2082,33 +2093,58 @@ def uid_expunge(uid_set) # *WARNING:* This is vulnerable to injection attacks when external # inputs are used. # - # ==== Return options + # ==== Supported return options # # For full definitions of the standard return options and return data, see # the relevant RFCs. # - # ===== +ESEARCH+ or +IMAP4rev2+ - # - # The following return options require either +ESEARCH+ or +IMAP4rev2+. - # See [{RFC4731 §3.1}[https://rfc-editor.org/rfc/rfc4731#section-3.1]] or - # [{IMAP4rev2 §6.4.4}[https://www.rfc-editor.org/rfc/rfc9051.html#section-6.4.4]]. - # # [+ALL+] # Returns ESearchResult#all with a SequenceSet of all matching sequence # numbers or UIDs. This is the default, when return options are empty. # # For compatibility with SearchResult, ESearchResult#to_a returns an # Array of message sequence numbers or UIDs. + # + # Requires either the +ESEARCH+ or +IMAP4rev2+ capabability. + # {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731] + # {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051] + # # [+COUNT+] # Returns ESearchResult#count with the number of matching messages. + # + # Requires either the +ESEARCH+ or +IMAP4rev2+ capabability. + # {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731] + # {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051] + # # [+MAX+] # Returns ESearchResult#max with the highest matching sequence number or # UID. + # + # Requires either the +ESEARCH+ or +IMAP4rev2+ capabability. + # {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731] + # {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051] + # # [+MIN+] # Returns ESearchResult#min with the lowest matching sequence number or # UID. # - # ===== +CONDSTORE+ + # Requires either the +ESEARCH+ or +IMAP4rev2+ capabability. + # {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731] + # {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051] + # + # [+PARTIAL+ _range_] + # Returns ESearchResult#partial with a SequenceSet of a subset of + # matching sequence numbers or UIDs, as selected by _range_. As with + # sequence numbers, the first result is +1+: 1..500 selects the + # first 500 search results (in mailbox order), 501..1000 the + # second 500, and so on. _range_ may also be negative: -500..-1 + # selects the last 500 search results. + # + # Requires either the CONTEXT=SEARCH or +PARTIAL+ capabability. + # {[RFC5267]}[https://rfc-editor.org/rfc/rfc5267] + # {[RFC9394]}[https://rfc-editor.org/rfc/rfc9394] + # + # ===== +MODSEQ+ return data # # ESearchResult#modseq return data does not have a corresponding return # option. Instead, it is returned if the +MODSEQ+ search key is used or @@ -2120,8 +2156,8 @@ def uid_expunge(uid_set) # # {RFC4466 §2.6}[https://www.rfc-editor.org/rfc/rfc4466.html#section-2.6] # defines standard syntax for search extensions. Net::IMAP allows sending - # unknown search return options and will parse unknown search extensions' - # return values into ExtensionData. Please note that this is an + # unsupported search return options and will parse unsupported search + # extensions' return values into ExtensionData. Please note that this is an # intentionally _unstable_ API. Future releases may return different # (incompatible) objects, without deprecation or warning. # @@ -2398,12 +2434,12 @@ def uid_search(...) # {[RFC7162]}[https://tools.ietf.org/html/rfc7162] in order to use the # +changedsince+ argument. Using +changedsince+ implicitly enables the # +CONDSTORE+ extension. - def fetch(set, attr, mod = nil, changedsince: nil) - fetch_internal("FETCH", set, attr, mod, changedsince: changedsince) + def fetch(...) + fetch_internal("FETCH", ...) end # :call-seq: - # uid_fetch(set, attr, changedsince: nil) -> array of FetchData + # uid_fetch(set, attr, changedsince: nil, partial: nil) -> array of FetchData # # Sends a {UID FETCH command [IMAP4rev1 §6.4.8]}[https://www.rfc-editor.org/rfc/rfc3501#section-6.4.8] # to retrieve data associated with a message in the mailbox. @@ -2420,13 +2456,44 @@ def fetch(set, attr, mod = nil, changedsince: nil) # # +changedsince+ (optional) behaves the same as with #fetch. # + # +partial+ is an optional range to limit the number of results returned. + # It's useful when +set+ contains an unknown number of messages. + # 1..500 returns the first 500 messages in +set+ (in mailbox + # order), 501..1000 the second 500, and so on. +partial+ may also + # be negative: -500..-1 selects the last 500 messages in +set+. + # Requires the +PARTIAL+ capabability. + # {[RFC9394]}[https://rfc-editor.org/rfc/rfc9394] + # + # For example: + # + # # Without partial, the size of the results may be unknown beforehand: + # results = imap.uid_fetch(next_uid_to_fetch.., %w(UID FLAGS)) + # # ... maybe wait for a long time ... and allocate a lot of memory ... + # results.size # => 0..2**32-1 + # process results # may also take a long time and use a lot of memory... + # + # # Using partial, the results may be paginated: + # loop do + # results = imap.uid_fetch(next_uid_to_fetch.., %w(UID FLAGS), + # partial: 1..500) + # # fetch should return quickly and allocate little memory + # results.size # => 0..500 + # break if results.empty? + # next_uid_to_fetch = results.last.uid + 1 + # process results + # end + # # Related: #fetch, FetchData # # ==== Capabilities # - # Same as #fetch. - def uid_fetch(set, attr, mod = nil, changedsince: nil) - fetch_internal("UID FETCH", set, attr, mod, changedsince: changedsince) + # The server's capabilities must include +PARTIAL+ + # {[RFC9394]}[https://rfc-editor.org/rfc/rfc9394] in order to use the + # +partial+ argument. + # + # Otherwise, the same as #fetch. + def uid_fetch(...) + fetch_internal("UID FETCH", ...) end # :call-seq: @@ -3372,7 +3439,12 @@ def search_internal(cmd, ...) end end - def fetch_internal(cmd, set, attr, mod = nil, changedsince: nil) + def fetch_internal(cmd, set, attr, mod = nil, partial: nil, changedsince: nil) + set = SequenceSet[set] + if partial + mod ||= [] + mod << "PARTIAL" << PartialRange[partial] + end if changedsince mod ||= [] mod << "CHANGEDSINCE" << Integer(changedsince) @@ -3389,9 +3461,9 @@ def fetch_internal(cmd, set, attr, mod = nil, changedsince: nil) synchronize do clear_responses("FETCH") if mod - send_command(cmd, SequenceSet.new(set), attr, mod) + send_command(cmd, set, attr, mod) else - send_command(cmd, SequenceSet.new(set), attr) + send_command(cmd, set, attr) end clear_responses("FETCH") end diff --git a/lib/net/imap/esearch_result.rb b/lib/net/imap/esearch_result.rb index c15c68ae..b19cadaa 100644 --- a/lib/net/imap/esearch_result.rb +++ b/lib/net/imap/esearch_result.rb @@ -35,16 +35,16 @@ def initialize(tag: nil, uid: nil, data: nil) # :call-seq: to_a -> Array of integers # - # When #all contains a SequenceSet of message sequence + # When either #all or #partial contains a SequenceSet of message sequence # numbers or UIDs, +to_a+ returns that set as an array of integers. # - # When #all is +nil+, either because the server - # returned no results or because +ALL+ was not included in + # When both #all and #partial are +nil+, either because the server + # returned no results or because +ALL+ and +PARTIAL+ were not included in # the IMAP#search +RETURN+ options, #to_a returns an empty array. # # Note that SearchResult also implements +to_a+, so it can be used without # checking if the server returned +SEARCH+ or +ESEARCH+ data. - def to_a; all&.numbers || [] end + def to_a; all&.numbers || partial&.to_a || [] end ## # attr_reader: tag @@ -135,6 +135,46 @@ def count; data.assoc("COUNT")&.last end # and +ESEARCH+ {[RFC4731]}[https://www.rfc-editor.org/rfc/rfc4731.html#section-3.2]. def modseq; data.assoc("MODSEQ")&.last end + # Returned by ESearchResult#partial. + # + # Requires +PARTIAL+ {[RFC9394]}[https://www.rfc-editor.org/rfc/rfc9394.html] + # or CONTEXT=SEARCH/CONTEXT=SORT + # {[RFC5267]}[https://www.rfc-editor.org/rfc/rfc5267.html] + # + # See also: #to_a + class PartialResult < Data.define(:range, :results) + def initialize(range:, results:) + range => Range + results = SequenceSet[results] unless results.nil? + super + end + + ## + # method: range + # :call-seq: range -> range + + ## + # method: results + # :call-seq: results -> sequence set or nil + + # Converts #results to an array of integers. + # + # See also: ESearchResult#to_a. + def to_a; results&.numbers || [] end + end + + # :call-seq: partial -> PartialResult or nil + # + # A PartialResult containing a subset of the message sequence numbers or + # UIDs that satisfy the SEARCH criteria. + # + # Requires +PARTIAL+ {[RFC9394]}[https://www.rfc-editor.org/rfc/rfc9394.html] + # or CONTEXT=SEARCH/CONTEXT=SORT + # {[RFC5267]}[https://www.rfc-editor.org/rfc/rfc5267.html] + # + # See also: #to_a + def partial; data.assoc("PARTIAL")&.last end + end end end diff --git a/lib/net/imap/response_parser.rb b/lib/net/imap/response_parser.rb index 8d87ec1b..2b055b37 100644 --- a/lib/net/imap/response_parser.rb +++ b/lib/net/imap/response_parser.rb @@ -1535,6 +1535,9 @@ def esearch_response # From RFC4731 (ESEARCH): # search-return-data =/ "MODSEQ" SP mod-sequence-value # + # From RFC9394 (PARTIAL): + # search-return-data =/ ret-data-partial + # def search_return_data label = search_modifier_name; SP! value = @@ -1544,11 +1547,41 @@ def search_return_data when "ALL" then sequence_set when "COUNT" then number when "MODSEQ" then mod_sequence_value # RFC7162: CONDSTORE + when "PARTIAL" then ret_data_partial__value # RFC9394: PARTIAL else search_return_value end [label, value] end + # From RFC5267 (CONTEXT=SEARCH, CONTEXT=SORT) and RFC9394 (PARTIAL): + # ret-data-partial = "PARTIAL" + # SP "(" partial-range SP partial-results ")" + def ret_data_partial__value + lpar + range = partial_range; SP! + results = partial_results + rpar + ESearchResult::PartialResult.new(range, results) + end + + # partial-range = partial-range-first / partial-range-last + # tagged-ext-simple =/ partial-range-last + def partial_range + case (str = atom) + when Patterns::PARTIAL_RANGE_FIRST, Patterns::PARTIAL_RANGE_LAST + min, max = [Integer($1), Integer($2)].minmax + min..max + else + parse_error("unexpected atom %p, expected partial-range", str) + end + end + + # partial-results = sequence-set / "NIL" + # ;; from [RFC3501]. + # ;; NIL indicates that no results correspond to + # ;; the requested range. + def partial_results; NIL? ? nil : sequence_set end + # search-modifier-name = tagged-ext-label alias search_modifier_name tagged_ext_label diff --git a/rakelib/rfcs.rake b/rakelib/rfcs.rake index d1b9bb7c..950c49cc 100644 --- a/rakelib/rfcs.rake +++ b/rakelib/rfcs.rake @@ -145,6 +145,7 @@ RFCS = { 8514 => "IMAP SAVEDATE", 8970 => "IMAP PREVIEW", 9208 => "IMAP QUOTA, QUOTA=, QUOTASET", + 9394 => "IMAP PARTIAL", # etc... 3629 => "UTF8", diff --git a/test/net/imap/fixtures/response_parser/rfc9394_partial.yml b/test/net/imap/fixtures/response_parser/rfc9394_partial.yml new file mode 100644 index 00000000..233a9096 --- /dev/null +++ b/test/net/imap/fixtures/response_parser/rfc9394_partial.yml @@ -0,0 +1,66 @@ +--- +:tests: + + "RFC9394 PARTIAL 3.1. example 1": + comment: | + Neither RFC9394 nor RFC5267 contain any examples of a normal unelided + sequence-set result. I've edited it to include a sequence-set here. + :response: "* ESEARCH (TAG \"A01\") UID PARTIAL (-1:-100 200:250,252:300)\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: ESEARCH + data: !ruby/object:Net::IMAP::ESearchResult + tag: A01 + uid: true + data: + - - PARTIAL + - !ruby/object:Net::IMAP::ESearchResult::PartialResult + range: !ruby/range + begin: -100 + end: -1 + excl: false + results: !ruby/object:Net::IMAP::SequenceSet + string: 200:250,252:300 + tuples: + - - 200 + - 250 + - - 252 + - 300 + raw_data: "* ESEARCH (TAG \"A01\") UID PARTIAL (-1:-100 200:250,252:300)\r\n" + + "RFC9394 PARTIAL 3.1. example 2": + :response: "* ESEARCH (TAG \"A02\") UID PARTIAL (23500:24000 55500:56000)\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: ESEARCH + data: !ruby/object:Net::IMAP::ESearchResult + tag: A02 + uid: true + data: + - - PARTIAL + - !ruby/object:Net::IMAP::ESearchResult::PartialResult + range: !ruby/range + begin: 23500 + end: 24000 + excl: false + results: !ruby/object:Net::IMAP::SequenceSet + string: 55500:56000 + tuples: + - - 55500 + - 56000 + raw_data: "* ESEARCH (TAG \"A02\") UID PARTIAL (23500:24000 55500:56000)\r\n" + + "RFC9394 PARTIAL 3.1. example 3": + :response: "* ESEARCH (TAG \"A04\") UID PARTIAL (24000:24500 NIL)\r\n" + :expected: !ruby/struct:Net::IMAP::UntaggedResponse + name: ESEARCH + data: !ruby/object:Net::IMAP::ESearchResult + tag: A04 + uid: true + data: + - - PARTIAL + - !ruby/object:Net::IMAP::ESearchResult::PartialResult + range: !ruby/range + begin: 24000 + end: 24500 + excl: false + results: + raw_data: "* ESEARCH (TAG \"A04\") UID PARTIAL (24000:24500 NIL)\r\n" diff --git a/test/net/imap/test_esearch_result.rb b/test/net/imap/test_esearch_result.rb index ca214e51..1e75e6bc 100644 --- a/test/net/imap/test_esearch_result.rb +++ b/test/net/imap/test_esearch_result.rb @@ -15,6 +15,10 @@ class ESearchResultTest < Test::Unit::TestCase assert_equal [], esearch.to_a esearch = ESearchResult.new(nil, false, [["ALL", SequenceSet["1,5:8"]]]) assert_equal [1, 5, 6, 7, 8], esearch.to_a + esearch = ESearchResult.new(nil, false, [ + ["PARTIAL", ESearchResult::PartialResult[1..5, "1,5:8"]] + ]) + assert_equal [1, 5, 6, 7, 8], esearch.to_a end test "#tag" do @@ -80,4 +84,17 @@ class ESearchResultTest < Test::Unit::TestCase assert_equal 12345, esearch.modseq end + test "#partial returns PARTIAL value (RFC9394: PARTIAL)" do + result = Net::IMAP::ResponseParser.new.parse( + "* ESEARCH (TAG \"A0006\") UID PARTIAL (-1:-100 200:250,252:300)\r\n" + ).data + assert_equal(ESearchResult, result.class) + assert_equal( + ESearchResult::PartialResult.new( + -100..-1, SequenceSet[200..250, 252..300] + ), + result.partial + ) + end + end diff --git a/test/net/imap/test_imap.rb b/test/net/imap/test_imap.rb index 1f20c773..e569dcfb 100644 --- a/test/net/imap/test_imap.rb +++ b/test/net/imap/test_imap.rb @@ -1214,6 +1214,27 @@ def test_enable end end + test "#uid_fetch with partial" do + with_fake_server select: "inbox" do |server, imap| + server.on("UID FETCH", &:done_ok) + imap.uid_fetch 1.., "FAST", partial: 1..500 + assert_equal("RUBY0002 UID FETCH 1:* FAST (PARTIAL 1:500)", + server.commands.pop.raw.strip) + imap.uid_fetch 1.., "FAST", partial: 1...501 + assert_equal("RUBY0003 UID FETCH 1:* FAST (PARTIAL 1:500)", + server.commands.pop.raw.strip) + imap.uid_fetch 1.., "FAST", partial: -500..-1 + assert_equal("RUBY0004 UID FETCH 1:* FAST (PARTIAL -500:-1)", + server.commands.pop.raw.strip) + imap.uid_fetch 1.., "FAST", partial: -500...-1 + assert_equal("RUBY0005 UID FETCH 1:* FAST (PARTIAL -500:-2)", + server.commands.pop.raw.strip) + imap.uid_fetch 1.., "FAST", partial: 1..20, changedsince: 1234 + assert_equal("RUBY0006 UID FETCH 1:* FAST (PARTIAL 1:20 CHANGEDSINCE 1234)", + server.commands.pop.raw.strip) + end + end + test "#store with unchangedsince" do with_fake_server select: "inbox" do |server, imap| server.on("STORE", &:done_ok) diff --git a/test/net/imap/test_imap_response_parser.rb b/test/net/imap/test_imap_response_parser.rb index 49bc14b6..ac8f2f04 100644 --- a/test/net/imap/test_imap_response_parser.rb +++ b/test/net/imap/test_imap_response_parser.rb @@ -103,6 +103,9 @@ def teardown # RFC 9208: QUOTA extension generate_tests_from fixture_file: "rfc9208_quota_responses.yml" + # RFC 9394: PARTIAL extension + generate_tests_from fixture_file: "rfc9394_partial.yml" + ############################################################################ # Workarounds or unspecified extensions: generate_tests_from fixture_file: "quirky_behaviors.yml"