🔀 Merge pull request #367 from ruby/RFC9394-PARTIAL

✨ Add support for `PARTIAL` extension (RFC9394)

lib/net/imap.rb

@@ -534,6 +534,11 @@ module Net
   #   See FetchData#emailid and FetchData#emailid.
   # - Updates #status with support for the +MAILBOXID+ status attribute.
   #
+  # ==== RFC9394: +PARTIAL+
+  # - Updates #search, #uid_search with the +PARTIAL+ return option which adds
+  #   ESearchResult#partial return data.
+  # - Updates #uid_fetch with the +partial+ modifier.
+  #
   # == References
   #
   # [{IMAP4rev1}[https://www.rfc-editor.org/rfc/rfc3501.html]]::
@@ -701,6 +706,11 @@ module Net
   #   Gondwana, B., Ed., "IMAP Extension for Object Identifiers",
   #   RFC 8474, DOI 10.17487/RFC8474, September 2018,
   #   <https://www.rfc-editor.org/info/rfc8474>.
+  # [PARTIAL[https://www.rfc-editor.org/info/rfc9394]]::
+  #   Melnikov, A., Achuthan, A., Nagulakonda, V., and L. Alves,
+  #   "IMAP PARTIAL Extension for Paged SEARCH and FETCH", RFC 9394,
+  #   DOI 10.17487/RFC9394, June 2023,
+  #   <https://www.rfc-editor.org/info/rfc9394>.
   #
   # === IANA registries
   # * {IMAP Capabilities}[http://www.iana.org/assignments/imap4-capabilities]
@@ -1971,8 +1981,9 @@ def uid_expunge(uid_set)
     # the server to return an ESearchResult instead of a SearchResult, but some
     # servers disobey this requirement.  <em>Requires an extended search
     # capability, such as +ESEARCH+ or +IMAP4rev2+.</em>
-    # See {"Argument translation"}[rdoc-ref:#search@Argument+translation]
-    # and {"Return options"}[rdoc-ref:#search@Return+options], below.
+    # See {"Argument translation"}[rdoc-ref:#search@Argument+translation] and
+    # {"Supported return options"}[rdoc-ref:#search@Supported+return+options],
+    # below.
     #
     # +charset+ is the name of the {registered character
     # set}[https://www.iana.org/assignments/character-sets/character-sets.xhtml]
@@ -2082,33 +2093,58 @@ def uid_expunge(uid_set)
     #   <em>*WARNING:* This is vulnerable to injection attacks when external
     #   inputs are used.</em>
     #
-    # ==== Return options
+    # ==== Supported return options
     #
     # For full definitions of the standard return options and return data, see
     # the relevant RFCs.
     #
-    # ===== +ESEARCH+ or +IMAP4rev2+
-    #
-    # The following return options require either +ESEARCH+ or +IMAP4rev2+.
-    # See [{RFC4731 §3.1}[https://rfc-editor.org/rfc/rfc4731#section-3.1]] or
-    # [{IMAP4rev2 §6.4.4}[https://www.rfc-editor.org/rfc/rfc9051.html#section-6.4.4]].
-    #
     # [+ALL+]
     #    Returns ESearchResult#all with a SequenceSet of all matching sequence
     #    numbers or UIDs.  This is the default, when return options are empty.
     #
     #    For compatibility with SearchResult, ESearchResult#to_a returns an
     #    Array of message sequence numbers or UIDs.
+    #
+    #    <em>Requires either the +ESEARCH+ or +IMAP4rev2+ capabability.</em>
+    #    {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731]
+    #    {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051]
+    #
     # [+COUNT+]
     #    Returns ESearchResult#count with the number of matching messages.
+    #
+    #    <em>Requires either the +ESEARCH+ or +IMAP4rev2+ capabability.</em>
+    #    {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731]
+    #    {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051]
+    #
     # [+MAX+]
     #    Returns ESearchResult#max with the highest matching sequence number or
     #    UID.
+    #
+    #    <em>Requires either the +ESEARCH+ or +IMAP4rev2+ capabability.</em>
+    #    {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731]
+    #    {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051]
+    #
     # [+MIN+]
     #    Returns ESearchResult#min with the lowest matching sequence number or
     #    UID.
     #
-    # ===== +CONDSTORE+
+    #    <em>Requires either the +ESEARCH+ or +IMAP4rev2+ capabability.</em>
+    #    {[RFC4731]}[https://rfc-editor.org/rfc/rfc4731]
+    #    {[RFC9051]}[https://rfc-editor.org/rfc/rfc9051]
+    #
+    # [+PARTIAL+ _range_]
+    #    Returns ESearchResult#partial with a SequenceSet of a subset of
+    #    matching sequence numbers or UIDs, as selected by _range_.  As with
+    #    sequence numbers, the first result is +1+: <tt>1..500</tt> selects the
+    #    first 500 search results (in mailbox order), <tt>501..1000</tt> the
+    #    second 500, and so on.  _range_ may also be negative: <tt>-500..-1</tt>
+    #    selects the last 500 search results.
+    #
+    #    <em>Requires either the <tt>CONTEXT=SEARCH</tt> or +PARTIAL+ capabability.</em>
+    #    {[RFC5267]}[https://rfc-editor.org/rfc/rfc5267]
+    #    {[RFC9394]}[https://rfc-editor.org/rfc/rfc9394]
+    #
+    # ===== +MODSEQ+ return data
     #
     # ESearchResult#modseq return data does not have a corresponding return
     # option.  Instead, it is returned if the +MODSEQ+ search key is used or
@@ -2120,8 +2156,8 @@ def uid_expunge(uid_set)
     #
     # {RFC4466 §2.6}[https://www.rfc-editor.org/rfc/rfc4466.html#section-2.6]
     # defines standard syntax for search extensions.  Net::IMAP allows sending
-    # unknown search return options and will parse unknown search extensions'
-    # return values into ExtensionData.  Please note that this is an
+    # unsupported search return options and will parse unsupported search
+    # extensions' return values into ExtensionData.  Please note that this is an
     # intentionally _unstable_ API.  Future releases may return different
     # (incompatible) objects, <em>without deprecation or warning</em>.
     #
@@ -2398,12 +2434,12 @@ def uid_search(...)
     # {[RFC7162]}[https://tools.ietf.org/html/rfc7162] in order to use the
     # +changedsince+ argument.  Using +changedsince+ implicitly enables the
     # +CONDSTORE+ extension.
-    def fetch(set, attr, mod = nil, changedsince: nil)
-      fetch_internal("FETCH", set, attr, mod, changedsince: changedsince)
+    def fetch(...)
+      fetch_internal("FETCH", ...)
     end
 
     # :call-seq:
-    #   uid_fetch(set, attr, changedsince: nil) -> array of FetchData
+    #   uid_fetch(set, attr, changedsince: nil, partial: nil) -> array of FetchData
     #
     # Sends a {UID FETCH command [IMAP4rev1 §6.4.8]}[https://www.rfc-editor.org/rfc/rfc3501#section-6.4.8]
     # to retrieve data associated with a message in the mailbox.
@@ -2420,13 +2456,44 @@ def fetch(set, attr, mod = nil, changedsince: nil)
     #
     # +changedsince+ (optional) behaves the same as with #fetch.
     #
+    # +partial+ is an optional range to limit the number of results returned.
+    # It's useful when +set+ contains an unknown number of messages.
+    # <tt>1..500</tt> returns the first 500 messages in +set+ (in mailbox
+    # order), <tt>501..1000</tt> the second 500, and so on.  +partial+ may also
+    # be negative: <tt>-500..-1</tt> selects the last 500 messages in +set+.
+    # <em>Requires the +PARTIAL+ capabability.</em>
+    # {[RFC9394]}[https://rfc-editor.org/rfc/rfc9394]
+    #
+    # For example:
+    #
+    #   # Without partial, the size of the results may be unknown beforehand:
+    #   results = imap.uid_fetch(next_uid_to_fetch.., %w(UID FLAGS))
+    #   # ... maybe wait for a long time ... and allocate a lot of memory ...
+    #   results.size # => 0..2**32-1
+    #   process results # may also take a long time and use a lot of memory...
+    #
+    #   # Using partial, the results may be paginated:
+    #   loop do
+    #     results = imap.uid_fetch(next_uid_to_fetch.., %w(UID FLAGS),
+    #                              partial: 1..500)
+    #     # fetch should return quickly and allocate little memory
+    #     results.size # => 0..500
+    #     break if results.empty?
+    #     next_uid_to_fetch = results.last.uid + 1
+    #     process results
+    #   end
+    #
     # Related: #fetch, FetchData
     #
     # ==== Capabilities
     #
-    # Same as #fetch.
-    def uid_fetch(set, attr, mod = nil, changedsince: nil)
-      fetch_internal("UID FETCH", set, attr, mod, changedsince: changedsince)
+    # The server's capabilities must include +PARTIAL+
+    # {[RFC9394]}[https://rfc-editor.org/rfc/rfc9394] in order to use the
+    # +partial+ argument.
+    #
+    # Otherwise, the same as #fetch.
+    def uid_fetch(...)
+      fetch_internal("UID FETCH", ...)
     end
 
     # :call-seq:
@@ -3372,7 +3439,12 @@ def search_internal(cmd, ...)
       end
     end
 
-    def fetch_internal(cmd, set, attr, mod = nil, changedsince: nil)
+    def fetch_internal(cmd, set, attr, mod = nil, partial: nil, changedsince: nil)
+      set = SequenceSet[set]
+      if partial
+        mod ||= []
+        mod << "PARTIAL" << PartialRange[partial]
+      end
       if changedsince
         mod ||= []
         mod << "CHANGEDSINCE" << Integer(changedsince)
@@ -3389,9 +3461,9 @@ def fetch_internal(cmd, set, attr, mod = nil, changedsince: nil)
       synchronize do
         clear_responses("FETCH")
         if mod
-          send_command(cmd, SequenceSet.new(set), attr, mod)
+          send_command(cmd, set, attr, mod)
         else
-          send_command(cmd, SequenceSet.new(set), attr)
+          send_command(cmd, set, attr)
         end
         clear_responses("FETCH")
       end

lib/net/imap/esearch_result.rb

@@ -35,16 +35,16 @@ def initialize(tag: nil, uid: nil, data: nil)
 
       # :call-seq: to_a -> Array of integers
       #
-      # When #all contains a SequenceSet of message sequence
+      # When either #all or #partial contains a SequenceSet of message sequence
       # numbers or UIDs, +to_a+ returns that set as an array of integers.
       #
-      # When #all is +nil+, either because the server
-      # returned no results or because +ALL+ was not included in
+      # When both #all and #partial are +nil+, either because the server
+      # returned no results or because +ALL+ and +PARTIAL+ were not included in
       # the IMAP#search +RETURN+ options, #to_a returns an empty array.
       #
       # Note that SearchResult also implements +to_a+, so it can be used without
       # checking if the server returned +SEARCH+ or +ESEARCH+ data.
-      def to_a; all&.numbers || [] end
+      def to_a; all&.numbers || partial&.to_a || [] end
 
       ##
       # attr_reader: tag
@@ -135,6 +135,46 @@ def count;      data.assoc("COUNT")&.last      end
       # and +ESEARCH+ {[RFC4731]}[https://www.rfc-editor.org/rfc/rfc4731.html#section-3.2].
       def modseq;     data.assoc("MODSEQ")&.last     end
 
+      # Returned by ESearchResult#partial.
+      #
+      # Requires +PARTIAL+ {[RFC9394]}[https://www.rfc-editor.org/rfc/rfc9394.html]
+      # or <tt>CONTEXT=SEARCH</tt>/<tt>CONTEXT=SORT</tt>
+      # {[RFC5267]}[https://www.rfc-editor.org/rfc/rfc5267.html]
+      #
+      # See also: #to_a
+      class PartialResult < Data.define(:range, :results)
+        def initialize(range:, results:)
+          range   => Range
+          results = SequenceSet[results] unless results.nil?
+          super
+        end
+
+        ##
+        # method: range
+        # :call-seq: range -> range
+
+        ##
+        # method: results
+        # :call-seq: results -> sequence set or nil
+
+        # Converts #results to an array of integers.
+        #
+        # See also: ESearchResult#to_a.
+        def to_a; results&.numbers || [] end
+      end
+
+      # :call-seq: partial -> PartialResult or nil
+      #
+      # A PartialResult containing a subset of the message sequence numbers or
+      # UIDs that satisfy the SEARCH criteria.
+      #
+      # Requires +PARTIAL+ {[RFC9394]}[https://www.rfc-editor.org/rfc/rfc9394.html]
+      # or <tt>CONTEXT=SEARCH</tt>/<tt>CONTEXT=SORT</tt>
+      # {[RFC5267]}[https://www.rfc-editor.org/rfc/rfc5267.html]
+      #
+      # See also: #to_a
+      def partial;    data.assoc("PARTIAL")&.last    end
+
     end
   end
 end

lib/net/imap/response_parser.rb

@@ -1535,6 +1535,9 @@ def esearch_response
       # From RFC4731 (ESEARCH):
       #   search-return-data    =/ "MODSEQ" SP mod-sequence-value
       #
+      # From RFC9394 (PARTIAL):
+      #   search-return-data  =/ ret-data-partial
+      #
       def search_return_data
         label = search_modifier_name; SP!
         value =
@@ -1544,11 +1547,41 @@ def search_return_data
           when "ALL"        then sequence_set
           when "COUNT"      then number
           when "MODSEQ"     then mod_sequence_value         # RFC7162: CONDSTORE
+          when "PARTIAL"    then ret_data_partial__value    # RFC9394: PARTIAL
           else search_return_value
           end
         [label, value]
       end
 
+      # From RFC5267 (CONTEXT=SEARCH, CONTEXT=SORT) and RFC9394 (PARTIAL):
+      #   ret-data-partial    = "PARTIAL"
+      #                         SP "(" partial-range SP partial-results ")"
+      def ret_data_partial__value
+        lpar
+        range   = partial_range;   SP!
+        results = partial_results
+        rpar
+        ESearchResult::PartialResult.new(range, results)
+      end
+
+      # partial-range       = partial-range-first / partial-range-last
+      # tagged-ext-simple   =/ partial-range-last
+      def partial_range
+        case (str = atom)
+        when Patterns::PARTIAL_RANGE_FIRST, Patterns::PARTIAL_RANGE_LAST
+          min, max = [Integer($1), Integer($2)].minmax
+          min..max
+        else
+          parse_error("unexpected atom %p, expected partial-range", str)
+        end
+      end
+
+      # partial-results     = sequence-set / "NIL"
+      #     ;; <sequence-set> from [RFC3501].
+      #     ;; NIL indicates that no results correspond to
+      #     ;; the requested range.
+      def partial_results; NIL? ? nil : sequence_set end
+
       # search-modifier-name = tagged-ext-label
       alias search_modifier_name tagged_ext_label
 

rakelib/rfcs.rake

@@ -145,6 +145,7 @@ RFCS = {
   8514 => "IMAP SAVEDATE",
   8970 => "IMAP PREVIEW",
   9208 => "IMAP QUOTA, QUOTA=, QUOTASET",
+  9394 => "IMAP PARTIAL",
 
   # etc...
   3629 => "UTF8",

test/net/imap/fixtures/response_parser/rfc9394_partial.yml

@@ -0,0 +1,66 @@
+---
+:tests:
+
+  "RFC9394 PARTIAL 3.1. example 1":
+    comment: |
+      Neither RFC9394 nor RFC5267 contain any examples of a normal unelided
+      sequence-set result.  I've edited it to include a sequence-set here.
+    :response: "* ESEARCH (TAG \"A01\") UID PARTIAL (-1:-100 200:250,252:300)\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: ESEARCH
+      data: !ruby/object:Net::IMAP::ESearchResult
+        tag: A01
+        uid: true
+        data:
+        - - PARTIAL
+          - !ruby/object:Net::IMAP::ESearchResult::PartialResult
+            range: !ruby/range
+              begin: -100
+              end: -1
+              excl: false
+            results: !ruby/object:Net::IMAP::SequenceSet
+              string: 200:250,252:300
+              tuples:
+              - - 200
+                - 250
+              - - 252
+                - 300
+      raw_data: "* ESEARCH (TAG \"A01\") UID PARTIAL (-1:-100 200:250,252:300)\r\n"
+
+  "RFC9394 PARTIAL 3.1. example 2":
+    :response: "* ESEARCH (TAG \"A02\") UID PARTIAL (23500:24000 55500:56000)\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: ESEARCH
+      data: !ruby/object:Net::IMAP::ESearchResult
+        tag: A02
+        uid: true
+        data:
+        - - PARTIAL
+          - !ruby/object:Net::IMAP::ESearchResult::PartialResult
+            range: !ruby/range
+              begin: 23500
+              end: 24000
+              excl: false
+            results: !ruby/object:Net::IMAP::SequenceSet
+              string: 55500:56000
+              tuples:
+              - - 55500
+                - 56000
+      raw_data: "* ESEARCH (TAG \"A02\") UID PARTIAL (23500:24000 55500:56000)\r\n"
+
+  "RFC9394 PARTIAL 3.1. example 3":
+    :response: "* ESEARCH (TAG \"A04\") UID PARTIAL (24000:24500 NIL)\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: ESEARCH
+      data: !ruby/object:Net::IMAP::ESearchResult
+        tag: A04
+        uid: true
+        data:
+        - - PARTIAL
+          - !ruby/object:Net::IMAP::ESearchResult::PartialResult
+            range: !ruby/range
+              begin: 24000
+              end: 24500
+              excl: false
+            results:
+      raw_data: "* ESEARCH (TAG \"A04\") UID PARTIAL (24000:24500 NIL)\r\n"