add support for filtering/paging/sorting options for "list_**" methods (

#18302)

* update method docstrings

* added tests for list jobs

* updated tests for list docs statuses

* added async tests for job listing

* update old async tests 'list doc statuses'

* fixing some errors

* bug fix async code

* added async tests for 'list all doc statuses'

* update docstrings for list_submitted_jobs

* add time convert helper

* update async client

* updated 'list_document_statuses'

* fix paging issues

* fix artifact parameter names in tests

* remove unwanted tests

* fix order_by parameters

* fix test function names

* remove unwanted imports from tests

* fix kwargs parm name

* fix test list jobs before datetime

* bug fixes

* fixed some tests

* updated tests

* update async tests

* updated 'list_jobs' tests

* skip tests

* temp commit

* updated tests for "list all document statuses" sync client

* updated async tests

* spelling mistakes

* update id parameter name

* fix 'skip' test

* fixed filter by status test

* fix filtering by 'created_before' tests

* fix linting errors

* fix async test

* add support for pytz in tests

* update mixed tests for 'list_submitted_jobs'

* PR comments - speccling mistakes

* PR reviews - min and max errors

* updated mixed-filter tests. still not working!

* fixed bug with converting page to list then calling iterator on empty list

* fix await problem in async test

* adding recorded tests

* white space -> trigger ci/cd pipeline

* update time-baesd failing tests with decorator

* update recorded tests

sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_client.py

@@ -20,7 +20,7 @@
 )
 from ._user_agent import USER_AGENT
 from ._polling import TranslationPolling
-from ._helpers import get_http_logging_policy
+from ._helpers import get_http_logging_policy, convert_datetime
 if TYPE_CHECKING:
     from azure.core.paging import ItemPaged
 
@@ -217,6 +217,17 @@ def list_submitted_jobs(self, **kwargs):
         # type: (**Any) -> ItemPaged[JobStatusResult]
         """List all the submitted translation jobs under the Document Translation resource.
 
+        :keyword int top: the total number of jobs to return (across all pages) from all submitted jobs.
+        :keyword int skip: the number of jobs to skip (from beginning of the all submitted jobs).
+            By default, we sort by all submitted jobs descendingly by start time.
+        :keyword int results_per_page: is the number of jobs returned per page.
+        :keyword list[str] job_ids: job ids to filter by.
+        :keyword list[str] statuses: job statuses to filter by.
+        :keyword Union[str, datetime.datetime] created_after: get jobs created after certain datetime.
+        :keyword Union[str, datetime.datetime] created_before: get jobs created before certain datetime.
+        :keyword list[str] order_by: the sorting query for the jobs returned.
+            format: ["parm1 asc/desc", "parm2 asc/desc", ...]
+            (ex: 'createdDateTimeUtc asc', 'createdDateTimeUtc desc').
         :return: ~azure.core.paging.ItemPaged[:class:`~azure.ai.translation.document.JobStatusResult`]
         :rtype: ~azure.core.paging.ItemPaged
         :raises ~azure.core.exceptions.HttpResponseError:
@@ -230,6 +241,12 @@ def list_submitted_jobs(self, **kwargs):
                 :dedent: 4
                 :caption: List all submitted jobs under the resource.
         """
+        created_after = kwargs.pop("created_after", None)
+        created_before = kwargs.pop("created_before", None)
+        created_after = convert_datetime(created_after) if created_after else None
+        created_before = convert_datetime(created_before) if created_before else None
+        results_per_page = kwargs.pop("results_per_page", None)
+        job_ids = kwargs.pop("job_ids", None)
 
         def _convert_from_generated_model(generated_model):  # pylint: disable=protected-access
             return JobStatusResult._from_generated(generated_model)  # pylint: disable=protected-access
@@ -242,15 +259,30 @@ def _convert_from_generated_model(generated_model):  # pylint: disable=protected
 
         return self._client.document_translation.get_operations(
             cls=model_conversion_function,
+            maxpagesize=results_per_page,
+            created_date_time_utc_start=created_after,
+            created_date_time_utc_end=created_before,
+            ids=job_ids,
             **kwargs
         )
 
     @distributed_trace
     def list_all_document_statuses(self, job_id, **kwargs):
         # type: (str, **Any) -> ItemPaged[DocumentStatusResult]
-        """List all the document statuses under a translation job.
-
-        :param str job_id: The translation job ID.
+        """List all the document statuses for a given translation job.
+
+        :param str job_id: ID of translation job to list documents for.
+        :keyword int top: the total number of documents to return (across all pages).
+        :keyword int skip: the number of documents to skip (from beginning).
+            By default, we sort by all documents descendingly by start time.
+        :keyword int results_per_page: is the number of documents returned per page.
+        :keyword list[str] document_ids: document IDs to filter by.
+        :keyword list[str] statuses: document statuses to filter by.
+        :keyword Union[str, datetime.datetime] translated_after: get document translated after certain datetime.
+        :keyword Union[str, datetime.datetime] translated_before: get document translated before certain datetime.
+        :keyword list[str] order_by: the sorting query for the documents.
+            format: ["parm1 asc/desc", "parm2 asc/desc", ...]
+            (ex: 'createdDateTimeUtc asc', 'createdDateTimeUtc desc').
         :return: ~azure.core.paging.ItemPaged[:class:`~azure.ai.translation.document.DocumentStatusResult`]
         :rtype: ~azure.core.paging.ItemPaged
         :raises ~azure.core.exceptions.HttpResponseError:
@@ -264,6 +296,13 @@ def list_all_document_statuses(self, job_id, **kwargs):
                 :dedent: 4
                 :caption: List all the document statuses under the translation job.
         """
+        translated_after = kwargs.pop("translated_after", None)
+        translated_before = kwargs.pop("translated_before", None)
+        translated_after = convert_datetime(translated_after) if translated_after else None
+        translated_before = convert_datetime(translated_before) if translated_before else None
+        results_per_page = kwargs.pop("results_per_page", None)
+        document_ids = kwargs.pop("document_ids", None)
+
 
         def _convert_from_generated_model(generated_model):
             return DocumentStatusResult._from_generated(generated_model)  # pylint: disable=protected-access
@@ -277,6 +316,10 @@ def _convert_from_generated_model(generated_model):
         return self._client.document_translation.get_operation_documents_status(
             id=job_id,
             cls=model_conversion_function,
+            maxpagesize=results_per_page,
+            created_date_time_utc_start=translated_after,
+            created_date_time_utc_end=translated_before,
+            ids=document_ids,
             **kwargs
         )
 

sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_helpers.py

@@ -4,6 +4,9 @@
 # Licensed under the MIT License.
 # ------------------------------------
 
+import datetime
+from typing import Union
+import six
 from azure.core.pipeline.policies import HttpLoggingPolicy
 
 
@@ -35,3 +38,18 @@ def get_http_logging_policy(**kwargs):
         }
     )
     return http_logging_policy
+
+
+def convert_datetime(date_time):
+    # type: (Union[str, datetime.datetime]) -> datetime.datetime
+    if isinstance(date_time, datetime.datetime):
+        return date_time
+    if isinstance(date_time, six.string_types):
+        try:
+            return datetime.datetime.strptime(date_time, "%Y-%m-%d")
+        except ValueError:
+            try:
+                return datetime.datetime.strptime(date_time, "%Y-%m-%dT%H:%M:%SZ")
+            except ValueError:
+                return datetime.datetime.strptime(date_time, "%Y-%m-%d %H:%M:%S")
+    raise TypeError("Bad datetime type")

sdk/translation/azure-ai-translation-document/azure/ai/translation/document/aio/_client_async.py

@@ -23,7 +23,7 @@
     FileFormat,
     DocumentStatusResult
 )
-from .._helpers import get_http_logging_policy
+from .._helpers import get_http_logging_policy, convert_datetime
 from .._polling import TranslationPolling
 COGNITIVE_KEY_HEADER = "Ocp-Apim-Subscription-Key"
 
@@ -217,6 +217,17 @@ def list_submitted_jobs(self, **kwargs):
         # type: (**Any) -> AsyncItemPaged[JobStatusResult]
         """List all the submitted translation jobs under the Document Translation resource.
 
+        :keyword int top: the total number of jobs to return (across all pages) from all submitted jobs.
+        :keyword int skip: the number of jobs to skip (from beginning of the all submitted jobs).
+            By default, we sort by all submitted jobs descendingly by start time.
+        :keyword int results_per_page: is the number of jobs returned per page.
+        :keyword list[str] job_ids: job ids to filter by.
+        :keyword list[str] statuses: job statuses to filter by.
+        :keyword Union[str, datetime.datetime] created_after: get jobs created after certain datetime.
+        :keyword Union[str, datetime.datetime] created_before: get jobs created before certain datetime.
+        :keyword list[str] order_by: the sorting query for the jobs returned.
+            format: ["parm1 asc/desc", "parm2 asc/desc", ...]
+            (ex: 'createdDateTimeUtc asc', 'createdDateTimeUtc desc').
         :return: ~azure.core.paging.AsyncItemPaged[:class:`~azure.ai.translation.document.JobStatusResult`]
         :rtype: ~azure.core.paging.AsyncItemPaged
         :raises ~azure.core.exceptions.HttpResponseError:
@@ -231,6 +242,13 @@ def list_submitted_jobs(self, **kwargs):
                 :caption: List all submitted jobs under the resource.
         """
 
+        created_after = kwargs.pop("created_after", None)
+        created_before = kwargs.pop("created_before", None)
+        created_after = convert_datetime(created_after) if created_after else None
+        created_before = convert_datetime(created_before) if created_before else None
+        results_per_page = kwargs.pop("results_per_page", None)
+        job_ids = kwargs.pop("job_ids", None)
+
         def _convert_from_generated_model(generated_model):
             # pylint: disable=protected-access
             return JobStatusResult._from_generated(generated_model)
@@ -242,15 +260,30 @@ def _convert_from_generated_model(generated_model):
 
         return self._client.document_translation.get_operations(
             cls=model_conversion_function,
+            maxpagesize=results_per_page,
+            created_date_time_utc_start=created_after,
+            created_date_time_utc_end=created_before,
+            ids=job_ids,
             **kwargs
         )
 
     @distributed_trace
     def list_all_document_statuses(self, job_id, **kwargs):
         # type: (str, **Any) -> AsyncItemPaged[DocumentStatusResult]
-        """List all the document statuses under a translation job.
-
-        :param str job_id: The translation job ID.
+        """List all the document statuses for a given translation job.
+
+        :param str job_id: ID of translation job to list documents for.
+        :keyword int top: the total number of documents to return (across all pages).
+        :keyword int skip: the number of documents to skip (from beginning).
+            By default, we sort by all documents descendingly by start time.
+        :keyword int results_per_page: is the number of documents returned per page.
+        :keyword list[str] document_ids: document IDs to filter by.
+        :keyword list[str] statuses: document statuses to filter by.
+        :keyword Union[str, datetime.datetime] translated_after: get document translated after certain datetime.
+        :keyword Union[str, datetime.datetime] translated_before: get document translated before certain datetime.
+        :keyword list[str] order_by: the sorting query for the documents.
+            format: ["parm1 asc/desc", "parm2 asc/desc", ...]
+            (ex: 'createdDateTimeUtc asc', 'createdDateTimeUtc desc').
         :return: ~azure.core.paging.AsyncItemPaged[:class:`~azure.ai.translation.document.DocumentStatusResult`]
         :rtype: ~azure.core.paging.AsyncItemPaged
         :raises ~azure.core.exceptions.HttpResponseError:
@@ -264,6 +297,12 @@ def list_all_document_statuses(self, job_id, **kwargs):
                 :dedent: 8
                 :caption: List all the document statuses under the translation job.
         """
+        translated_after = kwargs.pop("translated_after", None)
+        translated_before = kwargs.pop("translated_before", None)
+        translated_after = convert_datetime(translated_after) if translated_after else None
+        translated_before = convert_datetime(translated_before) if translated_before else None
+        results_per_page = kwargs.pop("results_per_page", None)
+        document_ids = kwargs.pop("document_ids", None)
 
         def _convert_from_generated_model(generated_model):
             # pylint: disable=protected-access
@@ -277,6 +316,10 @@ def _convert_from_generated_model(generated_model):
         return self._client.document_translation.get_operation_documents_status(
             id=job_id,
             cls=model_conversion_function,
+            maxpagesize=results_per_page,
+            created_date_time_utc_start=translated_after,
+            created_date_time_utc_end=translated_before,
+            ids=document_ids,
             **kwargs
         )
 

sdk/translation/azure-ai-translation-document/dev_requirements.txt

@@ -4,3 +4,4 @@
 ../../storage/azure-storage-blob
 ../../nspkg/azure-ai-translation-nspkg
 aiohttp>=3.0; python_version >= '3.5'
+pytz

sdk/translation/azure-ai-translation-document/tests/asynctestcase.py

@@ -23,23 +23,21 @@ async def _submit_and_validate_translation_job_async(self, async_client, transla
         return job_details.id
 
     # client helpers
-    async def _create_and_submit_sample_translation_jobs_async(self, async_client, jobs_count):
+    async def _create_and_submit_sample_translation_jobs_async(self, async_client, jobs_count, **kwargs):
+        wait_for_job = kwargs.pop('wait', True)
+        language_code = kwargs.pop('language_code', "es")
+        docs_per_job = kwargs.pop('docs_per_job', 2)
         result_job_ids = []
         for i in range(jobs_count):
             # prepare containers and test data
             '''
-                # WARNING!!
-                TOTAL_DOC_COUNT_IN_JOB = 1
-                if you plan to create more docs in the job,
-                please update this variable TOTAL_DOC_COUNT_IN_JOB in respective test
-
                 # note
                 since we're only testing the client library
                 we can use sync container calls in here
                 no need for async container clients!
             '''
-            blob_data = b'This is some text'  
-            source_container_sas_url = self.create_source_container(data=Document(data=blob_data))
+            blob_data = Document.create_dummy_docs(docs_per_job)
+            source_container_sas_url = self.create_source_container(data=blob_data)
             target_container_sas_url = self.create_target_container()
 
             # prepare translation inputs
@@ -49,7 +47,7 @@ async def _create_and_submit_sample_translation_jobs_async(self, async_client, j
                     targets=[
                         TranslationTarget(
                             target_url=target_container_sas_url,
-                            language_code="es"
+                            language_code=language_code
                         )
                     ]
                 )
@@ -58,18 +56,14 @@ async def _create_and_submit_sample_translation_jobs_async(self, async_client, j
             # submit multiple jobs
             job_details = await async_client.create_translation_job(translation_inputs)
             self.assertIsNotNone(job_details.id)
-            await async_client.wait_until_done(job_details.id)
+            if wait_for_job:
+                await async_client.wait_until_done(job_details.id)
             result_job_ids.append(job_details.id)
 
         return result_job_ids
 
 
     async def _create_translation_job_with_dummy_docs_async(self, async_client, docs_count, **kwargs):
-        '''
-            appropriated this method from another PR! #18302
-            please resolve conflict before merge
-            keep in mind it's the exact same method
-        '''
         # get input parms
         wait_for_job = kwargs.pop('wait', False)
         language_code = kwargs.pop('language_code', "es")