diff --git a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py index 97ae8be9bfb15..231625a12d228 100644 --- a/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py +++ b/sdks/python/apache_beam/coders/proto2_coder_test_messages_pb2.py @@ -18,10 +18,10 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: apache_beam/coders/proto2_coder_test_messages.proto - +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -30,293 +30,28 @@ -DESCRIPTOR = _descriptor.FileDescriptor( - name='apache_beam/coders/proto2_coder_test_messages.proto', - package='proto2_coder_test_messages', - syntax='proto2', - serialized_options=b'\n\'org.apache.beam.sdk.extensions.protobuf', - create_key=_descriptor._internal_create_key, - serialized_pb=b'\n3apache_beam/coders/proto2_coder_test_messages.proto\x12\x1aproto2_coder_test_messages\"P\n\x08MessageA\x12\x0e\n\x06\x66ield1\x18\x01 \x01(\t\x12\x34\n\x06\x66ield2\x18\x02 \x03(\x0b\x32$.proto2_coder_test_messages.MessageB\"\x1a\n\x08MessageB\x12\x0e\n\x06\x66ield1\x18\x01 \x01(\x08\"\x10\n\x08MessageC*\x04\x08\x64\x10j\"\xad\x01\n\x0eMessageWithMap\x12\x46\n\x06\x66ield1\x18\x01 \x03(\x0b\x32\x36.proto2_coder_test_messages.MessageWithMap.Field1Entry\x1aS\n\x0b\x46ield1Entry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.proto2_coder_test_messages.MessageA:\x02\x38\x01\"V\n\x18ReferencesMessageWithMap\x12:\n\x06\x66ield1\x18\x01 \x03(\x0b\x32*.proto2_coder_test_messages.MessageWithMap:Z\n\x06\x66ield1\x12$.proto2_coder_test_messages.MessageC\x18\x65 \x01(\x0b\x32$.proto2_coder_test_messages.MessageA:Z\n\x06\x66ield2\x12$.proto2_coder_test_messages.MessageC\x18\x66 \x01(\x0b\x32$.proto2_coder_test_messages.MessageBB)\n\'org.apache.beam.sdk.extensions.protobuf' -) - - -FIELD1_FIELD_NUMBER = 101 -field1 = _descriptor.FieldDescriptor( - name='field1', full_name='proto2_coder_test_messages.field1', index=0, - number=101, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key) -FIELD2_FIELD_NUMBER = 102 -field2 = _descriptor.FieldDescriptor( - name='field2', full_name='proto2_coder_test_messages.field2', index=1, - number=102, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key) - - -_MESSAGEA = _descriptor.Descriptor( - name='MessageA', - full_name='proto2_coder_test_messages.MessageA', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='field1', full_name='proto2_coder_test_messages.MessageA.field1', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='field2', full_name='proto2_coder_test_messages.MessageA.field2', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=83, - serialized_end=163, -) - - -_MESSAGEB = _descriptor.Descriptor( - name='MessageB', - full_name='proto2_coder_test_messages.MessageB', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='field1', full_name='proto2_coder_test_messages.MessageB.field1', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=165, - serialized_end=191, -) - - -_MESSAGEC = _descriptor.Descriptor( - name='MessageC', - full_name='proto2_coder_test_messages.MessageC', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=True, - syntax='proto2', - extension_ranges=[(100, 106), ], - oneofs=[ - ], - serialized_start=193, - serialized_end=209, -) - - -_MESSAGEWITHMAP_FIELD1ENTRY = _descriptor.Descriptor( - name='Field1Entry', - full_name='proto2_coder_test_messages.MessageWithMap.Field1Entry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='proto2_coder_test_messages.MessageWithMap.Field1Entry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - _descriptor.FieldDescriptor( - name='value', full_name='proto2_coder_test_messages.MessageWithMap.Field1Entry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=302, - serialized_end=385, -) - -_MESSAGEWITHMAP = _descriptor.Descriptor( - name='MessageWithMap', - full_name='proto2_coder_test_messages.MessageWithMap', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='field1', full_name='proto2_coder_test_messages.MessageWithMap.field1', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[_MESSAGEWITHMAP_FIELD1ENTRY, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=212, - serialized_end=385, -) - - -_REFERENCESMESSAGEWITHMAP = _descriptor.Descriptor( - name='ReferencesMessageWithMap', - full_name='proto2_coder_test_messages.ReferencesMessageWithMap', - filename=None, - file=DESCRIPTOR, - containing_type=None, - create_key=_descriptor._internal_create_key, - fields=[ - _descriptor.FieldDescriptor( - name='field1', full_name='proto2_coder_test_messages.ReferencesMessageWithMap.field1', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=387, - serialized_end=473, -) - -_MESSAGEA.fields_by_name['field2'].message_type = _MESSAGEB -_MESSAGEWITHMAP_FIELD1ENTRY.fields_by_name['value'].message_type = _MESSAGEA -_MESSAGEWITHMAP_FIELD1ENTRY.containing_type = _MESSAGEWITHMAP -_MESSAGEWITHMAP.fields_by_name['field1'].message_type = _MESSAGEWITHMAP_FIELD1ENTRY -_REFERENCESMESSAGEWITHMAP.fields_by_name['field1'].message_type = _MESSAGEWITHMAP -DESCRIPTOR.message_types_by_name['MessageA'] = _MESSAGEA -DESCRIPTOR.message_types_by_name['MessageB'] = _MESSAGEB -DESCRIPTOR.message_types_by_name['MessageC'] = _MESSAGEC -DESCRIPTOR.message_types_by_name['MessageWithMap'] = _MESSAGEWITHMAP -DESCRIPTOR.message_types_by_name['ReferencesMessageWithMap'] = _REFERENCESMESSAGEWITHMAP -DESCRIPTOR.extensions_by_name['field1'] = field1 -DESCRIPTOR.extensions_by_name['field2'] = field2 -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -MessageA = _reflection.GeneratedProtocolMessageType('MessageA', (_message.Message,), { - 'DESCRIPTOR' : _MESSAGEA, - '__module__' : 'apache_beam.coders.proto2_coder_test_messages_pb2' - # @@protoc_insertion_point(class_scope:proto2_coder_test_messages.MessageA) - }) -_sym_db.RegisterMessage(MessageA) - -MessageB = _reflection.GeneratedProtocolMessageType('MessageB', (_message.Message,), { - 'DESCRIPTOR' : _MESSAGEB, - '__module__' : 'apache_beam.coders.proto2_coder_test_messages_pb2' - # @@protoc_insertion_point(class_scope:proto2_coder_test_messages.MessageB) - }) -_sym_db.RegisterMessage(MessageB) - -MessageC = _reflection.GeneratedProtocolMessageType('MessageC', (_message.Message,), { - 'DESCRIPTOR' : _MESSAGEC, - '__module__' : 'apache_beam.coders.proto2_coder_test_messages_pb2' - # @@protoc_insertion_point(class_scope:proto2_coder_test_messages.MessageC) - }) -_sym_db.RegisterMessage(MessageC) - -MessageWithMap = _reflection.GeneratedProtocolMessageType('MessageWithMap', (_message.Message,), { - - 'Field1Entry' : _reflection.GeneratedProtocolMessageType('Field1Entry', (_message.Message,), { - 'DESCRIPTOR' : _MESSAGEWITHMAP_FIELD1ENTRY, - '__module__' : 'apache_beam.coders.proto2_coder_test_messages_pb2' - # @@protoc_insertion_point(class_scope:proto2_coder_test_messages.MessageWithMap.Field1Entry) - }) - , - 'DESCRIPTOR' : _MESSAGEWITHMAP, - '__module__' : 'apache_beam.coders.proto2_coder_test_messages_pb2' - # @@protoc_insertion_point(class_scope:proto2_coder_test_messages.MessageWithMap) - }) -_sym_db.RegisterMessage(MessageWithMap) -_sym_db.RegisterMessage(MessageWithMap.Field1Entry) - -ReferencesMessageWithMap = _reflection.GeneratedProtocolMessageType('ReferencesMessageWithMap', (_message.Message,), { - 'DESCRIPTOR' : _REFERENCESMESSAGEWITHMAP, - '__module__' : 'apache_beam.coders.proto2_coder_test_messages_pb2' - # @@protoc_insertion_point(class_scope:proto2_coder_test_messages.ReferencesMessageWithMap) - }) -_sym_db.RegisterMessage(ReferencesMessageWithMap) - -field1.message_type = _MESSAGEA -MessageC.RegisterExtension(field1) -field2.message_type = _MESSAGEB -MessageC.RegisterExtension(field2) - -DESCRIPTOR._options = None -_MESSAGEWITHMAP_FIELD1ENTRY._options = None +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n3apache_beam/coders/proto2_coder_test_messages.proto\x12\x1aproto2_coder_test_messages\"P\n\x08MessageA\x12\x0e\n\x06\x66ield1\x18\x01 \x01(\t\x12\x34\n\x06\x66ield2\x18\x02 \x03(\x0b\x32$.proto2_coder_test_messages.MessageB\"\x1a\n\x08MessageB\x12\x0e\n\x06\x66ield1\x18\x01 \x01(\x08\"\x10\n\x08MessageC*\x04\x08\x64\x10j\"\xad\x01\n\x0eMessageWithMap\x12\x46\n\x06\x66ield1\x18\x01 \x03(\x0b\x32\x36.proto2_coder_test_messages.MessageWithMap.Field1Entry\x1aS\n\x0b\x46ield1Entry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x33\n\x05value\x18\x02 \x01(\x0b\x32$.proto2_coder_test_messages.MessageA:\x02\x38\x01\"V\n\x18ReferencesMessageWithMap\x12:\n\x06\x66ield1\x18\x01 \x03(\x0b\x32*.proto2_coder_test_messages.MessageWithMap:Z\n\x06\x66ield1\x12$.proto2_coder_test_messages.MessageC\x18\x65 \x01(\x0b\x32$.proto2_coder_test_messages.MessageA:Z\n\x06\x66ield2\x12$.proto2_coder_test_messages.MessageC\x18\x66 \x01(\x0b\x32$.proto2_coder_test_messages.MessageBB)\n\'org.apache.beam.sdk.extensions.protobuf') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'apache_beam.coders.proto2_coder_test_messages_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + MessageC.RegisterExtension(field1) + MessageC.RegisterExtension(field2) + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\'org.apache.beam.sdk.extensions.protobuf' + _MESSAGEWITHMAP_FIELD1ENTRY._options = None + _MESSAGEWITHMAP_FIELD1ENTRY._serialized_options = b'8\001' + _MESSAGEA._serialized_start=83 + _MESSAGEA._serialized_end=163 + _MESSAGEB._serialized_start=165 + _MESSAGEB._serialized_end=191 + _MESSAGEC._serialized_start=193 + _MESSAGEC._serialized_end=209 + _MESSAGEWITHMAP._serialized_start=212 + _MESSAGEWITHMAP._serialized_end=385 + _MESSAGEWITHMAP_FIELD1ENTRY._serialized_start=302 + _MESSAGEWITHMAP_FIELD1ENTRY._serialized_end=385 + _REFERENCESMESSAGEWITHMAP._serialized_start=387 + _REFERENCESMESSAGEWITHMAP._serialized_end=473 # @@protoc_insertion_point(module_scope) diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py index be912ae337619..17568cdd72577 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py @@ -477,7 +477,7 @@ def process(self, element): client_element = self.element_to_client_batch_item(element) self._batch_elements.append(client_element) self.add_to_batch(client_element) - self._batch_bytes_size += self._batch.mutations[-1].ByteSize() + self._batch_bytes_size += self._batch.mutations[-1]._pb.ByteSize() if (len(self._batch.mutations) >= self._target_batch_size or self._batch_bytes_size > util.WRITE_BATCH_MAX_BYTES_SIZE): diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py index 8a7977e475cb9..076a95178d83d 100644 --- a/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py +++ b/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py @@ -49,6 +49,20 @@ client = None +# used for internal testing only +class FakeMessage: + def __init__(self, entity, key): + self.entity = entity + self.key = key + + def ByteSize(self): + if self.entity is not None: + return helpers.entity_to_protobuf(self.entity)._pb.ByteSize() + else: + return self.key.to_protobuf()._pb.ByteSize() + + +# used for internal testing only class FakeMutation(object): def __init__(self, entity=None, key=None): """Fake mutation request object. @@ -63,12 +77,7 @@ def __init__(self, entity=None, key=None): """ self.entity = entity self.key = key - - def ByteSize(self): - if self.entity is not None: - return helpers.entity_to_protobuf(self.entity).ByteSize() - else: - return self.key.to_protobuf().ByteSize() + self._pb = FakeMessage(entity, key) class FakeBatch(object): diff --git a/sdks/python/apache_beam/ml/gcp/__init__.py b/sdks/python/apache_beam/ml/gcp/__init__.py index cce3acad34a49..63e52f1d1f177 100644 --- a/sdks/python/apache_beam/ml/gcp/__init__.py +++ b/sdks/python/apache_beam/ml/gcp/__init__.py @@ -14,3 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +# TODO(https://github.com/apache/beam/issues/25625) + +# update code. For example, +# use class something(beam.PTransform) instead of decorator @beam.ptransform_fn +# on a function. + +# update type annotations to follow pep 484 https://peps.python.org/pep-0484/ +# and also update doc strings. diff --git a/sdks/python/apache_beam/ml/gcp/naturallanguageml.py b/sdks/python/apache_beam/ml/gcp/naturallanguageml.py index 7817eb9c4c237..4f63aef68232b 100644 --- a/sdks/python/apache_beam/ml/gcp/naturallanguageml.py +++ b/sdks/python/apache_beam/ml/gcp/naturallanguageml.py @@ -26,8 +26,7 @@ try: from google.cloud import language - from google.cloud.language import enums # pylint: disable=unused-import - from google.cloud.language import types + from google.cloud import language_v1 except ImportError: raise ImportError( 'Google Cloud Natural Language API not supported for this execution ' @@ -42,7 +41,7 @@ class Document(object): Args: content (str): The content of the input or the Google Cloud Storage URI where the file is stored. - type (`Union[str, google.cloud.language.enums.Document.Type]`): Text type. + type (`Union[str, google.cloud.language_v1.Document.Type]`): Text type. Possible values are `HTML`, `PLAIN_TEXT`. The default value is `PLAIN_TEXT`. language_hint (`Optional[str]`): The language of the text. If not specified, @@ -57,7 +56,7 @@ class Document(object): def __init__( self, content, # type: str - type='PLAIN_TEXT', # type: Union[str, enums.Document.Type] + type='PLAIN_TEXT', # type: Union[str, language_v1.Document.Type] language_hint=None, # type: Optional[str] encoding='UTF8', # type: Optional[str] from_gcs=False # type: bool @@ -84,7 +83,7 @@ def to_dict(document): @beam.ptransform_fn def AnnotateText( pcoll, # type: beam.pvalue.PCollection - features, # type: Union[Mapping[str, bool], types.AnnotateTextRequest.Features] + features, # type: Union[Mapping[str, bool], language_v1.AnnotateTextRequest.Features] timeout=None, # type: Optional[float] metadata=None # type: Optional[Sequence[Tuple[str, str]]] ): @@ -98,7 +97,6 @@ def AnnotateText( features (`Union[Mapping[str, bool], types.AnnotateTextRequest.Features]`): A dictionary of natural language operations to be performed on given text in the following format:: - {'extact_syntax'=True, 'extract_entities'=True} timeout (`Optional[float]`): The amount of time, in seconds, to wait @@ -111,11 +109,11 @@ def AnnotateText( @beam.typehints.with_input_types(Document) -@beam.typehints.with_output_types(types.AnnotateTextResponse) +@beam.typehints.with_output_types(language_v1.AnnotateTextResponse) class _AnnotateTextFn(beam.DoFn): def __init__( self, - features, # type: Union[Mapping[str, bool], types.AnnotateTextRequest.Features] + features, # type: Union[Mapping[str, bool], language_v1.AnnotateTextRequest.Features] timeout, # type: Optional[float] metadata=None # type: Optional[Sequence[Tuple[str, str]]] ): diff --git a/sdks/python/apache_beam/ml/gcp/naturallanguageml_test.py b/sdks/python/apache_beam/ml/gcp/naturallanguageml_test.py index e6395176091f2..bad7443d0d944 100644 --- a/sdks/python/apache_beam/ml/gcp/naturallanguageml_test.py +++ b/sdks/python/apache_beam/ml/gcp/naturallanguageml_test.py @@ -65,7 +65,7 @@ def test_annotate_test_called(self): '._get_api_client'): p = TestPipeline() features = [ - naturallanguageml.types.AnnotateTextRequest.Features( + naturallanguageml.language_v1.AnnotateTextRequest.Features( extract_syntax=True) ] _ = ( diff --git a/sdks/python/apache_beam/ml/gcp/videointelligenceml.py b/sdks/python/apache_beam/ml/gcp/videointelligenceml.py index fb0d7f045ddea..ebd35d2426c0a 100644 --- a/sdks/python/apache_beam/ml/gcp/videointelligenceml.py +++ b/sdks/python/apache_beam/ml/gcp/videointelligenceml.py @@ -63,7 +63,7 @@ def __init__( context_side_input=None): """ Args: - features: (List[``videointelligence_v1.enums.Feature``]) Required. + features: (List[``videointelligence_v1.Feature``]) Required. The Video Intelligence API features to detect location_id: (str) Optional. Cloud region where annotation should take place. @@ -82,9 +82,9 @@ def __init__( video_contexts = [('gs://cloud-samples-data/video/cat.mp4', Union[dict, - ``videointelligence_v1.types.VideoContext``]), + ``videointelligence_v1.VideoContext``]), ('gs://some-other-video/sample.mp4', Union[dict, - ``videointelligence_v1.types.VideoContext``]),] + ``videointelligence_v1.VideoContext``]),] context_side_input = ( @@ -113,11 +113,11 @@ def expand(self, pvalue): @typehints.with_input_types( - Union[str, bytes], Optional[videointelligence.types.VideoContext]) + Union[str, bytes], Optional[videointelligence.VideoContext]) class _VideoAnnotateFn(DoFn): """A DoFn that sends each input element to the GCP Video Intelligence API service and outputs an element with the return result of the API - (``google.cloud.videointelligence_v1.types.AnnotateVideoResponse``). + (``google.cloud.videointelligence_v1.AnnotateVideoResponse``). """ def __init__(self, features, location_id, metadata, timeout): super().__init__() @@ -166,7 +166,7 @@ class AnnotateVideoWithContext(AnnotateVideo): Element is a tuple of (Union[str, bytes], - Optional[videointelligence.types.VideoContext]) + Optional[videointelligence.VideoContext]) where the former is either an URI (e.g. a GCS URI) or bytes base64-encoded video data @@ -174,7 +174,7 @@ class AnnotateVideoWithContext(AnnotateVideo): def __init__(self, features, location_id=None, metadata=None, timeout=120): """ Args: - features: (List[``videointelligence_v1.enums.Feature``]) Required. + features: (List[``videointelligence_v1.Feature``]) Required. the Video Intelligence API features to detect location_id: (str) Optional. Cloud region where annotation should take place. @@ -202,12 +202,12 @@ def expand(self, pvalue): @typehints.with_input_types( - Tuple[Union[str, bytes], Optional[videointelligence.types.VideoContext]]) + Tuple[Union[str, bytes], Optional[videointelligence.VideoContext]]) class _VideoAnnotateFnWithContext(_VideoAnnotateFn): """A DoFn that unpacks each input tuple to element, video_context variables and sends these to the GCP Video Intelligence API service and outputs an element with the return result of the API - (``google.cloud.videointelligence_v1.types.AnnotateVideoResponse``). + (``google.cloud.videointelligence_v1.AnnotateVideoResponse``). """ def __init__(self, features, location_id, metadata, timeout): super().__init__( diff --git a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test.py b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test.py index 3215cebb4a88d..79c841938cdbf 100644 --- a/sdks/python/apache_beam/ml/gcp/videointelligenceml_test.py +++ b/sdks/python/apache_beam/ml/gcp/videointelligenceml_test.py @@ -47,11 +47,11 @@ def setUp(self): self.m2 = mock.Mock() self.m2.result.return_value = None self._mock_client.annotate_video.return_value = self.m2 - self.features = [videointelligence.enums.Feature.LABEL_DETECTION] + self.features = [videointelligence.Feature.LABEL_DETECTION] self.location_id = 'us-west1' - config = videointelligence.types.SpeechTranscriptionConfig( + config = videointelligence.SpeechTranscriptionConfig( language_code='en-US', enable_automatic_punctuation=True) - self.video_ctx = videointelligence.types.VideoContext( + self.video_ctx = videointelligence.VideoContext( speech_transcription_config=config) def test_AnnotateVideo_with_side_input_context(self): diff --git a/sdks/python/apache_beam/ml/inference/pytorch_inference.py b/sdks/python/apache_beam/ml/inference/pytorch_inference.py index 71a4ccc63a27b..818dd8325dcc1 100644 --- a/sdks/python/apache_beam/ml/inference/pytorch_inference.py +++ b/sdks/python/apache_beam/ml/inference/pytorch_inference.py @@ -97,7 +97,7 @@ def _load_model( "Loading state_dict_path %s onto a %s device", state_dict_path, device) if not torch_script_model_path: file = FileSystems.open(state_dict_path, 'rb') - model = model_class(**model_params) # type: ignore[misc] + model = model_class(**model_params) # type: ignore[arg-type,misc] state_dict = torch.load(file, map_location=device) model.load_state_dict(state_dict) else: diff --git a/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py b/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py index bdc0291dd1ed1..9c814062e6ed4 100644 --- a/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/tensorflow_inference_it_test.py @@ -18,6 +18,7 @@ """End-to-End test for Tensorflow Inference""" import logging +import sys import unittest import uuid from pathlib import Path @@ -66,6 +67,10 @@ def clear_tf_hub_temp_dir(model_path): rmdir(local_path) +@unittest.skipIf( + sys.version_info.major == 3 and sys.version_info.minor == 7, + "Tensorflow tests on Python 3.7 with Apache Beam 2.47.0 or " + "greater are skipped since tensorflow>=2.12 doesn't support Python 3.7") @unittest.skipIf( tf is None, 'Missing dependencies. ' 'Test depends on tensorflow') diff --git a/sdks/python/apache_beam/ml/inference/tensorflow_tests_requirements.txt b/sdks/python/apache_beam/ml/inference/tensorflow_tests_requirements.txt index 8a9deba61dd89..03dca6ca23447 100644 --- a/sdks/python/apache_beam/ml/inference/tensorflow_tests_requirements.txt +++ b/sdks/python/apache_beam/ml/inference/tensorflow_tests_requirements.txt @@ -15,7 +15,7 @@ # limitations under the License. # -tensorflow>=1.0.0 +tensorflow>=2.12.0rc1 tensorflow_hub>=0.10.0 Pillow>=9.0.0 diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py index a4966e64559dc..102b8b60d69a9 100644 --- a/sdks/python/apache_beam/runners/pipeline_context.py +++ b/sdks/python/apache_beam/runners/pipeline_context.py @@ -310,7 +310,15 @@ def get_or_create_environment_with_resource_hints( """Creates an environment that has necessary hints and returns its id.""" template_env = self.environments.get_proto_from_id(template_env_id) cloned_env = beam_runner_api_pb2.Environment() - cloned_env.CopyFrom(template_env) + # (TODO https://github.com/apache/beam/issues/25615) + # Remove the suppress warning for type once mypy is updated to 0.941 or + # higher. + # mypy 0.790 throws the warning below but 0.941 doesn't. + # error: Argument 1 to "CopyFrom" of "Message" has incompatible type + # "Message"; expected "Environment" [arg-type] + # Here, Environment is a subclass of Message but mypy still + # throws an error. + cloned_env.CopyFrom(template_env) # type: ignore[arg-type] cloned_env.resource_hints.clear() cloned_env.resource_hints.update(resource_hints) diff --git a/sdks/python/apache_beam/runners/portability/abstract_job_service.py b/sdks/python/apache_beam/runners/portability/abstract_job_service.py index a369af94e7dd4..1aa841df4c313 100644 --- a/sdks/python/apache_beam/runners/portability/abstract_job_service.py +++ b/sdks/python/apache_beam/runners/portability/abstract_job_service.py @@ -194,7 +194,7 @@ class AbstractBeamJob(object): def __init__(self, job_id, # type: str - job_name, # type: Optional[str] + job_name, # type: str pipeline, # type: beam_runner_api_pb2.Pipeline options # type: struct_pb2.Struct ): diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py index 400cf27f34447..0a59b53111376 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py @@ -500,7 +500,7 @@ def __init__(self, execution_context, windowing_strategy_proto): self._counter = 0 # Lazily created in make_process_bundle_descriptor() self._process_bundle_descriptor = None - self._bundle_processor_id = None # type: Optional[str] + self._bundle_processor_id = '' # type: str self.windowed_input_coder_impl = None # type: Optional[CoderImpl] self.windowed_output_coder_impl = None # type: Optional[CoderImpl] @@ -677,7 +677,7 @@ def make_coder(urn, *components): windowing_strategy_id=global_windowing_strategy_id, coder_id=output_coder_id), }, - coders=coders, # type: ignore + coders=coders, windowing_strategies={ global_windowing_strategy_id: global_windowing_strategy_proto, }, diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py index 052029e1cf7bb..8d957068d08b0 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py @@ -1105,7 +1105,7 @@ class ExtendedProvisionInfo(object): def __init__(self, provision_info=None, # type: Optional[beam_provision_api_pb2.ProvisionInfo] artifact_staging_dir=None, # type: Optional[str] - job_name=None, # type: Optional[str] + job_name='', # type: str ): # type: (...) -> None self.provision_info = ( diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/translations.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/translations.py index 0a289056a6a81..df3578e649740 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/translations.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/translations.py @@ -256,7 +256,7 @@ def has_as_main_input(self, pcoll): transform.spec.payload, beam_runner_api_pb2.ParDoPayload) local_side_inputs = payload.side_inputs else: - local_side_inputs = {} + local_side_inputs = {} # type: ignore[assignment] for local_id, pipeline_id in transform.inputs.items(): if pcoll == pipeline_id and local_id not in local_side_inputs: return True diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py index abb356d5ff4e7..b11c8349909c6 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py @@ -428,7 +428,7 @@ def GetProvisionInfo(self, request, context=None): worker_id = dict(context.invocation_metadata())['worker_id'] worker = self._worker_manager.get_worker(worker_id) info = copy.copy(worker.provision_info.provision_info) - info.logging_endpoint.CopyFrom(worker.logging_api_service_descriptor()) # type: ignore + info.logging_endpoint.CopyFrom(worker.logging_api_service_descriptor()) info.artifact_endpoint.CopyFrom(worker.artifact_api_service_descriptor()) info.control_endpoint.CopyFrom(worker.control_api_service_descriptor()) else: diff --git a/sdks/python/apache_beam/runners/portability/local_job_service.py b/sdks/python/apache_beam/runners/portability/local_job_service.py index 7f1908a7e7e1c..91ddb3fced15c 100644 --- a/sdks/python/apache_beam/runners/portability/local_job_service.py +++ b/sdks/python/apache_beam/runners/portability/local_job_service.py @@ -239,7 +239,7 @@ class BeamJob(abstract_job_service.AbstractBeamJob): """ def __init__(self, - job_id, # type: str + job_id, # type: str pipeline, options, provision_info, # type: fn_runner.ExtendedProvisionInfo diff --git a/sdks/python/apache_beam/runners/worker/log_handler.py b/sdks/python/apache_beam/runners/worker/log_handler.py index 5731a4be05b0e..16aef3b0707a7 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler.py +++ b/sdks/python/apache_beam/runners/worker/log_handler.py @@ -112,7 +112,7 @@ def connect(self): return self._logging_stub.Logging(self._write_log_entries()) def map_log_level(self, level): - # type: (int) -> beam_fn_api_pb2.LogEntry.Severity.Enum + # type: (int) -> beam_fn_api_pb2.LogEntry.Severity.Enum.ValueType try: return LOG_LEVEL_TO_LOGENTRY_MAP[level] except KeyError: diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index cdc96b52b3783..47aaeff43a6fa 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -3164,8 +3164,8 @@ class Windowing(object): def __init__(self, windowfn, # type: WindowFn triggerfn=None, # type: typing.Optional[TriggerFn] - accumulation_mode=None, # type: typing.Optional[beam_runner_api_pb2.AccumulationMode.Enum] - timestamp_combiner=None, # type: typing.Optional[beam_runner_api_pb2.OutputTime.Enum] + accumulation_mode=None, # type: typing.Optional[beam_runner_api_pb2.AccumulationMode.Enum.ValueType] + timestamp_combiner=None, # type: typing.Optional[beam_runner_api_pb2.OutputTime.Enum.ValueType] allowed_lateness=0, # type: typing.Union[int, float] environment_id=None, # type: typing.Optional[str] ): diff --git a/sdks/python/apache_beam/transforms/external.py b/sdks/python/apache_beam/transforms/external.py index 1c4a6dd05197b..3ab0b87b09ed1 100644 --- a/sdks/python/apache_beam/transforms/external.py +++ b/sdks/python/apache_beam/transforms/external.py @@ -595,7 +595,7 @@ def expand(self, pvalueish): components = context.to_runner_api() request = beam_expansion_api_pb2.ExpansionRequest( components=components, - namespace=self._external_namespace, # type: ignore # mypy thinks self._namespace is threading.local + namespace=self._external_namespace, transform=transform_proto, output_coder_requests=output_coders) diff --git a/sdks/python/build-requirements.txt b/sdks/python/build-requirements.txt index f6bd7ffdbf264..067528cd99d18 100644 --- a/sdks/python/build-requirements.txt +++ b/sdks/python/build-requirements.txt @@ -18,20 +18,9 @@ # TODO(https://github.com/apache/beam/issues/20051): Consider PEP-517/PEP-518 instead of this file. setuptools -# grpcio-tools depends on grpcio and the grpcio>1.50.0 results in error(ImportModuleError six) -# when installing Apache Beam source via pip install -e . -# Adding six as part of build dependencies. -# https://github.com/apache/beam/issues/24432 -six wheel>=0.36.0 - -grpcio-tools==1.37.0 -# TODO(https://github.com/apache/beam/issues/23734): the sdist for grpcio==1.50.0 is failing on GH workers -# pin grpcio to the previous version. -grpcio==1.51.3;sys_platform=="darwin" -mypy-protobuf==1.18 -protobuf==3.19.4;python_version=="3.10" and sys_platform=="darwin" - +grpcio-tools==1.51.1 +mypy-protobuf==3.4.0 # Avoid https://github.com/pypa/virtualenv/issues/2006 distlib==0.3.6 diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile index fe5238d22ce31..c9437504588ed 100644 --- a/sdks/python/container/Dockerfile +++ b/sdks/python/container/Dockerfile @@ -42,8 +42,8 @@ RUN \ pip install --no-deps -r /tmp/base_image_requirements.txt && \ python -c "import nltk; nltk.download('stopwords')" && \ rm /root/nltk_data/corpora/stopwords.zip && \ - # Check that the fast implementation of protobuf is used. - python -c "from google.protobuf.internal import api_implementation; assert api_implementation._default_implementation_type == 'cpp'; print ('Verified fast protobuf used.')" && \ + # Check that the protobuf upb(also called micro protobuf) is used. + python -c "from google.protobuf.internal import api_implementation; assert api_implementation._implementation_type == 'upb'; print ('Verified fast protobuf used.')" && \ # Remove pip cache. rm -rf /root/.cache/pip && \ rm -rf /tmp/base_image_requirements.txt diff --git a/sdks/python/container/base_image_requirements_manual.txt b/sdks/python/container/base_image_requirements_manual.txt index b3612fa6af394..71cd6c2dcb979 100644 --- a/sdks/python/container/base_image_requirements_manual.txt +++ b/sdks/python/container/base_image_requirements_manual.txt @@ -32,9 +32,6 @@ cython<1 # some versions of libraries that launch Beam pipelines, like tensorflow-transform. # Leaving 'future' in our containers for now prevent breaking tft users. future -# TODO: Remove the upper bound once Tensorflow 2.11 is released. -# https://github.com/apache/beam/issues/23355 -google-cloud-profiler<4.0.0 guppy3 # Memory profiler mmh3 # Optimizes execution of some Beam codepaths. TODO: Make it Beam's dependency. nltk # Commonly used for natural language processing. @@ -42,4 +39,4 @@ nose==1.3.7 # For Dataflow internal testing. TODO: remove this. python-snappy # Optimizes execution of some Beam codepaths. scipy scikit-learn -tensorflow +tensorflow>=2.12.0rc1;python_version>="3.8" diff --git a/sdks/python/container/license_scripts/dep_urls_py.yaml b/sdks/python/container/license_scripts/dep_urls_py.yaml index 9aa1111e85db8..a0b1cd9597356 100644 --- a/sdks/python/container/license_scripts/dep_urls_py.yaml +++ b/sdks/python/container/license_scripts/dep_urls_py.yaml @@ -81,6 +81,9 @@ pip_dependencies: notice: "https://raw.githubusercontent.com/grpc/grpc/master/NOTICE.txt" grpcio-gcp: license: "https://raw.githubusercontent.com/GoogleCloudPlatform/grpc-gcp-python/master/LICENSE" + grpcio-status: + license: "https://raw.githubusercontent.com/grpc/grpc/master/LICENSE" + notice: "https://raw.githubusercontent.com/grpc/grpc/master/NOTICE.txt" guppy: license: "https://raw.githubusercontent.com/joshwcomeau/guppy/master/LICENSE.md" guppy3: diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt index 98997055d64e7..6098423b23a2d 100644 --- a/sdks/python/container/py310/base_image_requirements.txt +++ b/sdks/python/container/py310/base_image_requirements.txt @@ -29,11 +29,11 @@ bs4==0.0.1 cachetools==4.2.4 certifi==2022.12.7 cffi==1.15.1 -charset-normalizer==3.0.1 +charset-normalizer==3.1.0 click==8.1.3 cloudpickle==2.2.1 crcmod==1.7 -cryptography==39.0.1 +cryptography==39.0.2 Cython==0.29.33 deprecation==2.1.0 dill==0.3.1.1 @@ -42,70 +42,68 @@ docker==6.0.1 docopt==0.6.2 exceptiongroup==1.1.0 execnet==1.9.0 -fastavro==1.7.1 +fastavro==1.7.3 fasteners==0.18 -flatbuffers==23.1.21 +flatbuffers==23.3.3 freezegun==1.2.2 future==0.18.3 gast==0.4.0 -google-api-core==2.8.2 -google-api-python-client==2.78.0 +google-api-core==2.11.0 google-apitools==0.5.31 -google-auth==2.16.0 +google-auth==2.16.2 google-auth-httplib2==0.1.0 google-auth-oauthlib==0.4.6 -google-cloud-bigquery==3.3.3 -google-cloud-bigquery-storage==2.16.0 -google-cloud-bigtable==1.7.3 +google-cloud-bigquery==3.6.0 +google-cloud-bigquery-storage==2.19.0 +google-cloud-bigtable==2.17.0 google-cloud-core==2.3.2 -google-cloud-datastore==1.15.5 -google-cloud-dlp==3.9.0 -google-cloud-language==1.3.2 -google-cloud-profiler==3.1.0 -google-cloud-pubsub==2.13.7 -google-cloud-pubsublite==1.6.0 -google-cloud-recommendations-ai==0.7.1 -google-cloud-spanner==3.22.0 -google-cloud-videointelligence==1.16.3 -google-cloud-vision==3.1.2 +google-cloud-datastore==2.15.0 +google-cloud-dlp==3.12.0 +google-cloud-language==2.9.0 +google-cloud-pubsub==2.15.0 +google-cloud-pubsublite==1.7.0 +google-cloud-recommendations-ai==0.10.2 +google-cloud-spanner==3.28.0 +google-cloud-videointelligence==2.11.0 +google-cloud-vision==3.4.0 google-crc32c==1.5.0 google-pasta==0.2.0 google-resumable-media==2.4.1 -googleapis-common-protos==1.56.4 +googleapis-common-protos==1.58.0 greenlet==2.0.2 -grpc-google-iam-v1==0.12.4 -grpcio==1.51.1 -grpcio-status==1.48.2 +grpc-google-iam-v1==0.12.6 +grpcio==1.51.3 +grpcio-status==1.51.3 guppy3==3.1.2 h5py==3.8.0 hdfs==2.7.0 httplib2==0.21.0 -hypothesis==6.68.1 +hypothesis==6.68.2 idna==3.4 iniconfig==2.0.0 +jax==0.4.6 joblib==1.2.0 -keras==2.11.0 +keras==2.12.0rc1 libclang==15.0.6.1 Markdown==3.4.1 MarkupSafe==2.1.2 mmh3==3.0.0 -mock==2.0.0 +mock==5.0.1 nltk==3.8.1 nose==1.3.7 -numpy==1.24.2 +numpy==1.23.5 oauth2client==4.1.3 oauthlib==3.2.2 objsize==0.6.1 opt-einsum==3.3.0 -orjson==3.8.6 +orjson==3.8.7 overrides==6.5.0 -packaging==21.3 +packaging==23.0 pandas==1.5.3 parameterized==0.8.1 -pbr==5.11.1 pluggy==1.0.0 proto-plus==1.22.2 -protobuf==3.19.4 +protobuf==4.22.1 psycopg2-binary==2.9.5 py==1.11.0 pyarrow==9.0.0 @@ -117,7 +115,7 @@ PyHamcrest==1.10.1 pymongo==4.3.3 PyMySQL==1.0.2 pyparsing==3.0.9 -pytest==7.2.1 +pytest==7.2.2 pytest-forked==1.6.0 pytest-timeout==2.1.0 pytest-xdist==2.5.0 @@ -130,29 +128,28 @@ requests==2.28.2 requests-mock==1.10.0 requests-oauthlib==1.3.1 rsa==4.9 -scikit-learn==1.2.1 -scipy==1.10.0 +scikit-learn==1.2.2 +scipy==1.10.1 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.4 SQLAlchemy==1.4.46 sqlparse==0.4.3 tenacity==5.1.5 -tensorboard==2.11.2 -tensorboard-data-server==0.6.1 +tensorboard==2.12.0 +tensorboard-data-server==0.7.0 tensorboard-plugin-wit==1.8.1 -tensorflow==2.11.0 -tensorflow-estimator==2.11.0 -tensorflow-io-gcs-filesystem==0.30.0 +tensorflow==2.12.0rc1 +tensorflow-estimator==2.12.0rc0 +tensorflow-io-gcs-filesystem==0.31.0 termcolor==2.2.0 testcontainers==3.7.1 threadpoolctl==3.1.0 tomli==2.0.1 -tqdm==4.64.1 -typing_extensions==4.4.0 -uritemplate==4.1.1 +tqdm==4.65.0 +typing_extensions==4.5.0 urllib3==1.26.14 websocket-client==1.5.1 Werkzeug==2.2.3 wrapt==1.14.1 -zstandard==0.19.0 +zstandard==0.20.0 diff --git a/sdks/python/container/py37/base_image_requirements.txt b/sdks/python/container/py37/base_image_requirements.txt index 5feb1651f0486..2464ac8f9073a 100644 --- a/sdks/python/container/py37/base_image_requirements.txt +++ b/sdks/python/container/py37/base_image_requirements.txt @@ -21,19 +21,17 @@ # https://s.apache.org/beam-python-dev-wiki # Reach out to a committer if you need help. -absl-py==1.4.0 -astunparse==1.6.3 attrs==22.2.0 beautifulsoup4==4.11.2 bs4==0.0.1 cachetools==4.2.4 certifi==2022.12.7 cffi==1.15.1 -charset-normalizer==3.0.1 +charset-normalizer==3.1.0 click==8.1.3 cloudpickle==2.2.1 crcmod==1.7 -cryptography==39.0.1 +cryptography==39.0.2 Cython==0.29.33 deprecation==2.1.0 dill==0.3.1.1 @@ -42,71 +40,57 @@ docker==6.0.1 docopt==0.6.2 exceptiongroup==1.1.0 execnet==1.9.0 -fastavro==1.7.1 +fastavro==1.7.3 fasteners==0.18 -flatbuffers==23.1.21 freezegun==1.2.2 future==0.18.3 -gast==0.4.0 google-api-core==2.11.0 -google-api-python-client==2.78.0 google-apitools==0.5.31 -google-auth==2.16.0 +google-auth==2.16.2 google-auth-httplib2==0.1.0 -google-auth-oauthlib==0.4.6 -google-cloud-bigquery==3.5.0 -google-cloud-bigquery-storage==2.16.2 -google-cloud-bigtable==1.7.3 +google-cloud-bigquery==3.6.0 +google-cloud-bigquery-storage==2.19.0 +google-cloud-bigtable==2.17.0 google-cloud-core==2.3.2 -google-cloud-datastore==1.15.5 -google-cloud-dlp==3.11.1 -google-cloud-language==1.3.2 -google-cloud-profiler==3.1.0 -google-cloud-pubsub==2.14.1 -google-cloud-pubsublite==1.6.0 -google-cloud-recommendations-ai==0.7.1 -google-cloud-spanner==3.27.1 -google-cloud-videointelligence==1.16.3 -google-cloud-vision==3.3.1 +google-cloud-datastore==2.15.0 +google-cloud-dlp==3.12.0 +google-cloud-language==2.9.0 +google-cloud-pubsub==2.15.0 +google-cloud-pubsublite==1.7.0 +google-cloud-recommendations-ai==0.10.2 +google-cloud-spanner==3.28.0 +google-cloud-videointelligence==2.11.0 +google-cloud-vision==3.4.0 google-crc32c==1.5.0 -google-pasta==0.2.0 google-resumable-media==2.4.1 googleapis-common-protos==1.58.0 greenlet==2.0.2 grpc-google-iam-v1==0.12.6 -grpcio==1.51.1 -grpcio-status==1.48.2 +grpcio==1.51.3 +grpcio-status==1.51.3 guppy3==3.1.2 -h5py==3.8.0 hdfs==2.7.0 httplib2==0.21.0 -hypothesis==6.68.1 +hypothesis==6.68.2 idna==3.4 importlib-metadata==6.0.0 iniconfig==2.0.0 joblib==1.2.0 -keras==2.11.0 -libclang==15.0.6.1 -Markdown==3.4.1 -MarkupSafe==2.1.2 mmh3==3.0.0 -mock==2.0.0 +mock==5.0.1 nltk==3.8.1 nose==1.3.7 numpy==1.21.6 oauth2client==4.1.3 -oauthlib==3.2.2 objsize==0.6.1 -opt-einsum==3.3.0 -orjson==3.8.6 +orjson==3.8.7 overrides==6.5.0 packaging==23.0 pandas==1.3.5 parameterized==0.8.1 -pbr==5.11.1 pluggy==1.0.0 proto-plus==1.22.2 -protobuf==3.19.6 +protobuf==4.22.1 psycopg2-binary==2.9.5 py==1.11.0 pyarrow==9.0.0 @@ -118,7 +102,7 @@ PyHamcrest==1.10.1 pymongo==4.3.3 PyMySQL==1.0.2 pyparsing==3.0.9 -pytest==7.2.1 +pytest==7.2.2 pytest-forked==1.6.0 pytest-timeout==2.1.0 pytest-xdist==2.5.0 @@ -129,7 +113,6 @@ PyYAML==6.0 regex==2022.10.31 requests==2.28.2 requests-mock==1.10.0 -requests-oauthlib==1.3.1 rsa==4.9 scikit-learn==1.0.2 scipy==1.7.3 @@ -139,22 +122,13 @@ soupsieve==2.4 SQLAlchemy==1.4.46 sqlparse==0.4.3 tenacity==5.1.5 -tensorboard==2.11.2 -tensorboard-data-server==0.6.1 -tensorboard-plugin-wit==1.8.1 -tensorflow==2.11.0 -tensorflow-estimator==2.11.0 -tensorflow-io-gcs-filesystem==0.30.0 -termcolor==2.2.0 testcontainers==3.7.1 threadpoolctl==3.1.0 tomli==2.0.1 -tqdm==4.64.1 -typing_extensions==4.4.0 -uritemplate==4.1.1 +tqdm==4.65.0 +typing_extensions==4.5.0 urllib3==1.26.14 websocket-client==1.5.1 -Werkzeug==2.2.3 -wrapt==1.14.1 -zipp==3.13.0 -zstandard==0.19.0 +wrapt==1.15.0 +zipp==3.15.0 +zstandard==0.20.0 diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt index 4cb9b634e049c..e70cd405ae7c4 100644 --- a/sdks/python/container/py38/base_image_requirements.txt +++ b/sdks/python/container/py38/base_image_requirements.txt @@ -29,11 +29,11 @@ bs4==0.0.1 cachetools==4.2.4 certifi==2022.12.7 cffi==1.15.1 -charset-normalizer==3.0.1 +charset-normalizer==3.1.0 click==8.1.3 cloudpickle==2.2.1 crcmod==1.7 -cryptography==39.0.1 +cryptography==39.0.2 Cython==0.29.33 deprecation==2.1.0 dill==0.3.1.1 @@ -42,71 +42,69 @@ docker==6.0.1 docopt==0.6.2 exceptiongroup==1.1.0 execnet==1.9.0 -fastavro==1.7.1 +fastavro==1.7.3 fasteners==0.18 -flatbuffers==23.1.21 +flatbuffers==23.3.3 freezegun==1.2.2 future==0.18.3 gast==0.4.0 google-api-core==2.11.0 -google-api-python-client==2.78.0 google-apitools==0.5.31 -google-auth==2.16.0 +google-auth==2.16.2 google-auth-httplib2==0.1.0 google-auth-oauthlib==0.4.6 -google-cloud-bigquery==3.5.0 -google-cloud-bigquery-storage==2.16.2 -google-cloud-bigtable==1.7.3 +google-cloud-bigquery==3.6.0 +google-cloud-bigquery-storage==2.19.0 +google-cloud-bigtable==2.17.0 google-cloud-core==2.3.2 -google-cloud-datastore==1.15.5 -google-cloud-dlp==3.11.1 -google-cloud-language==1.3.2 -google-cloud-profiler==3.1.0 -google-cloud-pubsub==2.14.1 -google-cloud-pubsublite==1.6.0 -google-cloud-recommendations-ai==0.7.1 -google-cloud-spanner==3.27.1 -google-cloud-videointelligence==1.16.3 -google-cloud-vision==3.3.1 +google-cloud-datastore==2.15.0 +google-cloud-dlp==3.12.0 +google-cloud-language==2.9.0 +google-cloud-pubsub==2.15.0 +google-cloud-pubsublite==1.7.0 +google-cloud-recommendations-ai==0.10.2 +google-cloud-spanner==3.28.0 +google-cloud-videointelligence==2.11.0 +google-cloud-vision==3.4.0 google-crc32c==1.5.0 google-pasta==0.2.0 google-resumable-media==2.4.1 googleapis-common-protos==1.58.0 greenlet==2.0.2 grpc-google-iam-v1==0.12.6 -grpcio==1.51.1 -grpcio-status==1.48.2 +grpcio==1.51.3 +grpcio-status==1.51.3 guppy3==3.1.2 h5py==3.8.0 hdfs==2.7.0 httplib2==0.21.0 -hypothesis==6.68.1 +hypothesis==6.68.2 idna==3.4 importlib-metadata==6.0.0 iniconfig==2.0.0 +jax==0.4.6 joblib==1.2.0 -keras==2.11.0 +keras==2.12.0rc1 libclang==15.0.6.1 Markdown==3.4.1 MarkupSafe==2.1.2 mmh3==3.0.0 -mock==2.0.0 +mock==5.0.1 nltk==3.8.1 nose==1.3.7 -numpy==1.24.2 +numpy==1.23.5 oauth2client==4.1.3 oauthlib==3.2.2 objsize==0.6.1 opt-einsum==3.3.0 -orjson==3.8.6 +orjson==3.8.7 overrides==6.5.0 packaging==23.0 pandas==1.5.3 parameterized==0.8.1 -pbr==5.11.1 pluggy==1.0.0 proto-plus==1.22.2 -protobuf==3.19.6 +protobuf==4.22.1 psycopg2-binary==2.9.5 py==1.11.0 pyarrow==9.0.0 @@ -118,7 +116,7 @@ PyHamcrest==1.10.1 pymongo==4.3.3 PyMySQL==1.0.2 pyparsing==3.0.9 -pytest==7.2.1 +pytest==7.2.2 pytest-forked==1.6.0 pytest-timeout==2.1.0 pytest-xdist==2.5.0 @@ -131,30 +129,29 @@ requests==2.28.2 requests-mock==1.10.0 requests-oauthlib==1.3.1 rsa==4.9 -scikit-learn==1.2.1 -scipy==1.10.0 +scikit-learn==1.2.2 +scipy==1.10.1 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.4 SQLAlchemy==1.4.46 sqlparse==0.4.3 tenacity==5.1.5 -tensorboard==2.11.2 -tensorboard-data-server==0.6.1 +tensorboard==2.12.0 +tensorboard-data-server==0.7.0 tensorboard-plugin-wit==1.8.1 -tensorflow==2.11.0 -tensorflow-estimator==2.11.0 -tensorflow-io-gcs-filesystem==0.30.0 +tensorflow==2.12.0rc1 +tensorflow-estimator==2.12.0rc0 +tensorflow-io-gcs-filesystem==0.31.0 termcolor==2.2.0 testcontainers==3.7.1 threadpoolctl==3.1.0 tomli==2.0.1 -tqdm==4.64.1 -typing_extensions==4.4.0 -uritemplate==4.1.1 +tqdm==4.65.0 +typing_extensions==4.5.0 urllib3==1.26.14 websocket-client==1.5.1 Werkzeug==2.2.3 wrapt==1.14.1 -zipp==3.13.0 -zstandard==0.19.0 +zipp==3.15.0 +zstandard==0.20.0 diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt index 1b4f9a115331c..ce8b8067b57ca 100644 --- a/sdks/python/container/py39/base_image_requirements.txt +++ b/sdks/python/container/py39/base_image_requirements.txt @@ -29,11 +29,11 @@ bs4==0.0.1 cachetools==4.2.4 certifi==2022.12.7 cffi==1.15.1 -charset-normalizer==3.0.1 +charset-normalizer==3.1.0 click==8.1.3 cloudpickle==2.2.1 crcmod==1.7 -cryptography==39.0.1 +cryptography==39.0.2 Cython==0.29.33 deprecation==2.1.0 dill==0.3.1.1 @@ -42,71 +42,69 @@ docker==6.0.1 docopt==0.6.2 exceptiongroup==1.1.0 execnet==1.9.0 -fastavro==1.7.1 +fastavro==1.7.3 fasteners==0.18 -flatbuffers==23.1.21 +flatbuffers==23.3.3 freezegun==1.2.2 future==0.18.3 gast==0.4.0 google-api-core==2.11.0 -google-api-python-client==2.78.0 google-apitools==0.5.31 -google-auth==2.16.0 +google-auth==2.16.2 google-auth-httplib2==0.1.0 google-auth-oauthlib==0.4.6 -google-cloud-bigquery==3.5.0 -google-cloud-bigquery-storage==2.16.2 -google-cloud-bigtable==1.7.3 +google-cloud-bigquery==3.6.0 +google-cloud-bigquery-storage==2.19.0 +google-cloud-bigtable==2.17.0 google-cloud-core==2.3.2 -google-cloud-datastore==1.15.5 -google-cloud-dlp==3.11.1 -google-cloud-language==1.3.2 -google-cloud-profiler==3.1.0 -google-cloud-pubsub==2.14.1 -google-cloud-pubsublite==1.6.0 -google-cloud-recommendations-ai==0.7.1 -google-cloud-spanner==3.27.1 -google-cloud-videointelligence==1.16.3 -google-cloud-vision==3.3.1 +google-cloud-datastore==2.15.0 +google-cloud-dlp==3.12.0 +google-cloud-language==2.9.0 +google-cloud-pubsub==2.15.0 +google-cloud-pubsublite==1.7.0 +google-cloud-recommendations-ai==0.10.2 +google-cloud-spanner==3.28.0 +google-cloud-videointelligence==2.11.0 +google-cloud-vision==3.4.0 google-crc32c==1.5.0 google-pasta==0.2.0 google-resumable-media==2.4.1 googleapis-common-protos==1.58.0 greenlet==2.0.2 grpc-google-iam-v1==0.12.6 -grpcio==1.51.1 -grpcio-status==1.48.2 +grpcio==1.51.3 +grpcio-status==1.51.3 guppy3==3.1.2 h5py==3.8.0 hdfs==2.7.0 httplib2==0.21.0 -hypothesis==6.68.1 +hypothesis==6.68.2 idna==3.4 importlib-metadata==6.0.0 iniconfig==2.0.0 +jax==0.4.6 joblib==1.2.0 -keras==2.11.0 +keras==2.12.0rc1 libclang==15.0.6.1 Markdown==3.4.1 MarkupSafe==2.1.2 mmh3==3.0.0 -mock==2.0.0 +mock==5.0.1 nltk==3.8.1 nose==1.3.7 -numpy==1.24.2 +numpy==1.23.5 oauth2client==4.1.3 oauthlib==3.2.2 objsize==0.6.1 opt-einsum==3.3.0 -orjson==3.8.6 +orjson==3.8.7 overrides==6.5.0 packaging==23.0 pandas==1.5.3 parameterized==0.8.1 -pbr==5.11.1 pluggy==1.0.0 proto-plus==1.22.2 -protobuf==3.19.6 +protobuf==4.22.1 psycopg2-binary==2.9.5 py==1.11.0 pyarrow==9.0.0 @@ -118,7 +116,7 @@ PyHamcrest==1.10.1 pymongo==4.3.3 PyMySQL==1.0.2 pyparsing==3.0.9 -pytest==7.2.1 +pytest==7.2.2 pytest-forked==1.6.0 pytest-timeout==2.1.0 pytest-xdist==2.5.0 @@ -131,30 +129,29 @@ requests==2.28.2 requests-mock==1.10.0 requests-oauthlib==1.3.1 rsa==4.9 -scikit-learn==1.2.1 -scipy==1.10.0 +scikit-learn==1.2.2 +scipy==1.10.1 six==1.16.0 sortedcontainers==2.4.0 soupsieve==2.4 SQLAlchemy==1.4.46 sqlparse==0.4.3 tenacity==5.1.5 -tensorboard==2.11.2 -tensorboard-data-server==0.6.1 +tensorboard==2.12.0 +tensorboard-data-server==0.7.0 tensorboard-plugin-wit==1.8.1 -tensorflow==2.11.0 -tensorflow-estimator==2.11.0 -tensorflow-io-gcs-filesystem==0.30.0 +tensorflow==2.12.0rc1 +tensorflow-estimator==2.12.0rc0 +tensorflow-io-gcs-filesystem==0.31.0 termcolor==2.2.0 testcontainers==3.7.1 threadpoolctl==3.1.0 tomli==2.0.1 -tqdm==4.64.1 -typing_extensions==4.4.0 -uritemplate==4.1.1 +tqdm==4.65.0 +typing_extensions==4.5.0 urllib3==1.26.14 websocket-client==1.5.1 Werkzeug==2.2.3 wrapt==1.14.1 -zipp==3.13.0 -zstandard==0.19.0 +zipp==3.15.0 +zstandard==0.20.0 diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py index 86d94d937e832..c90c76d178ed2 100644 --- a/sdks/python/gen_protos.py +++ b/sdks/python/gen_protos.py @@ -123,7 +123,7 @@ def generate_urn_files(out_dir, api_path): This is executed at build time rather than dynamically on import to ensure that it is compatible with static type checkers like mypy. """ - import google.protobuf.pyext._message as pyext_message + from google._upb import _message from google.protobuf import message class Context(object): @@ -179,8 +179,9 @@ def python_repr(self, obj): obj, ( list, - pyext_message.RepeatedCompositeContainer, # pylint: disable=c-extension-no-member - pyext_message.RepeatedScalarContainer)): # pylint: disable=c-extension-no-member + _message.RepeatedScalarContainer, + _message.RepeatedCompositeContainer, + )): # pylint: disable=c-extension-no-member return '[%s]' % ', '.join(self.python_repr(x) for x in obj) else: return repr(obj) @@ -203,11 +204,10 @@ def message_repr(self, msg): def write_enum(self, enum_name, enum, indent): ctx = Context(indent=indent) - with ctx.indent(): - for v in enum.DESCRIPTOR.values: - extensions = v.GetOptions().Extensions - + for enum_value_name in enum.values_by_name: + enum_value_descriptor = enum.values_by_name[enum_value_name] + extensions = enum_value_descriptor.GetOptions().Extensions prop = ( extensions[beam_runner_api_pb2.beam_urn], extensions[beam_runner_api_pb2.beam_constant], @@ -219,7 +219,7 @@ def write_enum(self, enum_name, enum, indent): continue ctx.line( '%s = PropertiesFromEnumValue(%s)' % - (v.name, ', '.join(self.python_repr(x) for x in prop))) + (enum_value_name, ', '.join(self.python_repr(x) for x in prop))) if ctx.lines: ctx.prepend('class %s(object):' % enum_name) @@ -232,10 +232,10 @@ def write_message(self, message_name, message, indent=0): with ctx.indent(): for obj_name, obj in inspect.getmembers(message): - if self.is_message_type(obj): - ctx.lines += self.write_message(obj_name, obj, ctx._indent) - elif self.is_enum_type(obj): - ctx.lines += self.write_enum(obj_name, obj, ctx._indent) + if obj_name == 'DESCRIPTOR': + for enum_name in obj.enum_types_by_name: + enum = obj.enum_types_by_name[enum_name] + ctx.lines += self.write_enum(enum_name, enum, ctx._indent) if ctx.lines: ctx.prepend('class %s(object):' % message_name) @@ -512,7 +512,12 @@ def generate_proto_files(force=False): ['--proto_path=%s' % d for d in proto_dirs] + ['--python_out=%s' % PYTHON_OUTPUT_PATH] + ['--plugin=protoc-gen-mypy=%s' % protoc_gen_mypy] + - ['--mypy_out=%s' % PYTHON_OUTPUT_PATH] + + # new version of mypy-protobuf converts None to zero default value + # and remove Optional from the param type annotation. This causes + # some mypy errors. So to mitigate and fall back to old behavior, + # use `relax_strict_optional_primitives` flag. more at + # https://github.com/nipunn1313/mypy-protobuf/tree/main#relax_strict_optional_primitives # pylint:disable=line-too-long + ['--mypy_out=relax_strict_optional_primitives:%s' % PYTHON_OUTPUT_PATH] + # TODO(robertwb): Remove the prefix once it's the default. ['--grpc_python_out=grpc_2_0:%s' % PYTHON_OUTPUT_PATH] + proto_files) @@ -536,7 +541,7 @@ def generate_proto_files(force=False): ('_pb2.py', '_pb2_grpc.py', '_pb2.pyi')): proto_packages.add(os.path.dirname(file_path)) lines = [] - with open(file_path) as f: + with open(file_path, encoding='utf-8') as f: for line in f: match_obj = compiled_import_re.match(line) if match_obj and \ diff --git a/sdks/python/setup.py b/sdks/python/setup.py index f0d49aa3603d1..45a247bdf476a 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -180,16 +180,6 @@ def get_portability_package_data(): generate_protos_first() # Keep all dependencies inlined in the setup call, otherwise Dependabot won't # be able to parse it. - if sys.platform == 'darwin' and ( - sys.version_info.major == 3 and sys.version_info.minor == 10): - # TODO (https://github.com/apache/beam/issues/23585): Protobuf wheels - # for version 3.19.5, 3.19.6 and 3.20.x on Python 3.10 and MacOS are - # rolled back due to some errors on MacOS. So, for Python 3.10 on MacOS - # restrict the protobuf with tight upper bound(3.19.4) - protobuf_dependency = ['protobuf>3.12.2,<3.19.5'] - else: - protobuf_dependency = ['protobuf>3.12.2,<4'] - setuptools.setup( name=PACKAGE_NAME, version=PACKAGE_VERSION, @@ -227,7 +217,7 @@ def get_portability_package_data(): 'apache_beam/utils/counters.py', 'apache_beam/utils/windowed_value.py', ]), - install_requires= protobuf_dependency + [ + install_requires = [ 'crcmod>=1.7,<2.0', 'orjson<4.0', # Dill doesn't have forwards-compatibility guarantees within minor @@ -252,6 +242,12 @@ def get_portability_package_data(): 'objsize>=0.6.1,<0.7.0', 'pymongo>=3.8.0,<5.0.0', 'proto-plus>=1.7.1,<2', + # use a tighter upper bound in protobuf dependency + # to make sure the minor version at job submission + # does not exceed the minor version at runtime. + # To avoid depending on an old dependency, update the minor version on + # every Beam release, see: https://github.com/apache/beam/issues/25590 + 'protobuf>=4.21.1,<4.23.0', 'pydot>=1.2.0,<2', 'python-dateutil>=2.8.0,<3', 'pytz>=2018.3', @@ -301,25 +297,25 @@ def get_portability_package_data(): # https://github.com/googleapis/google-cloud-python/issues/10566 'google-auth>=1.18.0,<3', 'google-auth-httplib2>=0.1.0,<0.2.0', - 'google-cloud-datastore>=1.8.0,<2', + 'google-cloud-datastore>=2.0.0,<3', 'google-cloud-pubsub>=2.1.0,<3', 'google-cloud-pubsublite>=1.2.0,<2', # GCP packages required by tests - 'google-cloud-bigquery>=1.6.0,<4', - 'google-cloud-bigquery-storage>=2.6.3,<2.17', - 'google-cloud-core>=0.28.1,<3', - 'google-cloud-bigtable>=0.31.1,<2', + 'google-cloud-bigquery>=2.0.0,<4', + 'google-cloud-bigquery-storage>=2.6.3,<3', + 'google-cloud-core>=2.0.0,<3', + 'google-cloud-bigtable>=2.0.0,<3', 'google-cloud-spanner>=3.0.0,<4', # GCP Packages required by ML functionality 'google-cloud-dlp>=3.0.0,<4', - 'google-cloud-language>=1.3.0,<2', - 'google-cloud-videointelligence>=1.8.0,<2', + 'google-cloud-language>=2.0,<3', + 'google-cloud-videointelligence>=2.0,<3', 'google-cloud-vision>=2,<4', 'google-cloud-recommendations-ai>=0.1.0,<0.11.0' ], 'interactive': [ - 'facets-overview>=1.0.0,<2', - 'google-cloud-dataproc>=3.0.0,<3.2.0', + 'facets-overview>=1.1.0,<2', + 'google-cloud-dataproc>=5.0.0,<6', # IPython>=8 is not compatible with Python<=3.7 'ipython>=7,<8;python_version<="3.7"', 'ipython>=8,<9;python_version>"3.7"', diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle index 5fcca066588fb..1351d5b1fc5b5 100644 --- a/sdks/python/test-suites/direct/common.gradle +++ b/sdks/python/test-suites/direct/common.gradle @@ -341,8 +341,10 @@ project.tasks.register("inferencePostCommitIT") { dependsOn = [ 'torchInferenceTest', 'sklearnInferenceTest', - 'tfxInferenceTest', 'tensorflowInferenceTest', 'xgboostInferenceTest' + // (TODO) https://github.com/apache/beam/issues/25799 + // uncomment tfx bsl tests once tfx supports protobuf 4.x + // 'tfxInferenceTest', ] } diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle index 7d582bd89c1a2..16b9297e7ca62 100644 --- a/sdks/python/test-suites/tox/py38/build.gradle +++ b/sdks/python/test-suites/tox/py38/build.gradle @@ -102,22 +102,15 @@ toxTask "testPy38pytorch-113", "py38-pytorch-113", "${posargs}" test.dependsOn "testPy38pytorch-113" preCommitPyCoverage.dependsOn "testPy38pytorch-113" +// TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task once onnx supports protobuf 4.x.x // Create a test task for each minor version of onnx -toxTask "testPy38onnx-113", "py38-onnx-113", "${posargs}" -test.dependsOn "testPy38onnx-113" -preCommitPyCoverage.dependsOn "testPy38onnx-113" +// toxTask "testPy38onnx-113", "py38-onnx-113", "${posargs}" +// test.dependsOn "testPy38onnx-113" +// preCommitPyCoverage.dependsOn "testPy38onnx-113" // Create a test task for each minor version of tensorflow -toxTask "testPy38tensorflow-29", "py38-tensorflow-29", "${posargs}" -test.dependsOn "testPy38tensorflow-29" -preCommitPyCoverage.dependsOn "testPy38tensorflow-29" - -toxTask "testPy38tensorflow-210", "py38-tensorflow-210", "${posargs}" -test.dependsOn "testPy38tensorflow-210" -preCommitPyCoverage.dependsOn "testPy38tensorflow-210" - -toxTask "testPy38tensorflow-211", "py38-tensorflow-211", "${posargs}" -test.dependsOn "testPy38tensorflow-211" -preCommitPyCoverage.dependsOn "testPy38tensorflow-211" +toxTask "testPy38tensorflow-212", "py38-tensorflow-212", "${posargs}" +test.dependsOn "testPy38tensorflow-212" +preCommitPyCoverage.dependsOn "testPy38tensorflow-212" toxTask "whitespacelint", "whitespacelint", "${posargs}" diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 58c2c0c854279..526eb4cf37d3a 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -132,7 +132,7 @@ commands = [testenv:py37-mypy] deps = -r build-requirements.txt - mypy==0.782 + mypy==0.790 dask==2022.01.0 distributed==2022.01.0 # make extras available in case any of these libs are typed @@ -323,6 +323,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' +# TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py38/build.gradle once onnx supports protobuf 4.x.x [testenv:py{37,38,39,310}-onnx-113] # TODO(https://github.com/apache/beam/issues/25443) # apparently tox has problem when substitution key has single value. Change back to -onnx-{113,...} @@ -342,12 +343,10 @@ commands = # Run all ONNX unit tests pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_onnx {posargs} -[testenv:py{37,38,39,310}-tensorflow-{29,210,211}] +[testenv:py{38,39,310}-tensorflow-212] deps = -r build-requirements.txt - 29: tensorflow>=2.9.0,<2.10.0 - 210: tensorflow>=2.10.0,<2.11.0 - 211: tensorflow>=2.11.0,<2.12.0 + 212: tensorflow>=2.12rc1,<2.13 extras = test,gcp commands = # Log tensorflow version for debugging