Add provinance to the registry entry (#37804)

## What
Adds the generation time and root metadata file info to the registry entry

![image.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/PTsI7qAmiIMkhFQg04QF/4aa5ca8b-011d-4e6f-90a2-081feae7756c.png)

Spun out of #32715 as a stack

closes airbytehq/airbyte-internal-issues#7466

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py

@@ -108,22 +108,18 @@ class GitInfo(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    commit_sha: Optional[str] = Field(
-        None,
-        description="The git commit sha of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
-    commit_timestamp: Optional[datetime] = Field(
-        None,
-        description="The git commit timestamp of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
-    commit_author: Optional[str] = Field(
-        None,
-        description="The git commit author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
-    commit_author_email: Optional[str] = Field(
-        None,
-        description="The git commit author email of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
+    commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
+    commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
+    commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
+    commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
+
+
+class SourceFileInfo(BaseModel):
+    metadata_etag: Optional[str] = None
+    metadata_file_path: Optional[str] = None
+    metadata_bucket_name: Optional[str] = None
+    metadata_last_modified: Optional[str] = None
+    registry_entry_generated_at: Optional[str] = None
 
 
 class JobTypeResourceLimit(BaseModel):
@@ -147,6 +143,7 @@ class Config:
 
 class GeneratedFields(BaseModel):
     git: Optional[GitInfo] = None
+    source_file_info: Optional[SourceFileInfo] = None
 
 
 class ActorDefinitionResourceRequirements(BaseModel):

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py

@@ -3,7 +3,7 @@
 
 from __future__ import annotations
 
-from datetime import date
+from datetime import date, datetime
 from typing import Any, Dict, List, Optional
 from uuid import UUID
 
@@ -79,6 +79,24 @@ class Config:
     ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
 
 
+class GitInfo(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
+    commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
+    commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
+    commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
+
+
+class SourceFileInfo(BaseModel):
+    metadata_etag: Optional[str] = None
+    metadata_file_path: Optional[str] = None
+    metadata_bucket_name: Optional[str] = None
+    metadata_last_modified: Optional[str] = None
+    registry_entry_generated_at: Optional[str] = None
+
+
 class JobTypeResourceLimit(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -91,6 +109,11 @@ class BreakingChangeScope(BaseModel):
     __root__: StreamBreakingChangeScope = Field(..., description="A scope that can be used to limit the impact of a breaking change.")
 
 
+class GeneratedFields(BaseModel):
+    git: Optional[GitInfo] = None
+    source_file_info: Optional[SourceFileInfo] = None
+
+
 class ActorDefinitionResourceRequirements(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -172,3 +195,4 @@ class Config:
     releases: Optional[ConnectorReleases] = None
     ab_internal: Optional[AirbyteInternal] = None
     supportsRefreshes: Optional[bool] = False
+    generated: Optional[GeneratedFields] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py

@@ -3,7 +3,7 @@
 
 from __future__ import annotations
 
-from datetime import date
+from datetime import date, datetime
 from typing import Any, Dict, List, Optional
 from uuid import UUID
 
@@ -75,6 +75,24 @@ class Config:
     ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
 
 
+class GitInfo(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
+    commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
+    commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
+    commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
+
+
+class SourceFileInfo(BaseModel):
+    metadata_etag: Optional[str] = None
+    metadata_file_path: Optional[str] = None
+    metadata_bucket_name: Optional[str] = None
+    metadata_last_modified: Optional[str] = None
+    registry_entry_generated_at: Optional[str] = None
+
+
 class JobTypeResourceLimit(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -87,6 +105,11 @@ class BreakingChangeScope(BaseModel):
     __root__: StreamBreakingChangeScope = Field(..., description="A scope that can be used to limit the impact of a breaking change.")
 
 
+class GeneratedFields(BaseModel):
+    git: Optional[GitInfo] = None
+    source_file_info: Optional[SourceFileInfo] = None
+
+
 class ActorDefinitionResourceRequirements(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -164,3 +187,4 @@ class Config:
     )
     releases: Optional[ConnectorReleases] = None
     ab_internal: Optional[AirbyteInternal] = None
+    generated: Optional[GeneratedFields] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py

@@ -3,7 +3,7 @@
 
 from __future__ import annotations
 
-from datetime import date
+from datetime import date, datetime
 from typing import Any, Dict, List, Optional
 from uuid import UUID
 
@@ -79,6 +79,24 @@ class Config:
     ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
 
 
+class GitInfo(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
+    commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
+    commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
+    commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
+
+
+class SourceFileInfo(BaseModel):
+    metadata_etag: Optional[str] = None
+    metadata_file_path: Optional[str] = None
+    metadata_bucket_name: Optional[str] = None
+    metadata_last_modified: Optional[str] = None
+    registry_entry_generated_at: Optional[str] = None
+
+
 class SuggestedStreams(BaseModel):
     class Config:
         extra = Extra.allow
@@ -101,6 +119,11 @@ class BreakingChangeScope(BaseModel):
     __root__: StreamBreakingChangeScope = Field(..., description="A scope that can be used to limit the impact of a breaking change.")
 
 
+class GeneratedFields(BaseModel):
+    git: Optional[GitInfo] = None
+    source_file_info: Optional[SourceFileInfo] = None
+
+
 class ActorDefinitionResourceRequirements(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -178,6 +201,7 @@ class Config:
     )
     releases: Optional[ConnectorReleases] = None
     ab_internal: Optional[AirbyteInternal] = None
+    generated: Optional[GeneratedFields] = None
 
 
 class ConnectorRegistryDestinationDefinition(BaseModel):
@@ -214,6 +238,7 @@ class Config:
     releases: Optional[ConnectorReleases] = None
     ab_internal: Optional[AirbyteInternal] = None
     supportsRefreshes: Optional[bool] = False
+    generated: Optional[GeneratedFields] = None
 
 
 class ConnectorRegistryV0(BaseModel):

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/GeneratedFields.py

@@ -13,23 +13,20 @@ class GitInfo(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    commit_sha: Optional[str] = Field(
-        None,
-        description="The git commit sha of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
-    commit_timestamp: Optional[datetime] = Field(
-        None,
-        description="The git commit timestamp of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
-    commit_author: Optional[str] = Field(
-        None,
-        description="The git commit author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
-    commit_author_email: Optional[str] = Field(
-        None,
-        description="The git commit author email of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
+    commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
+    commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
+    commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
+    commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")
+
+
+class SourceFileInfo(BaseModel):
+    metadata_etag: Optional[str] = None
+    metadata_file_path: Optional[str] = None
+    metadata_bucket_name: Optional[str] = None
+    metadata_last_modified: Optional[str] = None
+    registry_entry_generated_at: Optional[str] = None
 
 
 class GeneratedFields(BaseModel):
     git: Optional[GitInfo] = None
+    source_file_info: Optional[SourceFileInfo] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/GitInfo.py

@@ -13,19 +13,7 @@ class GitInfo(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    commit_sha: Optional[str] = Field(
-        None,
-        description="The git commit sha of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
-    commit_timestamp: Optional[datetime] = Field(
-        None,
-        description="The git commit timestamp of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
-    commit_author: Optional[str] = Field(
-        None,
-        description="The git commit author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
-    commit_author_email: Optional[str] = Field(
-        None,
-        description="The git commit author email of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
-    )
+    commit_sha: Optional[str] = Field(None, description="The git commit sha of the last commit that modified this file.")
+    commit_timestamp: Optional[datetime] = Field(None, description="The git commit timestamp of the last commit that modified this file.")
+    commit_author: Optional[str] = Field(None, description="The git commit author of the last commit that modified this file.")
+    commit_author_email: Optional[str] = Field(None, description="The git commit author email of the last commit that modified this file.")

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SourceFileInfo.py

@@ -0,0 +1,16 @@
+# generated by datamodel-codegen:
+#   filename:  SourceFileInfo.yaml
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class SourceFileInfo(BaseModel):
+    metadata_etag: Optional[str] = None
+    metadata_file_path: Optional[str] = None
+    metadata_bucket_name: Optional[str] = None
+    metadata_last_modified: Optional[str] = None
+    registry_entry_generated_at: Optional[str] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py

@@ -16,5 +16,6 @@
 from .ReleaseStage import *
 from .RemoteRegistries import *
 from .ResourceRequirements import *
+from .SourceFileInfo import *
 from .SuggestedStreams import *
 from .SupportLevel import *

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml

@@ -75,3 +75,5 @@ properties:
   supportsRefreshes:
     type: boolean
     default: false
+  generated:
+    "$ref": GeneratedFields.yaml

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml

@@ -74,3 +74,5 @@ properties:
     "$ref": ConnectorReleases.yaml
   ab_internal:
     "$ref": AirbyteInternal.yaml
+  generated:
+    "$ref": GeneratedFields.yaml

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GeneratedFields.yaml

@@ -7,3 +7,5 @@ type: object
 properties:
   git:
     "$ref": GitInfo.yaml
+  source_file_info:
+    "$ref": SourceFileInfo.yaml

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GitInfo.yaml

@@ -2,20 +2,20 @@
 "$schema": http://json-schema.org/draft-07/schema#
 "$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GitInfo.yaml
 title: GitInfo
-description: Information about the author of the last commit that modified this file
+description: Information about the author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
 type: object
 additionalProperties: false
 properties:
   commit_sha:
     type: string
-    description: The git commit sha of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
+    description: The git commit sha of the last commit that modified this file.
   commit_timestamp:
     type: string
     format: date-time
-    description: The git commit timestamp of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
+    description: The git commit timestamp of the last commit that modified this file.
   commit_author:
     type: string
-    description: The git commit author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
+    description: The git commit author of the last commit that modified this file.
   commit_author_email:
     type: string
-    description: The git commit author email of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.
+    description: The git commit author email of the last commit that modified this file.

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SourceFileInfo.yaml

@@ -0,0 +1,17 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SourceFileInfo.yaml
+title: SourceFileInfo
+description: Information about the source file that generated the registry entry
+type: object
+properties:
+  metadata_etag:
+    type: string
+  metadata_file_path:
+    type: string
+  metadata_bucket_name:
+    type: string
+  metadata_last_modified:
+    type: string
+  registry_entry_generated_at:
+    type: string

airbyte-ci/connectors/metadata_service/lib/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metadata-service"
-version = "0.4.0"
+version = "0.5.0"
 description = ""
 authors = ["Ben Church <ben@airbyte.io>"]
 readme = "README.md"

airbyte-ci/connectors/metadata_service/orchestrator/.env.template

@@ -5,8 +5,10 @@ NIGHTLY_REPORT_CHANNEL=""
 # METADATA_CDN_BASE_URL="https://connectors.airbyte.com/files"
 DOCKER_HUB_USERNAME=""
 DOCKER_HUB_PASSWORD=""
+
 SLACK_TOKEN = ""
-PUBLISH_UPDATE_CHANNEL="#ben-test"
+SLACK_NOTIFICATIONS_DISABLED = "true"
+PUBLISH_UPDATE_CHANNEL="#test-ci-slack-intergrations"
 # SENTRY_DSN=""
 # SENTRY_ENVIRONMENT="dev"
 # SENTRY_TRACES_SAMPLE_RATE=1.0

airbyte-ci/connectors/metadata_service/orchestrator/README.md

@@ -169,3 +169,12 @@ open http://localhost:3000
 ```
 
 And run the `generate_registry` job
+
+## Additional Notes
+
+### Testing Slack Notifications
+You will need to add the following environment variables to your `.env` file:
+
+- `SLACK_TOKEN`: Set to an OAuth token for the [Connector Ops Dagster Bot](https://airbytehq-team.slack.com/apps/A05K845HBE0-connector-ops-dagster-bot?settings=1)
+- `PUBLISH_UPDATE_CHANNEL`: Set to `#test-ci-slack-intergrations`
+- `SLACK_NOTIFICATIONS_DISABLED`: Set to `False`

airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py

@@ -130,6 +130,10 @@ def metadata_definitions(context: OpExecutionContext) -> List[LatestMetadataEntr
         metadata_entry = LatestMetadataEntry(
             metadata_definition=metadata_def,
             icon_url=icon_url,
+            last_modified=blob.last_modified,
+            etag=blob.etag,
+            file_path=blob.name,
+            bucket_name=blob.bucket.name,
         )
         metadata_entries.append(metadata_entry)