From 1c0f8ddbb0bea1840a26663d449cb10d40676154 Mon Sep 17 00:00:00 2001 From: tuancamtbtx Date: Tue, 2 Jul 2024 11:39:59 +0700 Subject: [PATCH] [feat] generate dags from yaml file --- airlake/console/git_sync.py | 6 +++++ airlake/factory/compiler.py | 2 ++ airlake/factory/enity.py | 44 +++++++++++++++++++++++++++++++++ airlake/factory/model.py | 0 airlake/manager/backfill_web.py | 2 ++ 5 files changed, 54 insertions(+) create mode 100644 airlake/console/git_sync.py create mode 100644 airlake/factory/enity.py create mode 100644 airlake/factory/model.py create mode 100644 airlake/manager/backfill_web.py diff --git a/airlake/console/git_sync.py b/airlake/console/git_sync.py new file mode 100644 index 0000000..2679218 --- /dev/null +++ b/airlake/console/git_sync.py @@ -0,0 +1,6 @@ + +def main(): + pass + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/airlake/factory/compiler.py b/airlake/factory/compiler.py index e69de29..940fecc 100644 --- a/airlake/factory/compiler.py +++ b/airlake/factory/compiler.py @@ -0,0 +1,2 @@ +class DagCompiler: + pass \ No newline at end of file diff --git a/airlake/factory/enity.py b/airlake/factory/enity.py new file mode 100644 index 0000000..f250d7d --- /dev/null +++ b/airlake/factory/enity.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass, field +from typing import Dict, Any, Optional, List +import dacite + +class RepoType: + Python = "python" + Yaml = "yaml" + +@dataclass +class TeamConnection: + conn_id: str + replace_fields: List[str] + @staticmethod + def from_list(items: List[Dict[str, Any]]): + return [dacite.from_dict(data_class=TeamConnection, data=i) for i in items] + +@dataclass +class Alerting: + kind: str + conn_id: str + + @staticmethod + def from_dict(conf): + return dacite.from_dict(data_class=Alerting, data=conf) + + +@dataclass +class TeamConfig: + name: str + prefix: str + owner: str + repo_id: Optional[str] + role_id: Optional[int] + alert: Optional[Alerting] + conns: Optional[List[TeamConnection]] + team_dir: Optional[str] + pool: Optional[str] + type: Optional[str] = "yaml" + + def is_yaml(self): + return not self.type or self.type == RepoType.Yaml + + def is_python(self): + return self.type == RepoType.Python diff --git a/airlake/factory/model.py b/airlake/factory/model.py new file mode 100644 index 0000000..e69de29 diff --git a/airlake/manager/backfill_web.py b/airlake/manager/backfill_web.py new file mode 100644 index 0000000..80f5460 --- /dev/null +++ b/airlake/manager/backfill_web.py @@ -0,0 +1,2 @@ +class Backfill: + pass \ No newline at end of file