From e718698b096c8b9006f9caf1bc7ab825240012da Mon Sep 17 00:00:00 2001 From: Pierre Jeambrun Date: Mon, 21 Oct 2024 17:22:14 +0800 Subject: [PATCH] AIP-84 Get Providers (#43159) --- .../endpoints/provider_endpoint.py | 2 + .../core_api/openapi/v1-generated.yaml | 69 +++++++++++++++++ .../core_api/routes/public/__init__.py | 2 + .../core_api/routes/public/providers.py | 55 ++++++++++++++ .../core_api/serializers/providers.py | 35 +++++++++ airflow/ui/openapi-gen/queries/common.ts | 19 +++++ airflow/ui/openapi-gen/queries/prefetch.ts | 24 ++++++ airflow/ui/openapi-gen/queries/queries.ts | 33 ++++++++ airflow/ui/openapi-gen/queries/suspense.ts | 33 ++++++++ .../ui/openapi-gen/requests/schemas.gen.ts | 41 ++++++++++ .../ui/openapi-gen/requests/services.gen.ts | 29 +++++++ airflow/ui/openapi-gen/requests/types.gen.ts | 39 ++++++++++ .../core_api/routes/public/test_providers.py | 75 +++++++++++++++++++ 13 files changed, 456 insertions(+) create mode 100644 airflow/api_fastapi/core_api/routes/public/providers.py create mode 100644 airflow/api_fastapi/core_api/serializers/providers.py create mode 100644 tests/api_fastapi/core_api/routes/public/test_providers.py diff --git a/airflow/api_connexion/endpoints/provider_endpoint.py b/airflow/api_connexion/endpoints/provider_endpoint.py index d9ba0c819b702..1eb032dee030e 100644 --- a/airflow/api_connexion/endpoints/provider_endpoint.py +++ b/airflow/api_connexion/endpoints/provider_endpoint.py @@ -28,6 +28,7 @@ ) from airflow.auth.managers.models.resource_details import AccessView from airflow.providers_manager import ProvidersManager +from airflow.utils.api_migration import mark_fastapi_migration_done if TYPE_CHECKING: from airflow.api_connexion.types import APIResponse @@ -46,6 +47,7 @@ def _provider_mapper(provider: ProviderInfo) -> Provider: ) +@mark_fastapi_migration_done @security.requires_access_view(AccessView.PROVIDERS) def get_providers() -> APIResponse: """Get providers.""" diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index 97aac85644a8c..88dc7428bed6b 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -1057,6 +1057,41 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /public/providers/: + get: + tags: + - Provider + summary: Get Providers + description: Get providers. + operationId: get_providers + parameters: + - name: limit + in: query + required: false + schema: + type: integer + default: 100 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + default: 0 + title: Offset + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/ProviderCollectionResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' components: schemas: BaseInfoSchema: @@ -1803,6 +1838,40 @@ components: - task_instance_states title: HistoricalMetricDataResponse description: Historical Metric Data serializer for responses. + ProviderCollectionResponse: + properties: + providers: + items: + $ref: '#/components/schemas/ProviderResponse' + type: array + title: Providers + total_entries: + type: integer + title: Total Entries + type: object + required: + - providers + - total_entries + title: ProviderCollectionResponse + description: Provider Collection serializer for responses. + ProviderResponse: + properties: + package_name: + type: string + title: Package Name + description: + type: string + title: Description + version: + type: string + title: Version + type: object + required: + - package_name + - description + - version + title: ProviderResponse + description: Provider serializer for responses. SchedulerInfoSchema: properties: status: diff --git a/airflow/api_fastapi/core_api/routes/public/__init__.py b/airflow/api_fastapi/core_api/routes/public/__init__.py index 12d98e44199da..0778d66cc4137 100644 --- a/airflow/api_fastapi/core_api/routes/public/__init__.py +++ b/airflow/api_fastapi/core_api/routes/public/__init__.py @@ -23,6 +23,7 @@ from airflow.api_fastapi.core_api.routes.public.dags import dags_router from airflow.api_fastapi.core_api.routes.public.monitor import monitor_router from airflow.api_fastapi.core_api.routes.public.pools import pools_router +from airflow.api_fastapi.core_api.routes.public.providers import providers_router from airflow.api_fastapi.core_api.routes.public.variables import variables_router public_router = AirflowRouter(prefix="/public") @@ -34,3 +35,4 @@ public_router.include_router(dag_run_router) public_router.include_router(monitor_router) public_router.include_router(pools_router) +public_router.include_router(providers_router) diff --git a/airflow/api_fastapi/core_api/routes/public/providers.py b/airflow/api_fastapi/core_api/routes/public/providers.py new file mode 100644 index 0000000000000..6c01578dd5f69 --- /dev/null +++ b/airflow/api_fastapi/core_api/routes/public/providers.py @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import re2 + +from airflow.api_fastapi.common.parameters import QueryLimit, QueryOffset +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.serializers.providers import ProviderCollectionResponse, ProviderResponse +from airflow.providers_manager import ProviderInfo, ProvidersManager + +providers_router = AirflowRouter(tags=["Provider"], prefix="/providers") + + +def _remove_rst_syntax(value: str) -> str: + return re2.sub("[`_<>]", "", value.strip(" \n.")) + + +def _provider_mapper(provider: ProviderInfo) -> ProviderResponse: + return ProviderResponse( + package_name=provider.data["package-name"], + description=_remove_rst_syntax(provider.data["description"]), + version=provider.version, + ) + + +@providers_router.get("/") +async def get_providers( + limit: QueryLimit, + offset: QueryOffset, +) -> ProviderCollectionResponse: + """Get providers.""" + providers = sorted( + [_provider_mapper(d) for d in ProvidersManager().providers.values()], key=lambda x: x.package_name + ) + total_entries = len(providers) + + if limit.value is not None and offset.value is not None: + providers = providers[offset.value : offset.value + limit.value] + return ProviderCollectionResponse(providers=providers, total_entries=total_entries) diff --git a/airflow/api_fastapi/core_api/serializers/providers.py b/airflow/api_fastapi/core_api/serializers/providers.py new file mode 100644 index 0000000000000..4e542f19f9f8e --- /dev/null +++ b/airflow/api_fastapi/core_api/serializers/providers.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from pydantic import BaseModel + + +class ProviderResponse(BaseModel): + """Provider serializer for responses.""" + + package_name: str + description: str + version: str + + +class ProviderCollectionResponse(BaseModel): + """Provider Collection serializer for responses.""" + + providers: list[ProviderResponse] + total_entries: int diff --git a/airflow/ui/openapi-gen/queries/common.ts b/airflow/ui/openapi-gen/queries/common.ts index 3a31b4c0ad993..98317d58ee4f3 100644 --- a/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow/ui/openapi-gen/queries/common.ts @@ -9,6 +9,7 @@ import { DashboardService, MonitorService, PoolService, + ProviderService, VariableService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; @@ -246,6 +247,24 @@ export const UseMonitorServiceGetHealthKeyFn = (queryKey?: Array) => [ useMonitorServiceGetHealthKey, ...(queryKey ?? []), ]; +export type ProviderServiceGetProvidersDefaultResponse = Awaited< + ReturnType +>; +export type ProviderServiceGetProvidersQueryResult< + TData = ProviderServiceGetProvidersDefaultResponse, + TError = unknown, +> = UseQueryResult; +export const useProviderServiceGetProvidersKey = "ProviderServiceGetProviders"; +export const UseProviderServiceGetProvidersKeyFn = ( + { + limit, + offset, + }: { + limit?: number; + offset?: number; + } = {}, + queryKey?: Array, +) => [useProviderServiceGetProvidersKey, ...(queryKey ?? [{ limit, offset }])]; export type VariableServicePostVariableMutationResult = Awaited< ReturnType >; diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow/ui/openapi-gen/queries/prefetch.ts index 98c706a63fc12..af05a26fc2618 100644 --- a/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow/ui/openapi-gen/queries/prefetch.ts @@ -8,6 +8,7 @@ import { DagService, DashboardService, MonitorService, + ProviderService, VariableService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; @@ -300,3 +301,26 @@ export const prefetchUseMonitorServiceGetHealth = (queryClient: QueryClient) => queryKey: Common.UseMonitorServiceGetHealthKeyFn(), queryFn: () => MonitorService.getHealth(), }); +/** + * Get Providers + * Get providers. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @returns ProviderCollectionResponse Successful Response + * @throws ApiError + */ +export const prefetchUseProviderServiceGetProviders = ( + queryClient: QueryClient, + { + limit, + offset, + }: { + limit?: number; + offset?: number; + } = {}, +) => + queryClient.prefetchQuery({ + queryKey: Common.UseProviderServiceGetProvidersKeyFn({ limit, offset }), + queryFn: () => ProviderService.getProviders({ limit, offset }), + }); diff --git a/airflow/ui/openapi-gen/queries/queries.ts b/airflow/ui/openapi-gen/queries/queries.ts index 6d358a7008a50..bf9a744f6cb53 100644 --- a/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow/ui/openapi-gen/queries/queries.ts @@ -14,6 +14,7 @@ import { DashboardService, MonitorService, PoolService, + ProviderService, VariableService, } from "../requests/services.gen"; import { DAGPatchBody, DagRunState, VariableBody } from "../requests/types.gen"; @@ -387,6 +388,38 @@ export const useMonitorServiceGetHealth = < queryFn: () => MonitorService.getHealth() as TData, ...options, }); +/** + * Get Providers + * Get providers. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @returns ProviderCollectionResponse Successful Response + * @throws ApiError + */ +export const useProviderServiceGetProviders = < + TData = Common.ProviderServiceGetProvidersDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + limit, + offset, + }: { + limit?: number; + offset?: number; + } = {}, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useQuery({ + queryKey: Common.UseProviderServiceGetProvidersKeyFn( + { limit, offset }, + queryKey, + ), + queryFn: () => ProviderService.getProviders({ limit, offset }) as TData, + ...options, + }); /** * Post Variable * Create a variable. diff --git a/airflow/ui/openapi-gen/queries/suspense.ts b/airflow/ui/openapi-gen/queries/suspense.ts index d3d8b7b3441a4..fad0d5b7a5a8e 100644 --- a/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow/ui/openapi-gen/queries/suspense.ts @@ -8,6 +8,7 @@ import { DagService, DashboardService, MonitorService, + ProviderService, VariableService, } from "../requests/services.gen"; import { DagRunState } from "../requests/types.gen"; @@ -381,3 +382,35 @@ export const useMonitorServiceGetHealthSuspense = < queryFn: () => MonitorService.getHealth() as TData, ...options, }); +/** + * Get Providers + * Get providers. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @returns ProviderCollectionResponse Successful Response + * @throws ApiError + */ +export const useProviderServiceGetProvidersSuspense = < + TData = Common.ProviderServiceGetProvidersDefaultResponse, + TError = unknown, + TQueryKey extends Array = unknown[], +>( + { + limit, + offset, + }: { + limit?: number; + offset?: number; + } = {}, + queryKey?: TQueryKey, + options?: Omit, "queryKey" | "queryFn">, +) => + useSuspenseQuery({ + queryKey: Common.UseProviderServiceGetProvidersKeyFn( + { limit, offset }, + queryKey, + ), + queryFn: () => ProviderService.getProviders({ limit, offset }) as TData, + ...options, + }); diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index f2214e2e30483..23458a325883d 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -1169,6 +1169,47 @@ export const $HistoricalMetricDataResponse = { description: "Historical Metric Data serializer for responses.", } as const; +export const $ProviderCollectionResponse = { + properties: { + providers: { + items: { + $ref: "#/components/schemas/ProviderResponse", + }, + type: "array", + title: "Providers", + }, + total_entries: { + type: "integer", + title: "Total Entries", + }, + }, + type: "object", + required: ["providers", "total_entries"], + title: "ProviderCollectionResponse", + description: "Provider Collection serializer for responses.", +} as const; + +export const $ProviderResponse = { + properties: { + package_name: { + type: "string", + title: "Package Name", + }, + description: { + type: "string", + title: "Description", + }, + version: { + type: "string", + title: "Version", + }, + }, + type: "object", + required: ["package_name", "description", "version"], + title: "ProviderResponse", + description: "Provider serializer for responses.", +} as const; + export const $SchedulerInfoSchema = { properties: { status: { diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow/ui/openapi-gen/requests/services.gen.ts index 64bf06c4d6f6f..0fa959e9725fa 100644 --- a/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow/ui/openapi-gen/requests/services.gen.ts @@ -42,6 +42,8 @@ import type { GetHealthResponse, DeletePoolData, DeletePoolResponse, + GetProvidersData, + GetProvidersResponse, } from "./types.gen"; export class AssetService { @@ -623,3 +625,30 @@ export class PoolService { }); } } + +export class ProviderService { + /** + * Get Providers + * Get providers. + * @param data The data for the request. + * @param data.limit + * @param data.offset + * @returns ProviderCollectionResponse Successful Response + * @throws ApiError + */ + public static getProviders( + data: GetProvidersData = {}, + ): CancelablePromise { + return __request(OpenAPI, { + method: "GET", + url: "/public/providers/", + query: { + limit: data.limit, + offset: data.offset, + }, + errors: { + 422: "Validation Error", + }, + }); + } +} diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow/ui/openapi-gen/requests/types.gen.ts index ef3847b23f2e0..8cbc0b7e0dabd 100644 --- a/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow/ui/openapi-gen/requests/types.gen.ts @@ -258,6 +258,23 @@ export type HistoricalMetricDataResponse = { task_instance_states: TaskInstanceState; }; +/** + * Provider Collection serializer for responses. + */ +export type ProviderCollectionResponse = { + providers: Array; + total_entries: number; +}; + +/** + * Provider serializer for responses. + */ +export type ProviderResponse = { + package_name: string; + description: string; + version: string; +}; + /** * Schema for Scheduler info. */ @@ -472,6 +489,13 @@ export type DeletePoolData = { export type DeletePoolResponse = void; +export type GetProvidersData = { + limit?: number; + offset?: number; +}; + +export type GetProvidersResponse = ProviderCollectionResponse; + export type $OpenApiTs = { "/ui/next_run_assets/{dag_id}": { get: { @@ -974,4 +998,19 @@ export type $OpenApiTs = { }; }; }; + "/public/providers/": { + get: { + req: GetProvidersData; + res: { + /** + * Successful Response + */ + 200: ProviderCollectionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; }; diff --git a/tests/api_fastapi/core_api/routes/public/test_providers.py b/tests/api_fastapi/core_api/routes/public/test_providers.py new file mode 100644 index 0000000000000..500913b7f6173 --- /dev/null +++ b/tests/api_fastapi/core_api/routes/public/test_providers.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers_manager import ProviderInfo + +pytestmark = [pytest.mark.db_test, pytest.mark.skip_if_database_isolation_mode] + +MOCK_PROVIDERS = { + "apache-airflow-providers-amazon": ProviderInfo( + "1.0.0", + { + "package-name": "apache-airflow-providers-amazon", + "name": "Amazon", + "description": "`Amazon Web Services (AWS) `__.\n", + "versions": ["1.0.0"], + }, + "package", + ), + "apache-airflow-providers-apache-cassandra": ProviderInfo( + "1.0.0", + { + "package-name": "apache-airflow-providers-apache-cassandra", + "name": "Apache Cassandra", + "description": "`Apache Cassandra `__.\n", + "versions": ["1.0.0"], + }, + "package", + ), +} + + +class TestGetProviders: + @pytest.mark.parametrize( + "query_params, expected_total_entries, expected_package_name", + [ + # Filters + ({}, 2, ["apache-airflow-providers-amazon", "apache-airflow-providers-apache-cassandra"]), + ({"limit": 1}, 2, ["apache-airflow-providers-amazon"]), + ({"offset": 1}, 2, ["apache-airflow-providers-apache-cassandra"]), + ], + ) + @mock.patch( + "airflow.providers_manager.ProvidersManager.providers", + new_callable=mock.PropertyMock, + return_value=MOCK_PROVIDERS, + ) + def test_get_dags( + self, mock_provider, test_client, query_params, expected_total_entries, expected_package_name + ): + response = test_client.get("/public/providers", params=query_params) + + assert response.status_code == 200 + body = response.json() + + assert body["total_entries"] == expected_total_entries + assert [provider["package_name"] for provider in body["providers"]] == expected_package_name