From 50273e6b0e965c5a89dd44718cbd0c77a0b05121 Mon Sep 17 00:00:00 2001 From: Allison King Date: Mon, 26 Jun 2023 17:01:02 -0400 Subject: [PATCH 1/3] Transform null names to an empty string before submission (#3683) --- CHANGELOG.md | 1 + .../privacy-declarations/PrivacyDeclarationManager.tsx | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19e52f9a13..92eb2b74de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ The types of changes are: - Fix bug where linking an integration would not update the tab when creating a new system [#3662](https://github.com/ethyca/fides/pull/3662) - Fix dataset yaml not properly reflecting the dataset in the dropdown of system integrations tab [#3666](https://github.com/ethyca/fides/pull/3666) - Fix privacy notices not being able to be edited via the UI after the addition of the `cookies` field [#3670](https://github.com/ethyca/fides/pull/3670) +- Add a transform in the case of `null` name fields in privacy declarations for the data use forms [#3683](https://github.com/ethyca/fides/pull/3683) ### Changed diff --git a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx index 3a8a1318db..5ea9760e4f 100644 --- a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx +++ b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx @@ -78,9 +78,15 @@ const PrivacyDeclarationManager = ({ updatedDeclarations: PrivacyDeclarationResponse[], isDelete?: boolean ) => { + // The API can return a null name, but cannot receive a null name, + // so do an additional transform here (fides#3862) + const transformedDeclarations = updatedDeclarations.map((d) => ({ + ...d, + name: d.name ?? "", + })); const systemBodyWithDeclaration = { ...system, - privacy_declarations: updatedDeclarations, + privacy_declarations: transformedDeclarations, }; const handleResult = ( result: From 243f6e2332cc8660703f13db36776ef006e6c7fa Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 27 Jun 2023 16:31:20 +0800 Subject: [PATCH 2/3] Fix AWS Scan Test (#3556) --- .github/workflows/backend_checks.yml | 4 +- tests/ctl/core/test_system.py | 221 ++++++++++++++------------- 2 files changed, 115 insertions(+), 110 deletions(-) diff --git a/.github/workflows/backend_checks.yml b/.github/workflows/backend_checks.yml index 500e5a11f2..4c3a44812c 100644 --- a/.github/workflows/backend_checks.yml +++ b/.github/workflows/backend_checks.yml @@ -260,8 +260,8 @@ jobs: env: SNOWFLAKE_FIDESCTL_PASSWORD: ${{ secrets.SNOWFLAKE_FIDESCTL_PASSWORD }} REDSHIFT_FIDESCTL_PASSWORD: ${{ secrets.REDSHIFT_FIDESCTL_PASSWORD }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_FIDESCTL_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_FIDESCTL_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CTL_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CTL_SECRET_ACCESS_KEY }} OKTA_CLIENT_TOKEN: ${{ secrets.OKTA_FIDESCTL_CLIENT_TOKEN }} AWS_DEFAULT_REGION: us-east-1 BIGQUERY_CONFIG: ${{ secrets.BIGQUERY_CONFIG }} diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index a6eaaf5fc0..c67f768891 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -99,18 +99,18 @@ def redshift_describe_clusters() -> Generator: { "ClusterIdentifier": "redshift-cluster-1", "Endpoint": { - "Address": "redshift-cluster-1.c2angfh5kpo4.us-east-1.redshift.amazonaws.com", + "Address": "redshift-cluster-1.cue8hjdl1kb1.us-east-1.redshift.amazonaws.com", "Port": 5439, }, - "ClusterNamespaceArn": "arn:aws:redshift:us-east-1:910934740016:namespace:057d5b0e-7eaa-4012-909c-3957c7149176", + "ClusterNamespaceArn": "arn:aws:redshift:us-east-1:469973866127:namespace:5eb1f195-7815-4c62-9140-e062dd98da83", }, { "ClusterIdentifier": "redshift-cluster-2", "Endpoint": { - "Address": "redshift-cluster-2.c2angfh5kpo4.us-east-1.redshift.amazonaws.com", + "Address": "redshift-cluster-2.cue8hjdl1kb1.us-east-1.redshift.amazonaws.com", "Port": 5439, }, - "ClusterNamespaceArn": "arn:aws:redshift:us-east-1:910934740016:namespace:057d5b0e-7eaa-4012-909c-3957c7149177", + "ClusterNamespaceArn": "arn:aws:redshift:us-east-1:469973866127:namespace:06ba7fe3-8cb3-4e1c-b2c6-cc2f2415a979", }, ] } @@ -120,28 +120,28 @@ def redshift_describe_clusters() -> Generator: @pytest.fixture() def redshift_systems() -> Generator: redshift_systems = [ - System( + System.construct( fides_key="redshift-cluster-1", organization_fides_key="default_organization", name="redshift-cluster-1", description="Fides Generated Description for Redshift Cluster: redshift-cluster-1", fidesctl_meta=SystemMetadata( - endpoint_address="redshift-cluster-1.c2angfh5kpo4.us-east-1.redshift.amazonaws.com", + endpoint_address="redshift-cluster-1.cue8hjdl1kb1.us-east-1.redshift.amazonaws.com", endpoint_port="5439", - resource_id="arn:aws:redshift:us-east-1:910934740016:namespace:057d5b0e-7eaa-4012-909c-3957c7149176", + resource_id="arn:aws:redshift:us-east-1:469973866127:namespace:5eb1f195-7815-4c62-9140-e062dd98da83", ), system_type="redshift_cluster", privacy_declarations=[], ), - System( + System.construct( fides_key="redshift-cluster-2", organization_fides_key="default_organization", name="redshift-cluster-2", description="Fides Generated Description for Redshift Cluster: redshift-cluster-2", fidesctl_meta=SystemMetadata( - endpoint_address="redshift-cluster-2.c2angfh5kpo4.us-east-1.redshift.amazonaws.com", + endpoint_address="redshift-cluster-2.cue8hjdl1kb1.us-east-1.redshift.amazonaws.com", endpoint_port="5439", - resource_id="arn:aws:redshift:us-east-1:910934740016:namespace:057d5b0e-7eaa-4012-909c-3957c7149177", + resource_id="arn:aws:redshift:us-east-1:469973866127:namespace:06ba7fe3-8cb3-4e1c-b2c6-cc2f2415a979", ), system_type="redshift_cluster", privacy_declarations=[], @@ -189,9 +189,9 @@ def rds_describe_clusters() -> Generator: "DBClusters": [ { "DBClusterIdentifier": "database-2", - "Endpoint": "database-2.cluster-ckrdpkkb4ukm.us-east-1.rds.amazonaws.com", + "Endpoint": "database-2.cluster-cjh1qplnnv3b.us-east-1.rds.amazonaws.com", "Port": 3306, - "DBClusterArn": "arn:aws:rds:us-east-1:910934740016:cluster:database-2", + "DBClusterArn": "arn:aws:rds:us-east-1:469973866127:cluster:database-2", }, ] } @@ -205,38 +205,16 @@ def rds_describe_instances() -> Generator: { "DBInstanceIdentifier": "database-1", "Endpoint": { - "Address": "database-1.ckrdpkkb4ukm.us-east-1.rds.amazonaws.com", + "Address": "database-1.cjh1qplnnv3b.us-east-1.rds.amazonaws.com", "Port": 3306, }, - "DBInstanceArn": "arn:aws:rds:us-east-1:910934740016:db:database-1", + "DBInstanceArn": "arn:aws:rds:us-east-1:469973866127:db:database-1", }, ] } yield describe_instances -@pytest.mark.unit -def test_get_system_resource_ids(redshift_systems: List[System]) -> None: - expected_result = [ - "arn:aws:redshift:us-east-1:910934740016:namespace:057d5b0e-7eaa-4012-909c-3957c7149176", - "arn:aws:redshift:us-east-1:910934740016:namespace:057d5b0e-7eaa-4012-909c-3957c7149177", - ] - actual_result = _system.get_system_resource_ids(redshift_systems) - assert actual_result == expected_result - - -@pytest.mark.unit -def test_find_missing_systems( - redshift_systems: List[System], rds_systems: List[System] -) -> None: - source_systems = rds_systems + redshift_systems - existing_systems = redshift_systems - actual_result = _system.find_missing_systems( - source_systems=source_systems, existing_systems=existing_systems - ) - assert actual_result == rds_systems - - @pytest.mark.integration def test_get_all_server_systems( test_config: FidesConfig, create_test_server_systems: Generator @@ -249,94 +227,121 @@ def test_get_all_server_systems( assert actual_result -@pytest.mark.external -def test_scan_system_aws_passes( - test_config: FidesConfig, create_external_server_systems: Generator -) -> None: - _system.scan_system_aws( - coverage_threshold=100, - manifest_dir="", - organization_key="default_organization", - aws_config=None, - url=test_config.cli.server_url, - headers=test_config.user.auth_header, - ) +class TestSystemAWS: + @pytest.mark.unit + def test_get_system_resource_ids(self, redshift_systems: List[System]) -> None: + expected_result = [ + "arn:aws:redshift:us-east-1:469973866127:namespace:5eb1f195-7815-4c62-9140-e062dd98da83", + "arn:aws:redshift:us-east-1:469973866127:namespace:06ba7fe3-8cb3-4e1c-b2c6-cc2f2415a979", + ] + actual_result = _system.get_system_resource_ids(redshift_systems) + assert actual_result == expected_result + + @pytest.mark.unit + def test_find_missing_systems( + self, redshift_systems: List[System], rds_systems: List[System] + ) -> None: + source_systems = rds_systems + redshift_systems + existing_systems = redshift_systems + actual_result = _system.find_missing_systems( + source_systems=source_systems, existing_systems=existing_systems + ) + assert actual_result == rds_systems + @pytest.mark.external + def test_scan_system_aws_passes( + self, test_config: FidesConfig, create_external_server_systems: Generator + ) -> None: + _system.scan_system_aws( + coverage_threshold=100, + manifest_dir="", + organization_key="default_organization", + aws_config=None, + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + ) -@pytest.mark.external -def test_generate_system_aws(tmpdir: LocalPath, test_config: FidesConfig) -> None: - actual_result = _system.generate_system_aws( - file_name=f"{tmpdir}/test_file.yml", - include_null=False, - organization_key="default_organization", - aws_config=None, - url=test_config.cli.server_url, - headers=test_config.user.auth_header, - ) - assert actual_result + @pytest.mark.external + def test_generate_system_aws( + self, tmpdir: LocalPath, test_config: FidesConfig + ) -> None: + actual_result = _system.generate_system_aws( + file_name=f"{tmpdir}/test_file.yml", + include_null=False, + organization_key="default_organization", + aws_config=None, + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + ) + assert actual_result OKTA_ORG_URL = "https://dev-78908748.okta.com" -@pytest.mark.external -def test_generate_system_okta(tmpdir: LocalPath, test_config: FidesConfig) -> None: - actual_result = _system.generate_system_okta( - file_name=f"{tmpdir}/test_file.yml", - include_null=False, - organization_key="default_organization", - okta_config=OktaConfig( - orgUrl=OKTA_ORG_URL, - token=os.environ["OKTA_CLIENT_TOKEN"], - ), - url=test_config.cli.server_url, - headers=test_config.user.auth_header, - ) - assert actual_result - - -@pytest.mark.external -def test_scan_system_okta_success(tmpdir: LocalPath, test_config: FidesConfig) -> None: - file_name = f"{tmpdir}/test_file.yml" - _system.generate_system_okta( - file_name=file_name, - include_null=False, - organization_key="default_organization", - okta_config=OktaConfig( - orgUrl=OKTA_ORG_URL, - token=os.environ["OKTA_CLIENT_TOKEN"], - ), - url=test_config.cli.server_url, - headers=test_config.user.auth_header, - ) - _system.scan_system_okta( - manifest_dir=file_name, - okta_config=OktaConfig( - orgUrl=OKTA_ORG_URL, - token=os.environ["OKTA_CLIENT_TOKEN"], - ), - organization_key="default_organization", - coverage_threshold=100, - url=test_config.cli.server_url, - headers=test_config.user.auth_header, - ) - assert True - - -@pytest.mark.external -def test_scan_system_okta_fail(tmpdir: LocalPath, test_config: FidesConfig) -> None: - with pytest.raises(SystemExit): +class TestSystemOkta: + @pytest.mark.external + def test_generate_system_okta( + self, tmpdir: LocalPath, test_config: FidesConfig + ) -> None: + actual_result = _system.generate_system_okta( + file_name=f"{tmpdir}/test_file.yml", + include_null=False, + organization_key="default_organization", + okta_config=OktaConfig( + orgUrl=OKTA_ORG_URL, + token=os.environ["OKTA_CLIENT_TOKEN"], + ), + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + ) + assert actual_result + + @pytest.mark.external + def test_scan_system_okta_success( + self, tmpdir: LocalPath, test_config: FidesConfig + ) -> None: + file_name = f"{tmpdir}/test_file.yml" + _system.generate_system_okta( + file_name=file_name, + include_null=False, + organization_key="default_organization", + okta_config=OktaConfig( + orgUrl=OKTA_ORG_URL, + token=os.environ["OKTA_CLIENT_TOKEN"], + ), + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + ) _system.scan_system_okta( - manifest_dir="", + manifest_dir=file_name, okta_config=OktaConfig( orgUrl=OKTA_ORG_URL, token=os.environ["OKTA_CLIENT_TOKEN"], ), - coverage_threshold=100, organization_key="default_organization", + coverage_threshold=100, url=test_config.cli.server_url, headers=test_config.user.auth_header, ) + assert True + + @pytest.mark.external + def test_scan_system_okta_fail( + self, tmpdir: LocalPath, test_config: FidesConfig + ) -> None: + with pytest.raises(SystemExit): + _system.scan_system_okta( + manifest_dir="", + okta_config=OktaConfig( + orgUrl=OKTA_ORG_URL, + token=os.environ["OKTA_CLIENT_TOKEN"], + ), + coverage_threshold=100, + organization_key="default_organization", + url=test_config.cli.server_url, + headers=test_config.user.auth_header, + ) class TestUpsertCookies: From f47bc5d546dc36610511fbbb92279ccb6861231d Mon Sep 17 00:00:00 2001 From: Robert Keyser <39230492+RobertKeyser@users.noreply.github.com> Date: Tue, 27 Jun 2023 15:41:50 -0500 Subject: [PATCH 3/3] Repoint Links to Moved Fides Documentation (#3643) --- CHANGELOG.md | 3 +++ clients/admin-ui/src/pages/management/about.tsx | 2 +- docs/README.md | 2 +- docs/fides/docs/development/documentation.md | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92eb2b74de..c5e4b25967 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ The types of changes are: ### Fixed - Render linebreaks in the Fides.js overlay descriptions, etc. [#3665](https://github.com/ethyca/fides/pull/3665) +- Broken link to Fides docs site on the About Fides page in Admin UI [#3643](https://github.com/ethyca/fides/pull/3643) + +### Developer Experience ### Changed - Moved GPC preferences slightly earlier in Fides.js lifecycle [#3561](https://github.com/ethyca/fides/pull/3561) diff --git a/clients/admin-ui/src/pages/management/about.tsx b/clients/admin-ui/src/pages/management/about.tsx index d012b3b4fa..6062d317a8 100644 --- a/clients/admin-ui/src/pages/management/about.tsx +++ b/clients/admin-ui/src/pages/management/about.tsx @@ -78,7 +78,7 @@ const About: NextPage = () => { Please visit{" "} docs.ethyca.com diff --git a/docs/README.md b/docs/README.md index 148ede534e..a73b65f351 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,4 +2,4 @@ This is where the autogenerated docs for Fides lives. To run the docs locally, use `nox -s docs_serve` and visit `localhost:8000` -The full, official docs live at https://docs.ethyca.com/fides/overview +The full, official docs live at https://docs.ethyca.com diff --git a/docs/fides/docs/development/documentation.md b/docs/fides/docs/development/documentation.md index 5f4b0b2683..330f5005f1 100644 --- a/docs/fides/docs/development/documentation.md +++ b/docs/fides/docs/development/documentation.md @@ -1,6 +1,6 @@ # Documentation -Primary documentation for Fides now lives at , however autogenerated documentation as well as developer documentation is still maintained and hosted within the [Fides project repo](https://github.com/ethyca/fides). +Primary documentation for Fides now lives at , however autogenerated documentation as well as developer documentation is still maintained and hosted within the [Fides project repo](https://github.com/ethyca/fides). ## Previewing Docs Locally