Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[dlp] fix: mitigate flakiness #3919

Merged
merged 5 commits into from
Jun 1, 2020
Merged

[dlp] fix: mitigate flakiness #3919

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0f436e1
[dlp] fix: mitigate flakiness
May 30, 2020
dd542ad
more retries, comment
May 30, 2020
3d92e61
30 seconds operation wait and 20 minutes retry delay
May 30, 2020
906ad3e
lint fix etc
May 30, 2020
4262575
limit the max retry wait time
May 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions dlp/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest


# Used in risk_test.py to limit the maximum wait time before the flaky retries.
def pytest_configure(config):
pytest.MAX_FLAKY_WAIT = 3600 # maximum of an hour
56 changes: 43 additions & 13 deletions dlp/risk_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

import os
import time
import uuid

import google.cloud.bigquery
Expand All @@ -36,9 +37,14 @@
BIGQUERY_TABLE_ID = "dlp_test_table" + UNIQUE_STRING
BIGQUERY_HARMFUL_TABLE_ID = "harmful" + UNIQUE_STRING

TIMEOUT = 30


# Create new custom topic/subscription
@pytest.fixture(scope="module")
# We observe sometimes all the tests in this file fail. In a
# hypothesis where DLP service somehow loses the connection to the
# topic, now we use function scope for Pub/Sub fixtures.
@pytest.fixture(scope="function")
def topic_id():
# Creates a pubsub topic, and tears it down.
publisher = google.cloud.pubsub.PublisherClient()
Expand All @@ -53,7 +59,7 @@ def topic_id():
publisher.delete_topic(topic_path)


@pytest.fixture(scope="module")
@pytest.fixture(scope="function")
def subscription_id(topic_id):
# Subscribes to a topic.
subscriber = google.cloud.pubsub.SubscriberClient()
Expand Down Expand Up @@ -160,7 +166,22 @@ def bigquery_project():
bigquery_client.delete_dataset(dataset_ref, delete_contents=True)


@pytest.mark.flaky
def delay(err, *args):
# 20 mins of delay. This sounds like too long a delay, but we
# occasionally observe consequtive time block where operations are
# slow which leads to the test failures. These situations tend to
# get self healed in 20 minutes or so, so I'm trying this strategy.
#
# There are 10 tests, so we don't want the retry delay happening
# for all the tests. When we exhaust the MAX_FLAKY_WAIT, we retry
# the test immediately.
wait_time = min(pytest.MAX_FLAKY_WAIT, 60*20)
pytest.MAX_FLAKY_WAIT -= wait_time
time.sleep(wait_time)
return True


@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_numerical_risk_analysis(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -172,13 +193,14 @@ def test_numerical_risk_analysis(
NUMERIC_FIELD,
topic_id,
subscription_id,
timeout=TIMEOUT,
)

out, _ = capsys.readouterr()
assert "Value Range:" in out


@pytest.mark.flaky
@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_categorical_risk_analysis_on_string_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -190,14 +212,14 @@ def test_categorical_risk_analysis_on_string_field(
UNIQUE_FIELD,
topic_id,
subscription_id,
timeout=180,
timeout=TIMEOUT,
)

out, _ = capsys.readouterr()
assert "Most common value occurs" in out


@pytest.mark.flaky
@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_categorical_risk_analysis_on_number_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -209,13 +231,14 @@ def test_categorical_risk_analysis_on_number_field(
NUMERIC_FIELD,
topic_id,
subscription_id,
timeout=TIMEOUT,
)

out, _ = capsys.readouterr()
assert "Most common value occurs" in out


@pytest.mark.flaky
@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_k_anonymity_analysis_single_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -227,14 +250,15 @@ def test_k_anonymity_analysis_single_field(
topic_id,
subscription_id,
[NUMERIC_FIELD],
timeout=TIMEOUT,
)

out, _ = capsys.readouterr()
assert "Quasi-ID values:" in out
assert "Class size:" in out


@pytest.mark.flaky(max_runs=3, min_passes=1)
@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_k_anonymity_analysis_multiple_fields(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -246,14 +270,15 @@ def test_k_anonymity_analysis_multiple_fields(
topic_id,
subscription_id,
[NUMERIC_FIELD, REPEATED_FIELD],
timeout=TIMEOUT,
)

out, _ = capsys.readouterr()
assert "Quasi-ID values:" in out
assert "Class size:" in out


@pytest.mark.flaky
@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_l_diversity_analysis_single_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -266,6 +291,7 @@ def test_l_diversity_analysis_single_field(
subscription_id,
UNIQUE_FIELD,
[NUMERIC_FIELD],
timeout=TIMEOUT,
)

out, _ = capsys.readouterr()
Expand All @@ -274,7 +300,7 @@ def test_l_diversity_analysis_single_field(
assert "Sensitive value" in out


@pytest.mark.flaky(max_runs=3, min_passes=1)
@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_l_diversity_analysis_multiple_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -287,6 +313,7 @@ def test_l_diversity_analysis_multiple_field(
subscription_id,
UNIQUE_FIELD,
[NUMERIC_FIELD, REPEATED_FIELD],
timeout=TIMEOUT,
)

out, _ = capsys.readouterr()
Expand All @@ -295,7 +322,7 @@ def test_l_diversity_analysis_multiple_field(
assert "Sensitive value" in out


@pytest.mark.flaky
@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_k_map_estimate_analysis_single_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -308,6 +335,7 @@ def test_k_map_estimate_analysis_single_field(
subscription_id,
[NUMERIC_FIELD],
["AGE"],
timeout=TIMEOUT,
)

out, _ = capsys.readouterr()
Expand All @@ -316,7 +344,7 @@ def test_k_map_estimate_analysis_single_field(
assert "Values" in out


@pytest.mark.flaky(max_runs=3, min_passes=1)
@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_k_map_estimate_analysis_multiple_field(
topic_id, subscription_id, bigquery_project, capsys
):
Expand All @@ -329,6 +357,7 @@ def test_k_map_estimate_analysis_multiple_field(
subscription_id,
[NUMERIC_FIELD, STRING_BOOLEAN_FIELD],
["AGE", "GENDER"],
timeout=TIMEOUT,
)

out, _ = capsys.readouterr()
Expand All @@ -337,7 +366,7 @@ def test_k_map_estimate_analysis_multiple_field(
assert "Values" in out


@pytest.mark.flaky
@pytest.mark.flaky(max_runs=2, min_passes=1, rerun_filter=delay)
def test_k_map_estimate_analysis_quasi_ids_info_types_equal(
topic_id, subscription_id, bigquery_project
):
Expand All @@ -351,4 +380,5 @@ def test_k_map_estimate_analysis_quasi_ids_info_types_equal(
subscription_id,
[NUMERIC_FIELD, STRING_BOOLEAN_FIELD],
["AGE"],
timeout=TIMEOUT,
)