diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ecc9bdf..cfce9d14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.6.1] - 2024-12-02 +## Added +- Added `maven-jar-plugin` in `beekeeper-integration-test`. +## Changed +- Changed scope of attributes to `protected` in `beekeeper-integration-test` + ## [3.6.0] - 2024-11-29 ## Added - Added a db migration file and implementation of `beekeeper-history` table to track beekeeper activity. diff --git a/README.md b/README.md index e4937cf9..76412787 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ - [Beekeeper-API](#beekeeper-api) - [External Links](#external-links) - [Legal](#legal) - + # Overview Beekeeper is a service that schedules orphaned paths and expired metadata for deletion. diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index 01c83095..cbd374fc 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -134,4 +134,24 @@ flyway-mysql + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + + test-jar + + + ${project.artifactId}-${project.version} + + + + + + diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index c1be07e1..057ebc8c 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -70,25 +70,25 @@ @Testcontainers public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase { - private static final int TIMEOUT = 30; - private static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; + protected static final int TIMEOUT = 30; + protected static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; - private static final String QUEUE = "apiary-receiver-queue"; - private static final String SCHEDULED_EXPIRED_METRIC = "metadata-scheduled"; - private static final String HEALTHCHECK_URI = "http://localhost:8080/actuator/health"; - private static final String PROMETHEUS_URI = "http://localhost:8080/actuator/prometheus"; + protected static final String QUEUE = "apiary-receiver-queue"; + protected static final String SCHEDULED_EXPIRED_METRIC = "metadata-scheduled"; + protected static final String HEALTHCHECK_URI = "http://localhost:8080/actuator/health"; + protected static final String PROMETHEUS_URI = "http://localhost:8080/actuator/prometheus"; - private static final String PARTITION_KEYS = "{ \"event_date\": \"date\", \"event_hour\": \"smallint\"}"; - private static final String PARTITION_A_VALUES = "[ \"2020-01-01\", \"0\" ]"; - private static final String PARTITION_B_VALUES = "[ \"2020-01-01\", \"1\" ]"; - private static final String PARTITION_A_NAME = "event_date=2020-01-01/event_hour=0"; - private static final String PARTITION_B_NAME = "event_date=2020-01-01/event_hour=1"; - private static final String LOCATION_A = "s3://bucket/table1/partition"; - private static final String LOCATION_B = "s3://bucket/table2/partition"; + protected static final String PARTITION_KEYS = "{ \"event_date\": \"date\", \"event_hour\": \"smallint\"}"; + protected static final String PARTITION_A_VALUES = "[ \"2020-01-01\", \"0\" ]"; + protected static final String PARTITION_B_VALUES = "[ \"2020-01-01\", \"1\" ]"; + protected static final String PARTITION_A_NAME = "event_date=2020-01-01/event_hour=0"; + protected static final String PARTITION_B_NAME = "event_date=2020-01-01/event_hour=1"; + protected static final String LOCATION_A = "s3://bucket/table1/partition"; + protected static final String LOCATION_B = "s3://bucket/table2/partition"; @Container - private static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); - private static AmazonSQS amazonSQS; + protected static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); + protected static AmazonSQS amazonSQS; @BeforeAll public static void init() { @@ -256,11 +256,11 @@ public void prometheus() { await().atMost(30, TimeUnit.SECONDS).until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } - private SendMessageRequest sendMessageRequest(String payload) { + protected SendMessageRequest sendMessageRequest(String payload) { return new SendMessageRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE), payload); } - private void assertExpiredMetadata(HousekeepingMetadata actual, String expectedPath, String partitionName) { + protected void assertExpiredMetadata(HousekeepingMetadata actual, String expectedPath, String partitionName) { assertHousekeepingMetadata(actual, expectedPath, partitionName); assertMetrics(); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 6060f5cf..e075aa17 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -118,7 +118,7 @@ public abstract class BeekeeperIntegrationTestBase { // MySQL DB CONTAINER AND UTILS @Container private static final MySQLContainer MY_SQL_CONTAINER = ContainerTestUtils.mySqlContainer(); - private static MySqlTestUtils mySQLTestUtils; + protected static MySqlTestUtils mySQLTestUtils; protected final ExecutorService executorService = Executors.newFixedThreadPool(1); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java index fdaff3f4..9252ac97 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2022 Expedia, Inc. + * Copyright (C) 2019-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -81,47 +81,47 @@ @Testcontainers public class BeekeeperMetadataCleanupIntegrationTest extends BeekeeperIntegrationTestBase { - private static final int TIMEOUT = 15; - private static final String SCHEDULER_DELAY_MS = "5000"; - private static final String HEALTHCHECK_URI = "http://localhost:9008/actuator/health"; - private static final String PROMETHEUS_URI = "http://localhost:9008/actuator/prometheus"; + protected static final int TIMEOUT = 15; + protected static final String SCHEDULER_DELAY_MS = "5000"; + protected static final String HEALTHCHECK_URI = "http://localhost:9008/actuator/health"; + protected static final String PROMETHEUS_URI = "http://localhost:9008/actuator/prometheus"; - private static final String SPRING_PROFILES_ACTIVE_PROPERTY = "spring.profiles.active"; - private static final String SCHEDULER_DELAY_MS_PROPERTY = "properties.scheduler-delay-ms"; - private static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; - private static final String AWS_S3_ENDPOINT_PROPERTY = "aws.s3.endpoint"; - private static final String METASTORE_URI_PROPERTY = "properties.metastore-uri"; - private static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disableGetObjectMD5Validation"; - private static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disablePutObjectMD5Validation"; + protected static final String SPRING_PROFILES_ACTIVE_PROPERTY = "spring.profiles.active"; + protected static final String SCHEDULER_DELAY_MS_PROPERTY = "properties.scheduler-delay-ms"; + protected static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; + protected static final String AWS_S3_ENDPOINT_PROPERTY = "aws.s3.endpoint"; + protected static final String METASTORE_URI_PROPERTY = "properties.metastore-uri"; + protected static final String AWS_DISABLE_GET_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disableGetObjectMD5Validation"; + protected static final String AWS_DISABLE_PUT_VALIDATION_PROPERTY = "com.amazonaws.services.s3.disablePutObjectMD5Validation"; - private static final String S3_ACCESS_KEY = "access"; - private static final String S3_SECRET_KEY = "secret"; + protected static final String S3_ACCESS_KEY = "access"; + protected static final String S3_SECRET_KEY = "secret"; - private static final String BUCKET = "test-path-bucket"; - private static final String TABLE_DATA = "1\tadam\tlondon\n2\tsusan\tglasgow\n"; - private static final String PARTITIONED_TABLE_NAME = TABLE_NAME_VALUE + "_partitioned"; - private static final String UNPARTITIONED_TABLE_NAME = TABLE_NAME_VALUE + "_unpartitioned"; + protected static final String BUCKET = "test-path-bucket"; + protected static final String TABLE_DATA = "1\tadam\tlondon\n2\tsusan\tglasgow\n"; + protected static final String PARTITIONED_TABLE_NAME = TABLE_NAME_VALUE + "_partitioned"; + protected static final String UNPARTITIONED_TABLE_NAME = TABLE_NAME_VALUE + "_unpartitioned"; - private static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A"; - private static final List PARTITION_VALUES = List.of("2020-01-01", "0", "A"); + protected static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A"; + protected static final List PARTITION_VALUES = List.of("2020-01-01", "0", "A"); - private static final String ROOT_PATH = "s3a://" + BUCKET + "/" + DATABASE_NAME_VALUE + "/"; + protected static final String ROOT_PATH = "s3a://" + BUCKET + "/" + DATABASE_NAME_VALUE + "/"; - private static final String PARTITIONED_TABLE_PATH = ROOT_PATH + PARTITIONED_TABLE_NAME + "/id1"; - private static final String PARTITION_ROOT_PATH = ROOT_PATH + "some_location/id1"; - private static final String PARTITION_PATH = PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; - private static final String PARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE + protected static final String PARTITIONED_TABLE_PATH = ROOT_PATH + PARTITIONED_TABLE_NAME + "/id1"; + protected static final String PARTITION_ROOT_PATH = ROOT_PATH + "some_location/id1"; + protected static final String PARTITION_PATH = PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1"; + protected static final String PARTITIONED_TABLE_OBJECT_KEY = DATABASE_NAME_VALUE + "/" + PARTITIONED_TABLE_NAME + "/id1"; - private static final String PARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE + protected static final String PARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE + "/some_location/id1/" + PARTITION_NAME + "/file1"; - private static final String UNPARTITIONED_TABLE_PATH = ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; - private static final String UNPARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE + protected static final String UNPARTITIONED_TABLE_PATH = ROOT_PATH + UNPARTITIONED_TABLE_NAME + "/id1"; + protected static final String UNPARTITIONED_OBJECT_KEY = DATABASE_NAME_VALUE + "/" + UNPARTITIONED_TABLE_NAME + "/id1/file1"; @@ -132,9 +132,9 @@ public class BeekeeperMetadataCleanupIntegrationTest extends BeekeeperIntegratio S3_CONTAINER.start(); } - private static AmazonS3 amazonS3; + protected static AmazonS3 amazonS3; private static final String S3_ENDPOINT = ContainerTestUtils.awsServiceEndpoint(S3_CONTAINER, S3); - private final ExecutorService executorService = Executors.newFixedThreadPool(1); + protected final ExecutorService executorService = Executors.newFixedThreadPool(1); private static Map metastoreProperties = ImmutableMap .builder() @@ -147,8 +147,8 @@ public class BeekeeperMetadataCleanupIntegrationTest extends BeekeeperIntegratio public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( DATABASE_NAME_VALUE, metastoreProperties); - private HiveTestUtils hiveTestUtils; - private HiveMetaStoreClient metastoreClient; + protected HiveTestUtils hiveTestUtils; + protected HiveMetaStoreClient metastoreClient; @BeforeAll public static void init() { @@ -178,7 +178,7 @@ public static void teardown() { } @BeforeEach - void setup() { + public void setup() { System.setProperty(METASTORE_URI_PROPERTY, thriftHiveMetaStore.getThriftConnectionUri()); metastoreClient = thriftHiveMetaStore.client(); hiveTestUtils = new HiveTestUtils(metastoreClient); @@ -192,7 +192,7 @@ void setup() { } @AfterEach - void stop() throws InterruptedException { + public void stop() throws InterruptedException { BeekeeperMetadataCleanup.stop(); executorService.awaitTermination(5, TimeUnit.SECONDS); } @@ -433,7 +433,7 @@ public void metrics() throws Exception { assertMetrics(); } - private void assertMetrics() { + protected void assertMetrics() { Set meterRegistry = ((CompositeMeterRegistry) BeekeeperMetadataCleanup.meterRegistry()).getRegistries(); assertThat(meterRegistry).hasSize(2); meterRegistry.forEach(registry -> { diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java index 8f4c4c17..64309c55 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -60,35 +60,35 @@ @Testcontainers public class BeekeeperPathCleanupIntegrationTest extends BeekeeperIntegrationTestBase { - private static final int TIMEOUT = 30; - private static final String SPRING_PROFILES_ACTIVE_PROPERTY = "spring.profiles.active"; - private static final String SCHEDULER_DELAY_MS_PROPERTY = "properties.scheduler-delay-ms"; - private static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; - private static final String AWS_S3_ENDPOINT_PROPERTY = "aws.s3.endpoint"; - - private static final String BUCKET = "test-path-bucket"; - private static final String DB_AND_TABLE_PREFIX = DATABASE_NAME_VALUE + "/" + TABLE_NAME_VALUE; - private static final String OBJECT_KEY_ROOT = DB_AND_TABLE_PREFIX + "/id1/partition1"; - private static final String OBJECT_KEY1 = DB_AND_TABLE_PREFIX + "/id1/partition1/file1"; - private static final String OBJECT_KEY2 = DB_AND_TABLE_PREFIX + "/id1/partition1/file2"; - private static final String OBJECT_KEY_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; - private static final String ABSOLUTE_PATH = "s3://" + BUCKET + "/" + OBJECT_KEY_ROOT; - - private static final String OBJECT_KEY_OTHER = DB_AND_TABLE_PREFIX + "/id1/partition10/file1"; - private static final String OBJECT_KEY_OTHER_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; - - private static final String SPRING_PROFILES_ACTIVE = "test"; - private static final String SCHEDULER_DELAY_MS = "5000"; - private static final String DRY_RUN_ENABLED = "false"; - private static final String CONTENT = "Content"; - private static final String HEALTHCHECK_URI = "http://localhost:8008/actuator/health"; - private static final String PROMETHEUS_URI = "http://localhost:8008/actuator/prometheus"; + protected static final int TIMEOUT = 30; + protected static final String SPRING_PROFILES_ACTIVE_PROPERTY = "spring.profiles.active"; + protected static final String SCHEDULER_DELAY_MS_PROPERTY = "properties.scheduler-delay-ms"; + protected static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; + protected static final String AWS_S3_ENDPOINT_PROPERTY = "aws.s3.endpoint"; + + protected static final String BUCKET = "test-path-bucket"; + protected static final String DB_AND_TABLE_PREFIX = DATABASE_NAME_VALUE + "/" + TABLE_NAME_VALUE; + protected static final String OBJECT_KEY_ROOT = DB_AND_TABLE_PREFIX + "/id1/partition1"; + protected static final String OBJECT_KEY1 = DB_AND_TABLE_PREFIX + "/id1/partition1/file1"; + protected static final String OBJECT_KEY2 = DB_AND_TABLE_PREFIX + "/id1/partition1/file2"; + protected static final String OBJECT_KEY_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition1_$folder$"; + protected static final String ABSOLUTE_PATH = "s3://" + BUCKET + "/" + OBJECT_KEY_ROOT; + + protected static final String OBJECT_KEY_OTHER = DB_AND_TABLE_PREFIX + "/id1/partition10/file1"; + protected static final String OBJECT_KEY_OTHER_SENTINEL = DB_AND_TABLE_PREFIX + "/id1/partition10_$folder$"; + + protected static final String SPRING_PROFILES_ACTIVE = "test"; + protected static final String SCHEDULER_DELAY_MS = "5000"; + protected static final String DRY_RUN_ENABLED = "false"; + protected static final String CONTENT = "Content"; + protected static final String HEALTHCHECK_URI = "http://localhost:8008/actuator/health"; + protected static final String PROMETHEUS_URI = "http://localhost:8008/actuator/prometheus"; @Container - private static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); - private static AmazonS3 amazonS3; + protected static final LocalStackContainer S3_CONTAINER = ContainerTestUtils.awsContainer(S3); + protected static AmazonS3 amazonS3; - private final ExecutorService executorService = Executors.newFixedThreadPool(1); + protected final ExecutorService executorService = Executors.newFixedThreadPool(1); @BeforeAll public static void init() { diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 9d3678d1..f2f36253 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2023 Expedia, Inc. + * Copyright (C) 2019-2024 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -70,17 +70,17 @@ @Testcontainers public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase { - private static final int TIMEOUT = 5; - private static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; + protected static final int TIMEOUT = 5; + protected static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; - private static final String QUEUE = "apiary-receiver-queue"; - private static final String SCHEDULED_ORPHANED_METRIC = "paths-scheduled"; - private static final String HEALTHCHECK_URI = "http://localhost:8080/actuator/health"; - private static final String PROMETHEUS_URI = "http://localhost:8080/actuator/prometheus"; + protected static final String QUEUE = "apiary-receiver-queue"; + protected static final String SCHEDULED_ORPHANED_METRIC = "paths-scheduled"; + protected static final String HEALTHCHECK_URI = "http://localhost:8080/actuator/health"; + protected static final String PROMETHEUS_URI = "http://localhost:8080/actuator/prometheus"; @Container - private static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); - private static AmazonSQS amazonSQS; + protected static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); + protected static AmazonSQS amazonSQS; @BeforeAll public static void init() { @@ -229,11 +229,11 @@ public void prometheus() { await().atMost(30, TimeUnit.SECONDS).until(() -> client.execute(request).getStatusLine().getStatusCode() == 200); } - private SendMessageRequest sendMessageRequest(String payload) { + protected SendMessageRequest sendMessageRequest(String payload) { return new SendMessageRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE), payload); } - private void assertUnreferencedPath(HousekeepingPath actual, String expectedPath) { + protected void assertUnreferencedPath(HousekeepingPath actual, String expectedPath) { assertHousekeepingEntity(actual, expectedPath); assertMetrics(); }