From a681efb4c04581c9036e0cc2a1b821148fc5e1f0 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Wed, 21 Dec 2022 17:33:50 +0100 Subject: [PATCH] Reduce Iceberg metrics reporting verbosity in logs Iceberg's new feature, metrics reporting, by default logs quite long log lines at INFO level for every table scanned in the query. This is quite verbose, potentially affecting log rotation on production installations and affecting Trino CI Github Actions logs. Move the metrics reporting to DEBUG log level, at least for table instances instantiated directly in Trino code. --- .../io/trino/plugin/iceberg/IcebergUtil.java | 5 ++- .../plugin/iceberg/TrinoMetricsReporter.java | 42 +++++++++++++++++++ .../catalog/glue/TrinoGlueCatalog.java | 3 +- 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TrinoMetricsReporter.java diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java index e729e12c299f..8ed7c89650fc 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergUtil.java @@ -105,6 +105,7 @@ import static io.trino.plugin.iceberg.IcebergTableProperties.getTableLocation; import static io.trino.plugin.iceberg.PartitionFields.parsePartitionFields; import static io.trino.plugin.iceberg.PartitionFields.toPartitionFields; +import static io.trino.plugin.iceberg.TrinoMetricsReporter.TRINO_METRICS_REPORTER; import static io.trino.plugin.iceberg.TypeConverter.toIcebergType; import static io.trino.plugin.iceberg.TypeConverter.toTrinoType; import static io.trino.plugin.iceberg.util.Timestamps.timestampTzFromMicros; @@ -172,7 +173,7 @@ public static Table loadIcebergTable(TrinoCatalog catalog, IcebergTableOperation table.getTableName(), Optional.empty(), Optional.empty()); - return new BaseTable(operations, quotedTableName(table)); + return new BaseTable(operations, quotedTableName(table), TRINO_METRICS_REPORTER); } public static Table getIcebergTableWithMetadata( @@ -190,7 +191,7 @@ public static Table getIcebergTableWithMetadata( Optional.empty(), Optional.empty()); operations.initializeFromMetadata(tableMetadata); - return new BaseTable(operations, quotedTableName(table)); + return new BaseTable(operations, quotedTableName(table), TRINO_METRICS_REPORTER); } public static Map getIcebergTableProperties(Table icebergTable) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TrinoMetricsReporter.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TrinoMetricsReporter.java new file mode 100644 index 000000000000..073c9b899f4b --- /dev/null +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TrinoMetricsReporter.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg; + +import io.airlift.log.Logger; +import org.apache.iceberg.metrics.LoggingMetricsReporter; +import org.apache.iceberg.metrics.MetricsReport; +import org.apache.iceberg.metrics.MetricsReporter; + +/** + * Trino default {@link MetricsReporter} implementation. + *

+ * Similar to Iceberg's default {@link MetricsReporter} implementation, + * the {@link LoggingMetricsReporter}, but does not log at {@code INFO} level, + * just eliminating log file size impact. + * + * @see LoggingMetricsReporter + */ +public enum TrinoMetricsReporter + implements MetricsReporter +{ + TRINO_METRICS_REPORTER; + + private static final Logger log = Logger.get(TrinoMetricsReporter.class); + + @Override + public void report(MetricsReport report) + { + log.debug("Received metrics report: %s", report); + } +} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java index b023a42b3b44..1af83791cca2 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java @@ -101,6 +101,7 @@ import static io.trino.plugin.iceberg.IcebergUtil.getIcebergTableWithMetadata; import static io.trino.plugin.iceberg.IcebergUtil.quotedTableName; import static io.trino.plugin.iceberg.IcebergUtil.validateTableCanBeDropped; +import static io.trino.plugin.iceberg.TrinoMetricsReporter.TRINO_METRICS_REPORTER; import static io.trino.plugin.iceberg.catalog.glue.GlueIcebergUtil.getMaterializedViewTableInput; import static io.trino.plugin.iceberg.catalog.glue.GlueIcebergUtil.getTableInput; import static io.trino.plugin.iceberg.catalog.glue.GlueIcebergUtil.getViewTableInput; @@ -330,7 +331,7 @@ public Table loadTable(ConnectorSession session, SchemaTableName table) table.getTableName(), Optional.empty(), Optional.empty()); - return new BaseTable(operations, quotedTableName(table)).operations().current(); + return new BaseTable(operations, quotedTableName(table), TRINO_METRICS_REPORTER).operations().current(); }); return getIcebergTableWithMetadata(