Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test tracing cleanup #20781

Merged
merged 2 commits into from
Feb 21, 2024
Merged

Test tracing cleanup #20781

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b97bae7
Protect against concurrent QueryRunner.getSpans usage
findepi Feb 20, 2024
563d6bd
Remove TrackingFileSystemFactory vestigials
findepi Feb 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions core/trino-main/src/main/java/io/trino/testing/StandaloneQueryRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,13 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;

import static com.google.common.base.Preconditions.checkState;
import static io.trino.execution.querystats.PlanOptimizersStatsCollector.createPlanOptimizersStatsCollector;
import static java.util.Objects.requireNonNull;

Expand All @@ -65,6 +67,8 @@ public final class StandaloneQueryRunner
private final InMemorySpanExporter spanExporter = InMemorySpanExporter.create();

private final ReadWriteLock lock = new ReentrantReadWriteLock();
private final AtomicInteger concurrentQueries = new AtomicInteger();
private boolean spansValid = true;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What guards accesses to this non-volatile field?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good point.
i can make it volatile, but it doesn't make this bullet proof anyway.
so conceptually this field is used normally in single-threaded scenario only (the only scenario where it's legit to run query and then ask for spans).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#20796


public StandaloneQueryRunner(Session defaultSession)
{
Expand Down Expand Up @@ -94,6 +98,7 @@ public StandaloneQueryRunner(Session defaultSession, Consumer<TestingTrinoServer
@Override
public List<SpanData> getSpans()
{
checkState(spansValid, "No valid spans, queries were executing concurrently");
return spanExporter.getFinishedSpanItems();
}

Expand All @@ -114,8 +119,14 @@ private DirectTrinoClient.Result executeInternal(Session session, @Language("SQL
{
lock.readLock().lock();
try {
spanExporter.reset();
return trinoClient.execute(session, sql);
spansValid = concurrentQueries.incrementAndGet() == 1;
try {
spanExporter.reset();
return trinoClient.execute(session, sql);
}
finally {
concurrentQueries.decrementAndGet();
}
}
catch (Throwable e) {
e.addSuppressed(new Exception("SQL: " + sql));
Expand All @@ -132,9 +143,15 @@ public Plan createPlan(Session session, String sql)
// session must be in a transaction registered with the transaction manager in this query runner
getTransactionManager().getTransactionInfo(session.getRequiredTransactionId());

spanExporter.reset();
Statement statement = server.getInstance(Key.get(SqlParser.class)).createStatement(sql);
return server.getQueryExplainer().getLogicalPlan(session, statement, ImmutableList.of(), WarningCollector.NOOP, createPlanOptimizersStatsCollector());
spansValid = concurrentQueries.incrementAndGet() == 1;
try {
spanExporter.reset();
Statement statement = server.getInstance(Key.get(SqlParser.class)).createStatement(sql);
return server.getQueryExplainer().getLogicalPlan(session, statement, ImmutableList.of(), WarningCollector.NOOP, createPlanOptimizersStatsCollector());
}
finally {
concurrentQueries.decrementAndGet();
}
}

@Override
Expand Down
8 changes: 4 additions & 4 deletions ...lesystem/src/test/java/io/trino/filesystem/cache/TestCacheFileSystemAccessOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,21 +45,21 @@
@TestInstance(Lifecycle.PER_CLASS)
public class TestCacheFileSystemAccessOperations
{
private TrinoFileSystemFactory trackingFileSystemFactory;
private TrinoFileSystemFactory tracingFileSystemFactory;
private CacheFileSystem fileSystem;
private final TestingTelemetry telemetry = TestingTelemetry.create("cache-file-system");

@BeforeAll
void setUp()
{
trackingFileSystemFactory = new TracingFileSystemFactory(telemetry.getTracer(), new MemoryFileSystemFactory());
fileSystem = new CacheFileSystem(trackingFileSystemFactory.create(TestingSession.SESSION), new TestingMemoryFileSystemCache(), new DefaultCacheKeyProvider());
tracingFileSystemFactory = new TracingFileSystemFactory(telemetry.getTracer(), new MemoryFileSystemFactory());
fileSystem = new CacheFileSystem(tracingFileSystemFactory.create(TestingSession.SESSION), new TestingMemoryFileSystemCache(), new DefaultCacheKeyProvider());
}

@AfterAll
void tearDown()
{
trackingFileSystemFactory = null;
tracingFileSystemFactory = null;
fileSystem = null;
}

Expand Down
2 changes: 1 addition & 1 deletion ...trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toCollection;

// single-threaded AccessTrackingFileSystemFactory is shared mutable state
// single-threaded as DistributedQueryRunner.spans is shared mutable state
@Execution(ExecutionMode.SAME_THREAD)
public class TestDeltaLakeFileOperations
extends AbstractTestQueryFramework
Expand Down
2 changes: 1 addition & 1 deletion ...in/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestTransactionLogAccess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;

@Execution(SAME_THREAD) // e.g. TrackingFileSystemFactory is shared mutable state
@Execution(SAME_THREAD) // e.g. DistributedQueryRunner.spans is shared mutable state
public class TestTransactionLogAccess
{
private static final Set<String> EXPECTED_ADD_FILE_PATHS = ImmutableSet.of(
Expand Down
2 changes: 1 addition & 1 deletion ...-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergAlluxioCacheFileOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toCollection;

// single-threaded AccessTrackingFileSystemFactory is shared mutable state
// single-threaded as DistributedQueryRunner.spans is shared mutable state
@Execution(ExecutionMode.SAME_THREAD)
public class TestIcebergAlluxioCacheFileOperations
extends AbstractTestQueryFramework
Expand Down
33 changes: 24 additions & 9 deletions testing/trino-testing/src/main/java/io/trino/testing/DistributedQueryRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ public class DistributedQueryRunner
private TestingTrinoClient trinoClient;

private final ReadWriteLock lock = new ReentrantReadWriteLock();
private final AtomicInteger concurrentQueries = new AtomicInteger();
private boolean spansValid = true;

private boolean closed;

Expand Down Expand Up @@ -337,6 +339,7 @@ public TestingTrinoClient getClient()
@Override
public List<SpanData> getSpans()
{
checkState(spansValid, "No valid spans, queries were executing concurrently");
return spanExporter.getFinishedSpanItems();
}

Expand Down Expand Up @@ -492,8 +495,14 @@ private ResultWithQueryId<MaterializedResult> executeInternal(Session session, @
{
lock.readLock().lock();
try {
spanExporter.reset();
return trinoClient.execute(session, sql);
spansValid = concurrentQueries.incrementAndGet() == 1;
try {
spanExporter.reset();
return trinoClient.execute(session, sql);
}
finally {
concurrentQueries.decrementAndGet();
}
}
catch (Throwable e) {
e.addSuppressed(new Exception("SQL: " + sql));
Expand All @@ -510,13 +519,19 @@ public Plan createPlan(Session session, String sql)
// session must be in a transaction registered with the transaction manager in this query runner
getTransactionManager().getTransactionInfo(session.getRequiredTransactionId());

spanExporter.reset();
return coordinator.getQueryExplainer().getLogicalPlan(
session,
coordinator.getInstance(Key.get(SqlParser.class)).createStatement(sql),
ImmutableList.of(),
WarningCollector.NOOP,
createPlanOptimizersStatsCollector());
spansValid = concurrentQueries.incrementAndGet() == 1;
try {
spanExporter.reset();
return coordinator.getQueryExplainer().getLogicalPlan(
session,
coordinator.getInstance(Key.get(SqlParser.class)).createStatement(sql),
ImmutableList.of(),
WarningCollector.NOOP,
createPlanOptimizersStatsCollector());
}
finally {
concurrentQueries.decrementAndGet();
}
}

public Plan getQueryPlan(QueryId queryId)
Expand Down
Loading