Skip to content

Commit

Permalink
Add a product test that loads all connectors
Browse files Browse the repository at this point in the history
  • Loading branch information
nineinchnick authored and electrum committed Apr 12, 2022
1 parent 4897052 commit 2267c61
Show file tree
Hide file tree
Showing 35 changed files with 237 additions and 0 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,10 @@ jobs:
- config: hdp3
suite: suite-2
jdk: 17
# this suite is not meant to be run with different configs
- config: default
suite: suite-all
jdk: 11
# PT Launcher's timeout defaults to 2h, add some margin
timeout-minutes: 130
needs: build-pt
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.tests.product.launcher.env.environment;

import io.trino.tests.product.launcher.docker.DockerFiles;
import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider;
import io.trino.tests.product.launcher.env.Environment;
import io.trino.tests.product.launcher.env.EnvironmentProvider;
import io.trino.tests.product.launcher.env.common.StandardMultinode;
import io.trino.tests.product.launcher.env.common.TestsEnvironment;

import javax.inject.Inject;

import java.util.List;

import static io.trino.tests.product.launcher.env.EnvironmentContainers.isPrestoContainer;
import static io.trino.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_ETC;
import static org.testcontainers.utility.MountableFile.forHostPath;

@TestsEnvironment
public final class EnvMultinodeAllConnectors
extends EnvironmentProvider
{
private final ResourceProvider configDir;

@Inject
public EnvMultinodeAllConnectors(StandardMultinode standardMultinode, DockerFiles dockerFiles)
{
super(standardMultinode);
this.configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/multinode-all/");
}

@Override
public void extendEnvironment(Environment.Builder builder)
{
// blackhole, jmx, tpch are already configured in Standard base env
List.of(
// TODO accumulo needs to connect to ZooKeeper, it won't start otherwise
//"accumulo",
"atop",
"bigquery",
"cassandra",
"clickhouse",
"druid",
"delta-lake",
"elasticsearch",
"gsheets",
"hive",
"iceberg",
"kafka",
"kinesis",
"kudu",
"localfile",
"memory",
"memsql",
"mongodb",
"mysql",
"oracle",
"phoenix",
"phoenix5",
"pinot",
"postgresql",
"prometheus",
"raptor-legacy",
"redis",
"redshift",
"sqlserver",
"trino-thrift",
"tpcds")
.forEach(connector -> builder.addConnector(
connector,
forHostPath(configDir.getPath(connector + ".properties"))));
builder.configureContainers(container -> {
if (isPrestoContainer(container.getLogicalName())) {
container.withCopyFileToContainer(
forHostPath(configDir.getPath("google-sheets-auth.json")),
CONTAINER_PRESTO_ETC + "/catalog/google-sheets-auth.json");
}
});
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.tests.product.launcher.suite.suites;

import com.google.common.collect.ImmutableList;
import io.trino.tests.product.launcher.env.EnvironmentConfig;
import io.trino.tests.product.launcher.env.environment.EnvMultinodeAllConnectors;
import io.trino.tests.product.launcher.suite.Suite;
import io.trino.tests.product.launcher.suite.SuiteTestRun;

import java.util.List;

import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment;

public class SuiteAll
extends Suite
{
@Override
public List<SuiteTestRun> getTestRuns(EnvironmentConfig config)
{
return ImmutableList.of(
testOnEnvironment(EnvMultinodeAllConnectors.class)
.withTests("TestConfiguredFeatures.selectConfiguredConnectors")
.build());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
connector.name=accumulo
accumulo.instance=host1.invalid
accumulo.zookeepers=host2.invalid
accumulo.username=username
accumulo.password=password
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=atop
atop.executable-path=/usr/bin/true
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
connector.name=bigquery
bigquery.project-id=xxx
bigquery.credentials-key=ewogICAgInR5cGUiOiAic2VydmljZV9hY2NvdW50IiwKICAgICJwcm9qZWN0X2lkIjogInByZXN0b3Rlc3QiLAogICAgInByaXZhdGVfa2V5X2lkIjogIngiLAogICAgInByaXZhdGVfa2V5IjogIi0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLVxuTUlJRXZRSUJBREFOQmdrcWhraUc5dzBCQVFFRkFBU0NCS2N3Z2dTakFnRUFBb0lCQVFDM3E0NkcrdlRtdllmR1xuUEVCcUZST2MwWllEUFE4Z1VRYlEvaWRiYXBQc0s3TUxIZEx1RUdzQkF4SjMyYkdKV2FXV1pKMlZvKzA2Y3E4UlxuM1VZdVJ5RDBvNVk5OTV3d0t5YUVMdjVHTHFxMlpkQ0U2MGNqbE8yeXM4dUg4MVJMQStOME9zUzI0WXAvTmw1V1xuMHl4bkVuaW5VSW5Hb1VteVJ0K1V1aTIxNTQ3WTZFeDFKMGdqVndoNWtBNTJBcG0xVzVjZ3JKUWgwMlZTNUZERlxudEtnWjlKNFUyZVQvM2RNUkFlN0dLaWtseGpMTktjSzJ6T3JuYVpzb0pBTnNrZ0xMdjhPaDJEdlpQbWd1dmtxL1xuUU4yRVRCSXVLRUxCbEN4ZHZkRVp5L2pPcUVmdW1xcWI4VTVqVk4vdit6Q0pZVHREcVRGVWxLRVZDbEVNV3BCUFxuR3dIUzg5MlRBZ01CQUFFQ2dnRUFVOWxuTE9vZXFjUTIydUlneWcwck1mbGdrY1ByUnVhV3hReHlMVUsvbXg3c1xuRXhRZmVuMVdURlQ1dG10VXFJNmJrTWdJUlF0Y1BzV2lkUFplbHJ2MEtKc1IrT0kwbEt6dVhZUVNvem1reDdZOVxuZHFEdWppanNSeHZidkFuekhuZjgrOC9raEZUODVFeU96dmFERzk4TDQ5NVp0NnRrT0pZd2RmWjA3Y2x6cGtQSVxudVNDMGRTMldFckZnT0JiK3BJZGFwU3dSN1gxOTlROGNsenhjYlVUUUJJaGJDMnFhUmUvelFBdHNIS3ArMHRRVFxuWEI0Z3A1bitXc0pGM2lmTlYwdkZ0VWRRUlNCNFBmRzExRW1lczBQTFpxV1ZYb2xGdWpVQW8xS0o3dXFWTVpoUlxuQTF6VEpEbzNaaklHUllvbmRHQWRHR1hrMU1rd1JCcGoxR0FRb08xSm9RS0JnUURjUFhLcjF0MlBVbnJMelN1UFxuNVM2ek9WMUVzNkpJbmpVaGtXNFFhcTQ4RFRYNEg2bkdTYkdyYW5tbVpyQXlWNytEeXZPWGVzaC9ITzJROWtlaFxuRlczaUVtQzBCZE9FeWVuRUJUNThidHR5VzlMVUtBVjhjendYTno4Q3lSQ0xGd292UDIzUjFkS3BZdGtsR0l6NVxuWjJaMEF1SEtzcWd2TC9Jeng2bU1QNU0rQXdLQmdRRFZmZ2ttMUJPVzhad3JvNFFjbE43bnlKN01lQ3BCTFZycVxuUU9OcThqeE5EaGpsT0VDSElZZmFTUUZLYXkwa2pBTndTQjRMYURuTXJTbmRJYWxqV3F1LzBtdThMLzNwQzg4MlxuOUhpNU1Mc1Jjb0trNDY5UnRLRVIxWEwvcE5sb1NTd2dkZWJtUWk3clhNUzFCQks4aFZ0UFFObmV0RE1sLy9JTFxuS3YzbmtycFZNUUtCZ0dnaUdiVU1PK2dIUEk1ZUxRbTFlRFkvbWt6Z2pvdTlXaXZNQW5sNnAzVTNYZHc2eEdBL1xuK2VTdHpHVVVTcDBUQmpkL1gxdXhMMW1DeVFUd25YK1pqVUlHSkhrYUJCL1dCRlN0a2hUdHFZN1J3Y2FVUWJ2TlxuRkkxNWpxNTNlUDM2MzlMbEw3eTJXQXZFOUJ6cEZjYmEwQU5zVld3c3V2N01zYjB2MjRlM2k1d1hBb0dCQUswWlxuL2kyZmN5ckdTRXdSendLbHFuN2c2ZkQ3MWJiM0lXb2lwc0tHR21LWDlaT1ZvcXh1Z1lwNSt6UHQ1ckpsWER4a1xuSFFnK3YrNjIwT1RkY0V5QXJoVmdkYjRtWTRmYjdXMnZsMXNBcWcwaGZkQllWRVM1WW9mbE85TVFSTDhMNVYyRVxuZTIxamFFdXA4a3liT3Qza2V2NnRwSG13UG5Dbk1BZmlHZkR6eFdWaEFvR0FYa1k5bjNsSDFISDJBUDhzMkNnNVxuN3o3NVhLYWtxWE9CMkNhTWFuOWxJd0FCVzhSam1IKzRiU2VVQ0kwM1hRRExrY1R3T0N3QStrL3FvZldBeW1ldVxudzU4Vzh5cGlWVGpDVDErUzh5VjhYL0htTERVa3VsTnUvY2psYlJPdnJmSlRIL2pNbVhhTEQxeVZlYXlxOFlGZFxubnl6SmpiR1BwdGsvYVRTYk5rQmpvdWM9XG4tLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tXG4iLAogICAgImNsaWVudF9lbWFpbCI6ICJ4IiwKICAgICJjbGllbnRfaWQiOiAieCIsCiAgICAiYXV0aF91cmkiOiAiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tL28vb2F1dGgyL2F1dGgiLAogICAgInRva2VuX3VyaSI6ICJodHRwczovL29hdXRoMi5nb29nbGVhcGlzLmNvbS90b2tlbiIsCiAgICAiYXV0aF9wcm92aWRlcl94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL29hdXRoMi92MS9jZXJ0cyIsCiAgICAiY2xpZW50X3g1MDlfY2VydF91cmwiOiAiaHR0cHM6Ly93d3cuZ29vZ2xlYXBpcy5jb20vcm9ib3QvdjEvbWV0YWRhdGEveDUwOS9wcmVzdG90ZXN0JTQwcHJlc3RvdGVzdC5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIKfQo=
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=cassandra
cassandra.contact-points=host1.invalid,host2.invalid
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
connector.name=clickhouse
connection-url=jdbc:clickhouse://host1.invalid:8123/
connection-user=exampleuser
connection-password=examplepassword
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=delta-lake
hive.metastore.uri=thrift://host1.invalid:9083
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=druid
connection-url=jdbc:avatica:remote:url=http://host1.invalid:8082/druid/v2/sql/avatica/
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
connector.name=elasticsearch
elasticsearch.host=host1.invalid
elasticsearch.port=9200
elasticsearch.default-schema-name=default
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"type": "service_account",
"project_id": "prestotest",
"private_key_id": "x",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3q46G+vTmvYfG\nPEBqFROc0ZYDPQ8gUQbQ/idbapPsK7MLHdLuEGsBAxJ32bGJWaWWZJ2Vo+06cq8R\n3UYuRyD0o5Y995wwKyaELv5GLqq2ZdCE60cjlO2ys8uH81RLA+N0OsS24Yp/Nl5W\n0yxnEninUInGoUmyRt+Uui21547Y6Ex1J0gjVwh5kA52Apm1W5cgrJQh02VS5FDF\ntKgZ9J4U2eT/3dMRAe7GKiklxjLNKcK2zOrnaZsoJANskgLLv8Oh2DvZPmguvkq/\nQN2ETBIuKELBlCxdvdEZy/jOqEfumqqb8U5jVN/v+zCJYTtDqTFUlKEVClEMWpBP\nGwHS892TAgMBAAECggEAU9lnLOoeqcQ22uIgyg0rMflgkcPrRuaWxQxyLUK/mx7s\nExQfen1WTFT5tmtUqI6bkMgIRQtcPsWidPZelrv0KJsR+OI0lKzuXYQSozmkx7Y9\ndqDujijsRxvbvAnzHnf8+8/khFT85EyOzvaDG98L495Zt6tkOJYwdfZ07clzpkPI\nuSC0dS2WErFgOBb+pIdapSwR7X199Q8clzxcbUTQBIhbC2qaRe/zQAtsHKp+0tQT\nXB4gp5n+WsJF3ifNV0vFtUdQRSB4PfG11Emes0PLZqWVXolFujUAo1KJ7uqVMZhR\nA1zTJDo3ZjIGRYondGAdGGXk1MkwRBpj1GAQoO1JoQKBgQDcPXKr1t2PUnrLzSuP\n5S6zOV1Es6JInjUhkW4Qaq48DTX4H6nGSbGranmmZrAyV7+DyvOXesh/HO2Q9keh\nFW3iEmC0BdOEyenEBT58bttyW9LUKAV8czwXNz8CyRCLFwovP23R1dKpYtklGIz5\nZ2Z0AuHKsqgvL/Izx6mMP5M+AwKBgQDVfgkm1BOW8Zwro4QclN7nyJ7MeCpBLVrq\nQONq8jxNDhjlOECHIYfaSQFKay0kjANwSB4LaDnMrSndIaljWqu/0mu8L/3pC882\n9Hi5MLsRcoKk469RtKER1XL/pNloSSwgdebmQi7rXMS1BBK8hVtPQNnetDMl//IL\nKv3nkrpVMQKBgGgiGbUMO+gHPI5eLQm1eDY/mkzgjou9WivMAnl6p3U3Xdw6xGA/\n+eStzGUUSp0TBjd/X1uxL1mCyQTwnX+ZjUIGJHkaBB/WBFStkhTtqY7RwcaUQbvN\nFI15jq53eP3639LlL7y2WAvE9BzpFcba0ANsVWwsuv7Msb0v24e3i5wXAoGBAK0Z\n/i2fcyrGSEwRzwKlqn7g6fD71bb3IWoipsKGGmKX9ZOVoqxugYp5+zPt5rJlXDxk\nHQg+v+620OTdcEyArhVgdb4mY4fb7W2vl1sAqg0hfdBYVES5YoflO9MQRL8L5V2E\ne21jaEup8kybOt3kev6tpHmwPnCnMAfiGfDzxWVhAoGAXkY9n3lH1HH2AP8s2Cg5\n7z75XKakqXOB2CaMan9lIwABW8RjmH+4bSeUCI03XQDLkcTwOCwA+k/qofWAymeu\nw58W8ypiVTjCT1+S8yV8X/HmLDUkulNu/cjlbROvrfJTH/jMmXaLD1yVeayq8YFd\nnyzJjbGPptk/aTSbNkBjouc=\n-----END PRIVATE KEY-----\n",
"client_email": "x",
"client_id": "x",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/prestotest%40prestotest.iam.gserviceaccount.com"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
connector.name=gsheets
credentials-path=/docker/presto-product-tests/conf/presto/etc/catalog/google-sheets-auth.json
metadata-sheet-id=exampleId
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=hive
hive.metastore.uri=thrift://host1.invalid:9083
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=iceberg
hive.metastore.uri=thrift://host1.invalid:9083
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
connector.name=kafka
kafka.table-names=table1,table2
kafka.nodes=host1.invalid:9092,host2.invalid:9092
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
connector.name=kinesis
kinesis.access-key=XXXXXX
kinesis.secret-key=XXXXXX
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=kudu
kudu.client.master-addresses=host1.invalid
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=localfile
trino-logs.http-request-log.location=/dev/null
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=memory
memory.max-data-per-node=128MB
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
connector.name=memsql
connection-url=jdbc:mariadb://host1.invalid:3306
connection-user=root
connection-password=secret
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=mongodb
mongodb.connection-url=mongodb://user:pass@host1.invalid:27017/
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
connector.name=mysql
connection-url=jdbc:mysql://host1.invalid:3306
connection-user=root
connection-password=secret
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
connector.name=oracle
connection-url=jdbc:oracle:thin:@//host1.invalid:1521/ORCLCDB
connection-user=root
connection-password=secret
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=phoenix
phoenix.connection-url=jdbc:phoenix:host1.invalid:2181:/hbase
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=phoenix5
phoenix.connection-url=jdbc:phoenix:host1.invalid:2181:/hbase
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=pinot
pinot.controller-urls=host1.invalid:8098,host2.invalid:8098
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
connector.name=postgresql
connection-url=jdbc:postgresql://host1.invalid:5432/database
connection-user=root
connection-password=secret
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
connector.name=prometheus
prometheus.uri=http://host1.invalid:9090
prometheus.query.chunk.size.duration=1d
prometheus.max.query.range.duration=21d
prometheus.cache.ttl=30s
prometheus.bearer.token.file=/path/to/bearer/token/file
prometheus.read-timeout=10s
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
connector.name=raptor-legacy
metadata.db.type=h2
metadata.db.filename=mem:raptor;DB_CLOSE_DELAY=-1
storage.data-directory=var/data
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
connector.name=redis
redis.table-names=schema1.table1,schema1.table2
redis.nodes=host1.invalid:6379
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
connector.name=redshift
connection-url=jdbc:redshift://host1.invalid:5439/database
connection-user=root
connection-password=secret
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
connector.name=sqlserver
connection-url=jdbc:sqlserver://host1.invalid:1433;database=invalid
connection-user=root
connection-password=secret
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
connector.name=tpcds
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
connector.name=trino-thrift
trino.thrift.client.addresses=host1.invalid:7777,host2.invalid:7779

0 comments on commit 2267c61

Please sign in to comment.