diff --git a/hadoop-ozone/dist/src/main/compose/ozone/iceberg.yaml b/hadoop-ozone/dist/src/main/compose/ozone/iceberg.yaml new file mode 100644 index 00000000000..e0cac3da079 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/iceberg.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.4" + +services: + spark-iceberg: + image: tabulario/spark-iceberg:3.3.2_1.2.1 + depends_on: + - iceberg-rest + - s3g + volumes: + - ./spark.conf:/opt/spark/conf + ports: + - 8888:8888 + - 8080:8080 + - 10000:10000 + - 10001:10001 + environment: + - AWS_ACCESS_KEY_ID=any + - AWS_SECRET_ACCESS_KEY=any + - AWS_REGION=us-east-1 + iceberg-rest: + image: tabulario/iceberg-rest:0.5.0 + ports: + - 8181:8181 + environment: + - AWS_ACCESS_KEY_ID=any + - AWS_SECRET_ACCESS_KEY=any + - AWS_REGION=us-east-1 + - CATALOG_WAREHOUSE=s3://warehouse/ + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://s3g:9878/ + - CATALOG_S3_PATH__STYLE__ACCESS=true diff --git a/hadoop-ozone/dist/src/main/compose/ozone/spark.conf/spark-defaults.conf b/hadoop-ozone/dist/src/main/compose/ozone/spark.conf/spark-defaults.conf new file mode 100755 index 00000000000..ef31f67f252 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/spark.conf/spark-defaults.conf @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions +spark.sql.catalog.demo org.apache.iceberg.spark.SparkCatalog +spark.sql.catalog.demo.catalog-impl org.apache.iceberg.rest.RESTCatalog +spark.sql.catalog.demo.uri http://iceberg-rest:8181 +spark.sql.catalog.demo.io-impl org.apache.iceberg.aws.s3.S3FileIO +spark.sql.catalog.demo.warehouse s3a://warehouse/ +spark.sql.catalog.demo.s3.endpoint http://s3g:9878/ +spark.sql.catalog.demo.s3.path-style-access true +spark.sql.defaultCatalog demo +spark.eventLog.enabled true +spark.eventLog.dir /home/iceberg/spark-events +spark.history.fs.logDirectory /home/iceberg/spark-events +spark.sql.catalogImplementation in-memory diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test-iceberg.sh b/hadoop-ozone/dist/src/main/compose/ozone/test-iceberg.sh new file mode 100755 index 00000000000..999d2f962f3 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozone/test-iceberg.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#suite:integration + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +export SECURITY_ENABLED=false +export OZONE_REPLICATION_FACTOR=3 +export COMPOSE_FILE=docker-compose.yaml:iceberg.yaml:trino.yaml + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env 3 + +export BUCKET=warehouse + +execute_command_in_container s3g ozone sh bucket create --layout OBJECT_STORE /s3v/${BUCKET} + +execute_command_in_container spark-iceberg spark-shell <