HDDS-8971. Example integration with Iceberg, Spark and Trino

hadoop-ozone/dist/src/main/compose/ozone/iceberg.yaml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+services:
+  spark-iceberg:
+    image: tabulario/spark-iceberg:3.3.2_1.2.1
+    depends_on:
+      - iceberg-rest
+      - s3g
+    volumes:
+      - ./spark.conf:/opt/spark/conf
+    ports:
+      - 8888:8888
+      - 8080:8080
+      - 10000:10000
+      - 10001:10001
+    environment:
+      - AWS_ACCESS_KEY_ID=any
+      - AWS_SECRET_ACCESS_KEY=any
+      - AWS_REGION=us-east-1
+  iceberg-rest:
+    image: tabulario/iceberg-rest:0.5.0
+    ports:
+      - 8181:8181
+    environment:
+      - AWS_ACCESS_KEY_ID=any
+      - AWS_SECRET_ACCESS_KEY=any
+      - AWS_REGION=us-east-1
+      - CATALOG_WAREHOUSE=s3://warehouse/
+      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+      - CATALOG_S3_ENDPOINT=http://s3g:9878/
+      - CATALOG_S3_PATH__STYLE__ACCESS=true

hadoop-ozone/dist/src/main/compose/ozone/spark.conf/spark-defaults.conf

@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spark.sql.extensions                        org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
+spark.sql.catalog.demo                      org.apache.iceberg.spark.SparkCatalog
+spark.sql.catalog.demo.catalog-impl         org.apache.iceberg.rest.RESTCatalog
+spark.sql.catalog.demo.uri                  http://iceberg-rest:8181
+spark.sql.catalog.demo.io-impl              org.apache.iceberg.aws.s3.S3FileIO
+spark.sql.catalog.demo.warehouse            s3a://warehouse/
+spark.sql.catalog.demo.s3.endpoint          http://s3g:9878/
+spark.sql.catalog.demo.s3.path-style-access true
+spark.sql.defaultCatalog                    demo
+spark.eventLog.enabled                      true
+spark.eventLog.dir                          /home/iceberg/spark-events
+spark.history.fs.logDirectory               /home/iceberg/spark-events
+spark.sql.catalogImplementation             in-memory

hadoop-ozone/dist/src/main/compose/ozone/test-iceberg.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:integration
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=false
+export OZONE_REPLICATION_FACTOR=3
+export COMPOSE_FILE=docker-compose.yaml:iceberg.yaml:trino.yaml
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env 3
+
+export BUCKET=warehouse
+
+execute_command_in_container s3g ozone sh bucket create --layout OBJECT_STORE /s3v/${BUCKET}
+
+execute_command_in_container spark-iceberg spark-shell <<EOF
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.Row
+val schema = StructType( Array(
+    StructField("vendor_id", LongType,true),
+    StructField("trip_id", LongType,true),
+    StructField("trip_distance", FloatType,true),
+    StructField("fare_amount", DoubleType,true),
+    StructField("store_and_fwd_flag", StringType,true)
+))
+spark.createDataFrame(spark.sparkContext.emptyRDD[Row],schema)
+    .writeTo("demo.nyc.taxis").create()
+
+val data = Seq(
+    Row(1: Long, 1000371: Long, 1.8f: Float, 15.32: Double, "N": String),
+    Row(2: Long, 1000372: Long, 2.5f: Float, 22.15: Double, "N": String),
+    Row(2: Long, 1000373: Long, 0.9f: Float, 9.01: Double, "N": String),
+    Row(1: Long, 1000374: Long, 8.4f: Float, 42.13: Double, "Y": String)
+)
+spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
+    .writeTo("demo.nyc.taxis").append()
+
+spark.table("demo.nyc.taxis").show()
+EOF
+
+execute_command_in_container trino trino <<EOF
+DESCRIBE iceberg.nyc.taxis;
+INSERT INTO iceberg.nyc.taxis VALUES (2, 1000375, 7.2, 555, 'N');
+SELECT * FROM iceberg.nyc.taxis;
+EOF
+
+execute_robot_test scm -v BUCKET:${BUCKET} integration/iceberg.robot

hadoop-ozone/dist/src/main/compose/ozone/trino.catalog/iceberg.properties

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+connector.name=iceberg
+iceberg.catalog.type=rest
+iceberg.rest-catalog.uri=http://iceberg-rest:8181
+iceberg.rest-catalog.warehouse=s3://warehouse/
+hive.s3.aws-access-key=any
+hive.s3.aws-secret-key=any
+hive.s3.endpoint=http://s3g:9878/
+hive.s3.path-style-access=true
+hive.s3.ssl.enabled=false

hadoop-ozone/dist/src/main/compose/ozone/trino.yaml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.4"
+
+services:
+  trino:
+    image: trinodb/trino:420
+    depends_on:
+      - spark-iceberg
+    volumes:
+      - ./trino.catalog:/etc/trino/catalog

hadoop-ozone/dist/src/main/smoketest/integration/iceberg.robot

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Test for Iceberg integration
+Resource            ../commonlib.robot
+Test Timeout        5 minutes
+
+*** Variables ***
+${BUCKET}    warehouse
+
+*** Test Cases ***
+Verify Warehouse contents
+    Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
+    ${output} =    Execute    ozone sh key list /s3v/${BUCKET} | jq -r '.[].name' | sort
+    Should Contain    ${output}    nyc/taxis/data/
+    Should Contain    ${output}    nyc/taxis/metadata/