HDDS-15325. [STS] Polaris Smoke Test

hadoop-ozone/dist/src/main/compose/common/init-kdc.sh

@@ -37,6 +37,9 @@ export_keytab testuser/om testuser
 export_keytab testuser/recon testuser
 export_keytab testuser/s3g testuser
 export_keytab testuser/scm testuser
+export_keytab svc-iceberg-rest-catalog/s3g svc-iceberg-rest-catalog
+export_keytab svc-iceberg-userA/s3g svc-iceberg-userA
+export_keytab svc-iceberg-userB/s3g svc-iceberg-userB
 
 export_keytab testuser2/dn testuser2
 export_keytab testuser2/httpfs testuser2

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config

@@ -103,6 +103,9 @@ OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
 OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
 OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
 
+# Enable S3 Gateway STS (AWS STS compatible) endpoint on s3g (http://s3g:9880/sts)
+OZONE-SITE.XML_ozone.s3g.sts.http.enabled=true
+
 OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
 OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
 OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-setup.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env sh
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+apk add --no-cache jq >/dev/null
+
+realm="${POLARIS_REALM:-POLARIS}"
+catalog_name="${POLARIS_CATALOG_NAME:-quickstart_catalog}"
+storage_location="${POLARIS_STORAGE_LOCATION:-s3://iceberg-obs/polaris-smoke}"
+s3_endpoint="${POLARIS_S3_ENDPOINT:-http://s3g:9878}"
+sts_endpoint="${POLARIS_STS_ENDPOINT:-http://s3g:9880/sts}"
+role_arn="${POLARIS_ROLE_ARN:-arn:aws:iam::123456789012:role/iceberg-data-all-access-obs}"
+
+if [ -z "${POLARIS_AWS_ACCESS_KEY_ID:-}" ] || [ -z "${POLARIS_AWS_SECRET_ACCESS_KEY:-}" ]; then
+  echo "POLARIS_AWS_ACCESS_KEY_ID and POLARIS_AWS_SECRET_ACCESS_KEY must be set"
+  exit 1
+fi
+
+echo "Waiting for S3 gateway at ${s3_endpoint}..."
+attempt=0
+while [ "${attempt}" -lt 30 ]; do
+  if curl --silent --show-error --include \
+      --user "${POLARIS_AWS_ACCESS_KEY_ID}:${POLARIS_AWS_SECRET_ACCESS_KEY}" \
+      --aws-sigv4 "aws:amz:us-west-2:s3" \
+      "${s3_endpoint}/" >/dev/null 2>&1; then
+    echo "${s3_endpoint} is available"
+    break
+  fi
+  attempt=$((attempt + 1))
+  sleep 2
+done
+if [ "${attempt}" -ge 30 ]; then
+  echo "Timed out waiting for S3 gateway at ${s3_endpoint}"
+  exit 1
+fi
+
+echo "Obtaining Polaris OAuth token..."
+token="$(
+  curl --fail-with-body --silent \
+    --user "${CLIENT_ID}:${CLIENT_SECRET}" \
+    -H "Polaris-Realm: ${realm}" \
+    -d grant_type=client_credentials \
+    -d scope=PRINCIPAL_ROLE:ALL \
+    "http://polaris:8181/api/catalog/v1/oauth/tokens" \
+    | jq -r .access_token
+)"
+if [ -z "${token}" ] || [ "${token}" = "null" ]; then
+  echo "Failed to obtain access token."
+  exit 1
+fi
+
+storage_config_info="$(
+  jq -n \
+    --arg endpoint "${s3_endpoint}" \
+    --arg endpointInternal "${s3_endpoint}" \
+    --arg stsEndpoint "${sts_endpoint}" \
+    --arg roleArn "${role_arn}" \
+    '{
+      storageType: "S3",
+      endpoint: $endpoint,
+      endpointInternal: $endpointInternal,
+      stsEndpoint: $stsEndpoint,
+      roleArn: $roleArn,
+      stsUnavailable: false,
+      pathStyleAccess: true,
+      region: "us-west-2"
+    }'
+)"
+
+payload="$(
+  jq -n \
+    --arg name "${catalog_name}" \
+    --arg location "${storage_location}" \
+    --argjson storageConfigInfo "${storage_config_info}" \
+    '{
+      catalog: {
+        name: $name,
+        type: "INTERNAL",
+        readOnly: false,
+        properties: {
+          "default-base-location": $location
+        },
+        storageConfigInfo: $storageConfigInfo
+      }
+    }'
+)"
+
+echo "Creating catalog ${catalog_name} in realm ${realm}..."
+curl --fail-with-body --silent \
+  -H "Authorization: Bearer ${token}" \
+  -H "Accept: application/json" \
+  -H "Content-Type: application/json" \
+  -H "Polaris-Realm: ${realm}" \
+  "http://polaris:8181/api/management/v1/catalogs" \
+  -d "${payload}"
+
+echo
+echo "Granting catalog_admin CATALOG_MANAGE_CONTENT on ${catalog_name}..."
+curl --fail-with-body --silent \
+  -H "Authorization: Bearer ${token}" \
+  -H "Content-Type: application/json" \
+  -H "Polaris-Realm: ${realm}" \
+  -X PUT \
+  "http://polaris:8181/api/management/v1/catalogs/${catalog_name}/catalog-roles/catalog_admin/grants" \
+  -d '{"type":"catalog", "privilege":"CATALOG_MANAGE_CONTENT"}'
+
+echo
+echo "Polaris catalog setup complete."

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris-smoketest.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e -u -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+if [[ -z "${RANGER_VERSION:-}" ]]; then
+  # shellcheck source=/dev/null
+  source "${COMPOSE_DIR}/.env"
+fi
+
+# shellcheck source=/dev/null
+source "${COMPOSE_DIR}/../testlib.sh"
+
+: "${POLARIS_IMAGE:=apache/polaris:1.4.1}"
+: "${SPARK_SQL_IMAGE:=apache/spark:3.5.7-scala2.12-java17-ubuntu}"
+: "${POLARIS_CATALOG_NAME:=quickstart_catalog}"
+: "${POLARIS_STORAGE_LOCATION:=s3://iceberg-obs/polaris-smoke}"
+: "${POLARIS_ICEBERG_SPARK_RUNTIME_VERSION:=1.10.1}"
+: "${ICEBERG_SVC_CATALOG_USER:=svc-iceberg-rest-catalog}"
+: "${ICEBERG_SVC_CATALOG_PRINCIPAL:=${ICEBERG_SVC_CATALOG_USER}/s3g@EXAMPLE.COM}"
+: "${ICEBERG_SVC_CATALOG_KEYTAB:=/etc/security/keytabs/${ICEBERG_SVC_CATALOG_USER}.keytab}"
+
+export POLARIS_IMAGE SPARK_SQL_IMAGE POLARIS_CATALOG_NAME POLARIS_STORAGE_LOCATION
+
+if [[ "${COMPOSE_FILE:-}" != *polaris.yaml* ]]; then
+  export COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.yaml:ranger.yaml:../common/ranger.yaml}:polaris.yaml"
+fi
+
+echo "Fetching permanent S3 credentials for ${ICEBERG_SVC_CATALOG_USER}..."
+s3_secret_output="$(
+  docker-compose exec -T s3g bash -lc \
+    "kinit -kt ${ICEBERG_SVC_CATALOG_KEYTAB} ${ICEBERG_SVC_CATALOG_PRINCIPAL} && ozone sh volume info s3v && ozone s3 getsecret"
+)"
+
+POLARIS_AWS_ACCESS_KEY_ID="$(
+  echo "${s3_secret_output}" | grep -o 'awsAccessKey=[^[:space:]]*' | head -1 | cut -d= -f2
+)"
+POLARIS_AWS_SECRET_ACCESS_KEY="$(
+  echo "${s3_secret_output}" | grep -o 'awsSecret=[^[:space:]]*' | head -1 | cut -d= -f2
+)"
+
+if [[ -z "${POLARIS_AWS_ACCESS_KEY_ID}" || -z "${POLARIS_AWS_SECRET_ACCESS_KEY}" ]]; then
+  echo "ERROR: Failed to parse S3 credentials from ozone s3 getsecret output:"
+  echo "${s3_secret_output}"
+  exit 1
+fi
+
+export POLARIS_AWS_ACCESS_KEY_ID POLARIS_AWS_SECRET_ACCESS_KEY
+
+echo "Starting Polaris (${POLARIS_IMAGE})..."
+docker-compose --ansi never up -d polaris
+
+wait_for_port polaris 8181 120
+
+echo "Provisioning Polaris catalog (${POLARIS_CATALOG_NAME})..."
+docker-compose --ansi never run --rm polaris-setup
+
+spark_packages="org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:${POLARIS_ICEBERG_SPARK_RUNTIME_VERSION},org.apache.iceberg:iceberg-aws-bundle:${POLARIS_ICEBERG_SPARK_RUNTIME_VERSION}"
+sql_file="/opt/hadoop/smoketest/security/ozone-secure-sts-polaris.sql"
+
+echo "Running Spark SQL against Polaris with STS vended credentials..."
+set +e
+spark_output="$(
+  docker-compose --ansi never run --rm spark-sql \
+    /opt/spark/bin/spark-sql \
+      --packages "${spark_packages}" \
+      --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
+      --conf spark.sql.catalog.polaris=org.apache.iceberg.spark.SparkCatalog \
+      --conf spark.sql.catalog.polaris.type=rest \
+      --conf spark.sql.catalog.polaris.uri=http://polaris:8181/api/catalog \
+      --conf spark.sql.catalog.polaris.rest.auth.type=oauth2 \
+      --conf spark.sql.catalog.polaris.oauth2-server-uri=http://polaris:8181/api/catalog/v1/oauth/tokens \
+      --conf spark.sql.catalog.polaris.token-refresh-enabled=false \
+      --conf spark.sql.catalog.polaris.warehouse="${POLARIS_CATALOG_NAME}" \
+      --conf spark.sql.catalog.polaris.scope=PRINCIPAL_ROLE:ALL \
+      --conf spark.sql.catalog.polaris.credential=root:s3cr3t \
+      --conf spark.sql.catalog.polaris.client.region=us-west-2 \
+      --conf spark.sql.catalog.polaris.header.X-Iceberg-Access-Delegation=vended-credentials \
+      -f "${sql_file}" 2>&1
+)"
+spark_exit_code=$?
+set -e
+
+echo "${spark_output}"
+
+if [[ "${spark_exit_code}" -ne 0 ]]; then
+  echo "ERROR: spark-sql exited with status ${spark_exit_code}"
+  exit "${spark_exit_code}"
+fi
+
+if ! echo "${spark_output}" | grep -Fq "testing STS"; then
+  echo "ERROR: Expected Spark output to contain inserted row value 'testing STS'"
+  exit 1
+fi
+
+echo "Polaris STS smoke test passed."

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/polaris.yaml

@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Apache Polaris + Spark SQL overlay for ozonesecure-ha STS smoketests.
+# Requires POLARIS_AWS_ACCESS_KEY_ID / POLARIS_AWS_SECRET_ACCESS_KEY at runtime
+# (fetched from ozone s3 getsecret by polaris-smoketest.sh).
+
+services:
+  polaris:
+    image: ${POLARIS_IMAGE}
+    hostname: polaris
+    dns_search: .
+    ports:
+      - 8181:8181
+    environment:
+      AWS_REGION: us-west-2
+      AWS_ACCESS_KEY_ID: ${POLARIS_AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${POLARIS_AWS_SECRET_ACCESS_KEY}
+      POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t
+      polaris.realm-context.realms: POLARIS
+      polaris.features."ALLOW_SETTING_S3_ENDPOINTS": "true"
+      quarkus.otel.sdk.disabled: "true"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8182/q/health"]
+      interval: 2s
+      timeout: 10s
+      retries: 60
+      start_period: 10s
+    networks:
+      ozone_net:
+        ipv4_address: 172.25.0.121
+
+  polaris-setup:
+    image: alpine/curl:8.19.0
+    dns_search: .
+    depends_on:
+      polaris:
+        condition: service_healthy
+    environment:
+      CLIENT_ID: root
+      CLIENT_SECRET: s3cr3t
+      POLARIS_REALM: POLARIS
+      POLARIS_CATALOG_NAME: ${POLARIS_CATALOG_NAME:-quickstart_catalog}
+      POLARIS_STORAGE_LOCATION: ${POLARIS_STORAGE_LOCATION:-s3://iceberg-obs/polaris-smoke}
+      POLARIS_S3_ENDPOINT: http://s3g:9878
+      POLARIS_STS_ENDPOINT: http://s3g:9880/sts
+      POLARIS_ROLE_ARN: arn:aws:iam::123456789012:role/iceberg-data-all-access-obs
+      POLARIS_AWS_ACCESS_KEY_ID: ${POLARIS_AWS_ACCESS_KEY_ID}
+      POLARIS_AWS_SECRET_ACCESS_KEY: ${POLARIS_AWS_SECRET_ACCESS_KEY}
+    volumes:
+      - ./polaris-setup.sh:/polaris-setup.sh:ro
+    entrypoint: ["/bin/sh", "/polaris-setup.sh"]
+    networks:
+      ozone_net: {}
+
+  spark-sql:
+    image: ${SPARK_SQL_IMAGE}
+    user: "0:0"
+    dns_search: .
+    volumes:
+      - ../..:/opt/hadoop
+      - ${RANGER_M2_DIR:-${HOME}/.m2}:/root/.m2
+    networks:
+      ozone_net: {}

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-ranger.sh

@@ -74,3 +74,5 @@ execute_robot_test s3g freon/generate.robot
 execute_robot_test s3g freon/validate.robot
 
 execute_robot_test s3g -v RANGER_ENDPOINT_URL:"http://ranger:6080" -v USER:hdfs security/ozone-secure-tenant.robot
+execute_robot_test s3g -v RANGER_ENDPOINT_URL:"http://ranger:6080" -v USER:hdfs security/ozone-secure-sts.robot
+"${COMPOSE_DIR}/polaris-smoketest.sh"

hadoop-ozone/dist/src/main/smoketest/security/generate_oversized_session_policy.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+
+def main() -> None:
+    statement = {
+        "Effect": "Allow",
+        "Action": "s3:GetObject",
+        "Resource": "arn:aws:s3:::bucket123/*",
+    }
+    policy = {"Version": "2012-10-17", "Statement": [statement]}
+    base = json.dumps(policy, separators=(",", ":"))
+    # Keep the payload comfortably above the STS policy size limit.
+    policy["Pad"] = "X" * (35000 - len(base) + 64)
+    print(json.dumps(policy, separators=(",", ":")))
+
+
+if __name__ == "__main__":
+    main()