diff --git a/bin/single-node/.env b/bin/single-node/.env index 2c949de0fbe..094ce25416a 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -58,6 +58,18 @@ STORAGE_ICEBERG_CATALOG_POSTGRES_URI_WITHOUT_SCHEME=texera-postgres:5432/texera_ STORAGE_ICEBERG_CATALOG_POSTGRES_USERNAME=texera STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD=password +LAKEKEEPER__PG_DATABASE_URL_READ=postgres://texera:password@texera-postgres:5432/texera_lakekeeper +LAKEKEEPER__PG_DATABASE_URL_WRITE=postgres://texera:password@texera-postgres:5432/texera_lakekeeper +LAKEKEEPER__PG_ENCRYPTION_KEY=texera_key +LAKEKEEPER_BASE_URI=http://texera-lakekeeper:8181 +STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME=texera +STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET=texera-iceberg +STORAGE_S3_AUTH_USERNAME=texera_minio +STORAGE_S3_AUTH_PASSWORD=password +STORAGE_ICEBERG_CATALOG_REST_REGION=us-west-2 + +STORAGE_ICEBERG_CATALOG_REST_URI=http://texera-lakekeeper:8181/catalog +STORAGE_ICEBERG_CATALOG_TYPE=rest # Admin credentials for Texera (used for login and example data loading) USER_SYS_ADMIN_USERNAME=texera -USER_SYS_ADMIN_PASSWORD=texera \ No newline at end of file +USER_SYS_ADMIN_PASSWORD=texera diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 8c63d0d3652..0fb5ded49ed 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -75,6 +75,142 @@ services: timeout: 5s retries: 10 + # Lakekeeper migration init container + # This runs once to migrate the database before the lakekeeper server starts + lakekeeper-migrate: + image: vakamo/lakekeeper:v0.11.0 + container_name: texera-lakekeeper-migrate + depends_on: + postgres: + condition: service_healthy + env_file: + - .env + restart: "no" + entrypoint: ["/home/nonroot/lakekeeper"] + command: ["migrate"] + + # Lakekeeper is the Iceberg REST catalog service + lakekeeper: + image: vakamo/lakekeeper:v0.11.0 + container_name: texera-lakekeeper + restart: always + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_started + lakekeeper-migrate: + condition: service_completed_successfully + env_file: + - .env + entrypoint: ["/home/nonroot/lakekeeper"] + command: ["serve"] + ports: + - "8181:8181" + healthcheck: + test: ["CMD", "/home/nonroot/lakekeeper", "healthcheck"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 10s + + lakekeeper-init: + image: alpine:3.19 + container_name: texera-lakekeeper-init + depends_on: + lakekeeper: + condition: service_healthy + minio: + condition: service_started + env_file: + - .env + restart: "no" + entrypoint: [ "/bin/sh", "-c" ] + command: + - | + set -e + + echo "Installing dependencies..." + apk add --no-cache curl ca-certificates + + echo "Installing MinIO Client..." + wget -q https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc + chmod +x /usr/local/bin/mc + + check_status() { + if [ "$$1" -ge 200 ] && [ "$$1" -lt 300 ]; then + echo "Created $$2 successfully (HTTP $$1)." + elif [ "$$1" -eq 409 ]; then + echo "$$2 already exists (HTTP 409). Treating as success." + else + echo "Failed to create $$2. HTTP Code: $$1" + echo "ERROR RESPONSE:" + if [ -f /tmp/response.txt ]; then cat /tmp/response.txt; fi + echo "" + exit 1 + fi + } + + echo "Step 1: Initializing MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET'..." + mc alias set minio "$$STORAGE_S3_ENDPOINT" "$$STORAGE_S3_AUTH_USERNAME" "$$STORAGE_S3_AUTH_PASSWORD" || true + if mc ls minio/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET > /dev/null 2>&1; then + echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' already exists." + else + mc mb minio/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET || { + echo "Failed to create MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET'" + exit 1 + } + echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' created successfully." + fi + + + echo "Step 2: Initializing Default Project..." + PROJECT_PAYLOAD='{"project-id": "00000000-0000-0000-0000-000000000000", "project-name": "default"}' + + PROJECT_CODE=$$(curl -s -o /tmp/response.txt -w "%{http_code}" \ + -X POST \ + -H "Content-Type: application/json" \ + -d "$$PROJECT_PAYLOAD" \ + "$$LAKEKEEPER_BASE_URI/management/v1/project" || echo "000") + + check_status "$$PROJECT_CODE" "Default Project" + + + echo "Step 3: Initializing Warehouse '$$STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME'..." + CREATE_PAYLOAD=$$(cat <