diff --git a/dify/code/.env.example b/dify/code/.env.example index 5818d291e..0bb881f98 100644 --- a/dify/code/.env.example +++ b/dify/code/.env.example @@ -1,1323 +1,198 @@ -# ------------------------------ -# Environment Variables for API service & worker -# ------------------------------ - -# ------------------------------ -# Common Variables -# ------------------------------ +# ------------------------------------------------------------------ +# Essential defaults for Docker Compose deployments. +# Only include variables required for services to start. +# +# For a default deployment, copy this file to .env and run: +# docker compose up -d +# +# Optional and provider-specific variables live under docker/envs/. +# Copy an optional *.env.example file beside itself without the +# .example suffix when you need those advanced settings. +# Values in docker/.env take precedence over docker/envs/*.env files. +# ------------------------------------------------------------------ -# The backend URL of the console API, -# used to concatenate the authorization callback. -# If empty, it is the same domain. -# Example: https://api.console.dify.ai +# Core service URLs CONSOLE_API_URL= - -# The front-end URL of the console web, -# used to concatenate some front-end addresses and for CORS configuration use. -# If empty, it is the same domain. -# Example: https://console.dify.ai CONSOLE_WEB_URL= - -# Service API Url, -# used to display Service API Base Url to the front-end. -# If empty, it is the same domain. -# Example: https://api.dify.ai SERVICE_API_URL= - -# Trigger external URL -# used to display trigger endpoint API Base URL to the front-end. -# Example: https://api.dify.ai TRIGGER_URL=http://localhost - -# WebApp API backend Url, -# used to declare the back-end URL for the front-end API. -# If empty, it is the same domain. -# Example: https://api.app.dify.ai APP_API_URL= - -# WebApp Url, -# used to display WebAPP API Base Url to the front-end. -# If empty, it is the same domain. -# Example: https://app.dify.ai APP_WEB_URL=https://$(PRIMARY_DOMAIN) - -# File preview or download Url prefix. -# used to display File preview or download Url to the front-end or as Multi-model inputs; -# Url is signed and has expiration time. -# Setting FILES_URL is required for file processing plugins. -# - For https://example.com, use FILES_URL=https://example.com -# - For http://example.com, use FILES_URL=http://example.com -# Recommendation: use a dedicated domain (e.g., https://upload.example.com). -# Alternatively, use http://:5001 or http://api:5001, -# ensuring port 5001 is externally accessible (see docker-compose.yaml). FILES_URL= - -# INTERNAL_FILES_URL is used for plugin daemon communication within Docker network. -# Set this to the internal Docker service URL for proper plugin file access. -# Example: INTERNAL_FILES_URL=http://api:5001 INTERNAL_FILES_URL= +ENDPOINT_URL_TEMPLATE=http://localhost/e/{hook_id} +NEXT_PUBLIC_SOCKET_URL=ws://localhost -# Ensure UTF-8 encoding +# Runtime and security LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=utf-8 - -# Set UV cache directory to avoid permission issues with non-existent home directory UV_CACHE_DIR=/tmp/.uv-cache +# Leave empty to auto-generate a persistent key in the storage directory. +SECRET_KEY= +INIT_PASSWORD= +DEPLOY_ENV=PRODUCTION +CHECK_UPDATE_URL=https://updates.dify.ai +OPENAI_API_BASE=https://api.openai.com/v1 +MIGRATION_ENABLED=true +FILES_ACCESS_TIMEOUT=300 +# Remove `collaboration` from COMPOSE_PROFILES to stop the dedicated websocket service. +ENABLE_COLLABORATION_MODE=true -# ------------------------------ -# Server Configuration -# ------------------------------ - -# The log level for the application. -# Supported values are `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` +# Logging and server workers LOG_LEVEL=INFO -# Log output format: text or json LOG_OUTPUT_FORMAT=text -# Log file path LOG_FILE=/app/logs/server.log -# Log file max size, the unit is MB LOG_FILE_MAX_SIZE=20 -# Log file max backup count LOG_FILE_BACKUP_COUNT=5 -# Log dateformat LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S -# Log Timezone LOG_TZ=UTC - -# Debug mode, default is false. -# It is recommended to turn on this configuration for local development -# to prevent some problems caused by monkey patch. DEBUG=false - -# Flask debug mode, it can output trace information at the interface when turned on, -# which is convenient for debugging. FLASK_DEBUG=false - -# Enable request logging, which will log the request and response information. -# And the log level is DEBUG ENABLE_REQUEST_LOGGING=False - -# A secret key that is used for securely signing the session cookie -# and encrypting sensitive information on the database. -# You can generate a strong key using `openssl rand -base64 42`. -SECRET_KEY= - -# Password for admin user initialization. -# If left unset, admin user will not be prompted for a password -# when creating the initial admin account. -# The length of the password cannot exceed 30 characters. -INIT_PASSWORD= - -# Deployment environment. -# Supported values are `PRODUCTION`, `TESTING`. Default is `PRODUCTION`. -# Testing environment. There will be a distinct color label on the front-end page, -# indicating that this environment is a testing environment. -DEPLOY_ENV=PRODUCTION - -# Whether to enable the version check policy. -# If set to empty, https://updates.dify.ai will be called for version check. -CHECK_UPDATE_URL=https://updates.dify.ai - -# Used to change the OpenAI base address, default is https://api.openai.com/v1. -# When OpenAI cannot be accessed in China, replace it with a domestic mirror address, -# or when a local model provides OpenAI compatible API, it can be replaced. -OPENAI_API_BASE=https://api.openai.com/v1 - -# When enabled, migrations will be executed prior to application startup -# and the application will start after the migrations have completed. -MIGRATION_ENABLED=true - -# File Access Time specifies a time interval in seconds for the file to be accessed. -# The default value is 300 seconds. -FILES_ACCESS_TIMEOUT=300 - -# Access token expiration time in minutes -ACCESS_TOKEN_EXPIRE_MINUTES=60 - -# Refresh token expiration time in days -REFRESH_TOKEN_EXPIRE_DAYS=30 - -# The default number of active requests for the application, where 0 means unlimited, should be a non-negative integer. -APP_DEFAULT_ACTIVE_REQUESTS=0 -# The maximum number of active requests for the application, where 0 means unlimited, should be a non-negative integer. -APP_MAX_ACTIVE_REQUESTS=0 -APP_MAX_EXECUTION_TIME=1200 - -# ------------------------------ -# Container Startup Related Configuration -# Only effective when starting with docker image or docker-compose. -# ------------------------------ - -# API service binding address, default: 0.0.0.0, i.e., all addresses can be accessed. DIFY_BIND_ADDRESS=0.0.0.0 - -# API service binding port number, default 5001. DIFY_PORT=5001 - -# The number of API server workers, i.e., the number of workers. -# Formula: number of cpu cores x 2 + 1 for sync, 1 for Gevent -# Reference: https://docs.gunicorn.org/en/stable/design.html#how-many-workers SERVER_WORKER_AMOUNT=1 - -# Defaults to gevent. If using windows, it can be switched to sync or solo. -# -# Warning: Changing this parameter requires disabling patching for -# psycopg2 and gRPC (see `gunicorn.conf.py` and `celery_entrypoint.py`). -# Modifying it may also decrease throughput. -# -# It is strongly discouraged to change this parameter. SERVER_WORKER_CLASS=gevent - -# Default number of worker connections, the default is 10. SERVER_WORKER_CONNECTIONS=10 - -# Similar to SERVER_WORKER_CLASS. -# If using windows, it can be switched to sync or solo. -# -# Warning: Changing this parameter requires disabling patching for -# psycopg2 and gRPC (see `gunicorn_conf.py` and `celery_entrypoint.py`). -# Modifying it may also decrease throughput. -# -# It is strongly discouraged to change this parameter. -CELERY_WORKER_CLASS= - -# Request handling timeout. The default is 200, -# it is recommended to set it to 360 to support a longer sse connection time. +API_WEBSOCKET_WORKER_CLASS=geventwebsocket.gunicorn.workers.GeventWebSocketWorker +API_WEBSOCKET_WORKER_CONNECTIONS=1000 +API_WEBSOCKET_GUNICORN_TIMEOUT=360 GUNICORN_TIMEOUT=360 - -# The number of Celery workers. The default is 1, and can be set as needed. -CELERY_WORKER_AMOUNT= - -# Flag indicating whether to enable autoscaling of Celery workers. -# -# Autoscaling is useful when tasks are CPU intensive and can be dynamically -# allocated and deallocated based on the workload. -# -# When autoscaling is enabled, the maximum and minimum number of workers can -# be specified. The autoscaling algorithm will dynamically adjust the number -# of workers within the specified range. -# -# Default is false (i.e., autoscaling is disabled). -# -# Example: -# CELERY_AUTO_SCALE=true +CELERY_WORKER_CLASS= +CELERY_WORKER_AMOUNT=4 CELERY_AUTO_SCALE=false - -# The maximum number of Celery workers that can be autoscaled. -# This is optional and only used when autoscaling is enabled. -# Default is not set. CELERY_MAX_WORKERS= - -# The minimum number of Celery workers that can be autoscaled. -# This is optional and only used when autoscaling is enabled. -# Default is not set. CELERY_MIN_WORKERS= +COMPOSE_WORKER_HEALTHCHECK_DISABLED=true +COMPOSE_WORKER_HEALTHCHECK_INTERVAL=30s +COMPOSE_WORKER_HEALTHCHECK_TIMEOUT=30s -# API Tool configuration -API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 -API_TOOL_DEFAULT_READ_TIMEOUT=60 - -# ------------------------------- -# Datasource Configuration -# -------------------------------- -ENABLE_WEBSITE_JINAREADER=true -ENABLE_WEBSITE_FIRECRAWL=true -ENABLE_WEBSITE_WATERCRAWL=true - -# Enable inline LaTeX rendering with single dollar signs ($...$) in the web frontend -# Default is false for security reasons to prevent conflicts with regular text -NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX=false - -# ------------------------------ -# Database Configuration -# The database uses PostgreSQL or MySQL. OceanBase and seekdb are also supported. Please use the public schema. -# It is consistent with the configuration in the database service below. -# You can adjust the database configuration according to your needs. -# ------------------------------ - -# Database type, supported values are `postgresql`, `mysql`, `oceanbase`, `seekdb` +# Database DB_TYPE=postgresql -# For MySQL, only `root` user is supported for now DB_USERNAME=postgres DB_PASSWORD=difyai123456 DB_HOST=db_postgres DB_PORT=5432 DB_DATABASE=dify - -# The size of the database connection pool. -# The default is 30 connections, which can be appropriately increased. SQLALCHEMY_POOL_SIZE=30 -# The default is 10 connections, which allows temporary overflow beyond the pool size. SQLALCHEMY_MAX_OVERFLOW=10 -# Database connection pool recycling time, the default is 3600 seconds. SQLALCHEMY_POOL_RECYCLE=3600 -# Whether to print SQL, default is false. SQLALCHEMY_ECHO=false -# If True, will test connections for liveness upon each checkout SQLALCHEMY_POOL_PRE_PING=false -# Whether to enable the Last in first out option or use default FIFO queue if is false SQLALCHEMY_POOL_USE_LIFO=false -# Number of seconds to wait for a connection from the pool before raising a timeout error. -# Default is 30 SQLALCHEMY_POOL_TIMEOUT=30 - -# Maximum number of connections to the database -# Default is 100 -# -# Reference: https://www.postgresql.org/docs/current/runtime-config-connection.html#GUC-MAX-CONNECTIONS -POSTGRES_MAX_CONNECTIONS=100 - -# Sets the amount of shared memory used for postgres's shared buffers. -# Default is 128MB -# Recommended value: 25% of available memory -# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-SHARED-BUFFERS +SQLALCHEMY_POOL_RESET_ON_RETURN=rollback +PGDATA=/var/lib/postgresql/data/pgdata +POSTGRES_MAX_CONNECTIONS=200 POSTGRES_SHARED_BUFFERS=128MB - -# Sets the amount of memory used by each database worker for working space. -# Default is 4MB -# -# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-WORK-MEM POSTGRES_WORK_MEM=4MB - -# Sets the amount of memory reserved for maintenance activities. -# Default is 64MB -# -# Reference: https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-MAINTENANCE-WORK-MEM POSTGRES_MAINTENANCE_WORK_MEM=64MB - -# Sets the planner's assumption about the effective cache size. -# Default is 4096MB -# -# Reference: https://www.postgresql.org/docs/current/runtime-config-query.html#GUC-EFFECTIVE-CACHE-SIZE POSTGRES_EFFECTIVE_CACHE_SIZE=4096MB - -# Sets the maximum allowed duration of any statement before termination. -# Default is 0 (no timeout). -# -# Reference: https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-STATEMENT-TIMEOUT -# A value of 0 prevents the server from timing out statements. POSTGRES_STATEMENT_TIMEOUT=0 - -# Sets the maximum allowed duration of any idle in-transaction session before termination. -# Default is 0 (no timeout). -# -# Reference: https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-IDLE-IN-TRANSACTION-SESSION-TIMEOUT -# A value of 0 prevents the server from terminating idle sessions. POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=0 -# MySQL Performance Configuration -# Maximum number of connections to MySQL -# -# Default is 1000 -MYSQL_MAX_CONNECTIONS=1000 - -# InnoDB buffer pool size -# Default is 512M -# Recommended value: 70-80% of available memory for dedicated MySQL server -# Reference: https://dev.mysql.com/doc/refman/8.0/en/innodb-parameters.html#sysvar_innodb_buffer_pool_size -MYSQL_INNODB_BUFFER_POOL_SIZE=512M - -# InnoDB log file size -# Default is 128M -# Reference: https://dev.mysql.com/doc/refman/8.0/en/innodb-parameters.html#sysvar_innodb_log_file_size -MYSQL_INNODB_LOG_FILE_SIZE=128M - -# InnoDB flush log at transaction commit -# Default is 2 (flush to OS cache, sync every second) -# Options: 0 (no flush), 1 (flush and sync), 2 (flush to OS cache) -# Reference: https://dev.mysql.com/doc/refman/8.0/en/innodb-parameters.html#sysvar_innodb_flush_log_at_trx_commit -MYSQL_INNODB_FLUSH_LOG_AT_TRX_COMMIT=2 - -# ------------------------------ -# Redis Configuration -# This Redis configuration is used for caching and for pub/sub during conversation. -# ------------------------------ - +# Redis and Celery REDIS_HOST=redis REDIS_PORT=6379 REDIS_USERNAME= REDIS_PASSWORD=difyai123456 REDIS_USE_SSL=false -# SSL configuration for Redis (when REDIS_USE_SSL=true) REDIS_SSL_CERT_REQS=CERT_NONE -# Options: CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED REDIS_SSL_CA_CERTS= -# Path to CA certificate file for SSL verification REDIS_SSL_CERTFILE= -# Path to client certificate file for SSL authentication REDIS_SSL_KEYFILE= -# Path to client private key file for SSL authentication REDIS_DB=0 -# Optional: limit total Redis connections used by API/Worker (unset for default) -# Align with API's REDIS_MAX_CONNECTIONS in configs +REDIS_KEY_PREFIX= REDIS_MAX_CONNECTIONS= - -# Whether to use Redis Sentinel mode. -# If set to true, the application will automatically discover and connect to the master node through Sentinel. -REDIS_USE_SENTINEL=false - -# List of Redis Sentinel nodes. If Sentinel mode is enabled, provide at least one Sentinel IP and port. -# Format: `:,:,:` -REDIS_SENTINELS= -REDIS_SENTINEL_SERVICE_NAME= -REDIS_SENTINEL_USERNAME= -REDIS_SENTINEL_PASSWORD= -REDIS_SENTINEL_SOCKET_TIMEOUT=0.1 - -# List of Redis Cluster nodes. If Cluster mode is enabled, provide at least one Cluster IP and port. -# Format: `:,:,:` -REDIS_USE_CLUSTERS=false -REDIS_CLUSTERS= -REDIS_CLUSTERS_PASSWORD= - -# ------------------------------ -# Celery Configuration -# ------------------------------ - -# Use standalone redis as the broker, and redis db 1 for celery broker. (redis_username is usually set by default as empty) -# Format as follows: `redis://:@:/`. -# Example: redis://:difyai123456@redis:6379/1 -# If use Redis Sentinel, format as follows: `sentinel://:@:/` -# For high availability, you can configure multiple Sentinel nodes (if provided) separated by semicolons like below example: -# Example: sentinel://:difyai123456@localhost:26379/1;sentinel://:difyai12345@localhost:26379/1;sentinel://:difyai12345@localhost:26379/1 +REDIS_RETRY_RETRIES=3 +REDIS_RETRY_BACKOFF_BASE=1.0 +REDIS_RETRY_BACKOFF_CAP=10.0 +REDIS_SOCKET_TIMEOUT=5.0 +REDIS_SOCKET_CONNECT_TIMEOUT=5.0 +REDIS_HEALTH_CHECK_INTERVAL=30 CELERY_BROKER_URL=redis://:difyai123456@redis:6379/1 CELERY_BACKEND=redis BROKER_USE_SSL=false - -# If you are using Redis Sentinel for high availability, configure the following settings. -CELERY_USE_SENTINEL=false -CELERY_SENTINEL_MASTER_NAME= -CELERY_SENTINEL_PASSWORD= -CELERY_SENTINEL_SOCKET_TIMEOUT=0.1 -# e.g. {"tasks.add": {"rate_limit": "10/s"}} CELERY_TASK_ANNOTATIONS=null +EVENT_BUS_REDIS_URL= +EVENT_BUS_REDIS_CHANNEL_TYPE=pubsub +EVENT_BUS_REDIS_USE_CLUSTERS=false -# ------------------------------ -# CORS Configuration -# Used to set the front-end cross-domain access policy. -# ------------------------------ - -# Specifies the allowed origins for cross-origin requests to the Web API, -# e.g. https://dify.app or * for all origins. +# Web and app limits WEB_API_CORS_ALLOW_ORIGINS=* - -# Specifies the allowed origins for cross-origin requests to the console API, -# e.g. https://cloud.dify.ai or * for all origins. CONSOLE_CORS_ALLOW_ORIGINS=* -# When the frontend and backend run on different subdomains, set COOKIE_DOMAIN to the site's top-level domain (e.g., `example.com`). Leading dots are optional. COOKIE_DOMAIN= -# When the frontend and backend run on different subdomains, set NEXT_PUBLIC_COOKIE_DOMAIN=1. NEXT_PUBLIC_COOKIE_DOMAIN= NEXT_PUBLIC_BATCH_CONCURRENCY=5 +API_SENTRY_DSN= +API_SENTRY_TRACES_SAMPLE_RATE=1.0 +API_SENTRY_PROFILES_SAMPLE_RATE=1.0 +WEB_SENTRY_DSN= +AMPLITUDE_API_KEY= +TEXT_GENERATION_TIMEOUT_MS=60000 +CSP_WHITELIST= +ALLOW_EMBED=false +ALLOW_INLINE_STYLES=false +ALLOW_UNSAFE_DATA_SCHEME=false +TOP_K_MAX_VALUE=10 +INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 +LOOP_NODE_MAX_COUNT=100 +MAX_TOOLS_NUM=10 +MAX_PARALLEL_LIMIT=10 +MAX_ITERATIONS_NUM=99 +MAX_TREE_DEPTH=50 +ENABLE_WEBSITE_JINAREADER=true +ENABLE_WEBSITE_FIRECRAWL=true +ENABLE_WEBSITE_WATERCRAWL=true +NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX=false +EXPERIMENTAL_ENABLE_VINEXT=false -# ------------------------------ -# File Storage Configuration -# ------------------------------ - -# The type of storage to use for storing user files. +# Storage and default vector store STORAGE_TYPE=opendal - -# Apache OpenDAL Configuration -# The configuration for OpenDAL consists of the following format: OPENDAL__. -# You can find all the service configurations (CONFIG_NAME) in the repository at: https://github.com/apache/opendal/tree/main/core/src/services. -# Dify will scan configurations starting with OPENDAL_ and automatically apply them. -# The scheme name for the OpenDAL storage. OPENDAL_SCHEME=fs -# Configurations for OpenDAL Local File System. OPENDAL_FS_ROOT=storage - -# ClickZetta Volume Configuration (for storage backend) -# To use ClickZetta Volume as storage backend, set STORAGE_TYPE=clickzetta-volume -# Note: ClickZetta Volume will reuse the existing CLICKZETTA_* connection parameters - -# Volume type selection (three types available): -# - user: Personal/small team use, simple config, user-level permissions -# - table: Enterprise multi-tenant, smart routing, table-level + user-level permissions -# - external: Data lake integration, external storage connection, volume-level + storage-level permissions -CLICKZETTA_VOLUME_TYPE=user - -# External Volume name (required only when TYPE=external) -CLICKZETTA_VOLUME_NAME= - -# Table Volume table prefix (used only when TYPE=table) -CLICKZETTA_VOLUME_TABLE_PREFIX=dataset_ - -# Dify file directory prefix (isolates from other apps, recommended to keep default) -CLICKZETTA_VOLUME_DIFY_PREFIX=dify_km - -# S3 Configuration -# -S3_ENDPOINT= -S3_REGION=us-east-1 -S3_BUCKET_NAME=difyai -S3_ACCESS_KEY= -S3_SECRET_KEY= -# Whether to use AWS managed IAM roles for authenticating with the S3 service. -# If set to false, the access key and secret key must be provided. -S3_USE_AWS_MANAGED_IAM=false - -# Workflow run and Conversation archive storage (S3-compatible) -ARCHIVE_STORAGE_ENABLED=false -ARCHIVE_STORAGE_ENDPOINT= -ARCHIVE_STORAGE_ARCHIVE_BUCKET= -ARCHIVE_STORAGE_EXPORT_BUCKET= -ARCHIVE_STORAGE_ACCESS_KEY= -ARCHIVE_STORAGE_SECRET_KEY= -ARCHIVE_STORAGE_REGION=auto - -# Azure Blob Configuration -# -AZURE_BLOB_ACCOUNT_NAME=difyai -AZURE_BLOB_ACCOUNT_KEY=difyai -AZURE_BLOB_CONTAINER_NAME=difyai-container -AZURE_BLOB_ACCOUNT_URL=https://.blob.core.windows.net - -# Google Storage Configuration -# -GOOGLE_STORAGE_BUCKET_NAME=your-bucket-name -GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64= - -# The Alibaba Cloud OSS configurations, -# -ALIYUN_OSS_BUCKET_NAME=your-bucket-name -ALIYUN_OSS_ACCESS_KEY=your-access-key -ALIYUN_OSS_SECRET_KEY=your-secret-key -ALIYUN_OSS_ENDPOINT=https://oss-ap-southeast-1-internal.aliyuncs.com -ALIYUN_OSS_REGION=ap-southeast-1 -ALIYUN_OSS_AUTH_VERSION=v4 -# Don't start with '/'. OSS doesn't support leading slash in object names. -ALIYUN_OSS_PATH=your-path -ALIYUN_CLOUDBOX_ID=your-cloudbox-id - -# Tencent COS Configuration -# -TENCENT_COS_BUCKET_NAME=your-bucket-name -TENCENT_COS_SECRET_KEY=your-secret-key -TENCENT_COS_SECRET_ID=your-secret-id -TENCENT_COS_REGION=your-region -TENCENT_COS_SCHEME=your-scheme -TENCENT_COS_CUSTOM_DOMAIN=your-custom-domain - -# Oracle Storage Configuration -# -OCI_ENDPOINT=https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com -OCI_BUCKET_NAME=your-bucket-name -OCI_ACCESS_KEY=your-access-key -OCI_SECRET_KEY=your-secret-key -OCI_REGION=us-ashburn-1 - -# Huawei OBS Configuration -# -HUAWEI_OBS_BUCKET_NAME=your-bucket-name -HUAWEI_OBS_SECRET_KEY=your-secret-key -HUAWEI_OBS_ACCESS_KEY=your-access-key -HUAWEI_OBS_SERVER=your-server-url -HUAWEI_OBS_PATH_STYLE=false - -# Volcengine TOS Configuration -# -VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name -VOLCENGINE_TOS_SECRET_KEY=your-secret-key -VOLCENGINE_TOS_ACCESS_KEY=your-access-key -VOLCENGINE_TOS_ENDPOINT=your-server-url -VOLCENGINE_TOS_REGION=your-region - -# Baidu OBS Storage Configuration -# -BAIDU_OBS_BUCKET_NAME=your-bucket-name -BAIDU_OBS_SECRET_KEY=your-secret-key -BAIDU_OBS_ACCESS_KEY=your-access-key -BAIDU_OBS_ENDPOINT=your-server-url - -# Supabase Storage Configuration -# -SUPABASE_BUCKET_NAME=your-bucket-name -SUPABASE_API_KEY=your-access-key -SUPABASE_URL=your-server-url - -# ------------------------------ -# Vector Database Configuration -# ------------------------------ - -# The type of vector store to use. -# Supported values are `weaviate`, `oceanbase`, `seekdb`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `opengauss`, `tablestore`, `vastbase`, `tidb`, `tidb_on_qdrant`, `baidu`, `lindorm`, `huawei_cloud`, `upstash`, `matrixone`, `clickzetta`, `alibabacloud_mysql`, `iris`. VECTOR_STORE=weaviate -# Prefix used to create collection name in vector database VECTOR_INDEX_NAME_PREFIX=Vector_index - -# The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`. WEAVIATE_ENDPOINT=http://weaviate:8080 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih WEAVIATE_GRPC_ENDPOINT=grpc://weaviate:50051 WEAVIATE_TOKENIZATION=word +WEAVIATE_PERSISTENCE_DATA_PATH=/var/lib/weaviate +WEAVIATE_QUERY_DEFAULTS_LIMIT=25 +WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=true +WEAVIATE_DEFAULT_VECTORIZER_MODULE=none +WEAVIATE_CLUSTER_HOSTNAME=node1 +WEAVIATE_AUTHENTICATION_APIKEY_ENABLED=true +WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih +WEAVIATE_AUTHENTICATION_APIKEY_USERS=hello@dify.ai +WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED=true +WEAVIATE_AUTHORIZATION_ADMINLIST_USERS=hello@dify.ai +WEAVIATE_DISABLE_TELEMETRY=false +WEAVIATE_ENABLE_TOKENIZER_GSE=false +WEAVIATE_ENABLE_TOKENIZER_KAGOME_JA=false +WEAVIATE_ENABLE_TOKENIZER_KAGOME_KR=false -# For OceanBase metadata database configuration, available when `DB_TYPE` is `oceanbase`. -# For OceanBase vector database configuration, available when `VECTOR_STORE` is `oceanbase` -# If you want to use OceanBase as both vector database and metadata database, you need to set both `DB_TYPE` and `VECTOR_STORE` to `oceanbase`, and set Database Configuration is the same as the vector database. -# seekdb is the lite version of OceanBase and shares the connection configuration with OceanBase. -OCEANBASE_VECTOR_HOST=oceanbase -OCEANBASE_VECTOR_PORT=2881 -OCEANBASE_VECTOR_USER=root@test -OCEANBASE_VECTOR_PASSWORD=difyai123456 -OCEANBASE_VECTOR_DATABASE=test -OCEANBASE_CLUSTER_NAME=difyai -OCEANBASE_MEMORY_LIMIT=6G -OCEANBASE_ENABLE_HYBRID_SEARCH=false -# For OceanBase vector database, built-in fulltext parsers are `ngram`, `beng`, `space`, `ngram2`, `ik` -# For OceanBase vector database, external fulltext parsers (require plugin installation) are `japanese_ftparser`, `thai_ftparser` -OCEANBASE_FULLTEXT_PARSER=ik -SEEKDB_MEMORY_LIMIT=2G - -# The Qdrant endpoint URL. Only available when VECTOR_STORE is `qdrant`. -QDRANT_URL=http://qdrant:6333 -QDRANT_API_KEY=difyai123456 -QDRANT_CLIENT_TIMEOUT=20 -QDRANT_GRPC_ENABLED=false -QDRANT_GRPC_PORT=6334 -QDRANT_REPLICATION_FACTOR=1 - -# Milvus configuration. Only available when VECTOR_STORE is `milvus`. -# The milvus uri. -MILVUS_URI=http://host.docker.internal:19530 -MILVUS_DATABASE= -MILVUS_TOKEN= -MILVUS_USER= -MILVUS_PASSWORD= -MILVUS_ENABLE_HYBRID_SEARCH=False -MILVUS_ANALYZER_PARAMS= - -# MyScale configuration, only available when VECTOR_STORE is `myscale` -# For multi-language support, please set MYSCALE_FTS_PARAMS with referring to: -# https://myscale.com/docs/en/text-search/#understanding-fts-index-parameters -MYSCALE_HOST=myscale -MYSCALE_PORT=8123 -MYSCALE_USER=default -MYSCALE_PASSWORD= -MYSCALE_DATABASE=dify -MYSCALE_FTS_PARAMS= - -# Couchbase configurations, only available when VECTOR_STORE is `couchbase` -# The connection string must include hostname defined in the docker-compose file (couchbase-server in this case) -COUCHBASE_CONNECTION_STRING=couchbase://couchbase-server -COUCHBASE_USER=Administrator -COUCHBASE_PASSWORD=password -COUCHBASE_BUCKET_NAME=Embeddings -COUCHBASE_SCOPE_NAME=_default - -# pgvector configurations, only available when VECTOR_STORE is `pgvector` -PGVECTOR_HOST=pgvector -PGVECTOR_PORT=5432 -PGVECTOR_USER=postgres -PGVECTOR_PASSWORD=difyai123456 -PGVECTOR_DATABASE=dify -PGVECTOR_MIN_CONNECTION=1 -PGVECTOR_MAX_CONNECTION=5 -PGVECTOR_PG_BIGM=false -PGVECTOR_PG_BIGM_VERSION=1.2-20240606 - -# vastbase configurations, only available when VECTOR_STORE is `vastbase` -VASTBASE_HOST=vastbase -VASTBASE_PORT=5432 -VASTBASE_USER=dify -VASTBASE_PASSWORD=Difyai123456 -VASTBASE_DATABASE=dify -VASTBASE_MIN_CONNECTION=1 -VASTBASE_MAX_CONNECTION=5 - -# pgvecto-rs configurations, only available when VECTOR_STORE is `pgvecto-rs` -PGVECTO_RS_HOST=pgvecto-rs -PGVECTO_RS_PORT=5432 -PGVECTO_RS_USER=postgres -PGVECTO_RS_PASSWORD=difyai123456 -PGVECTO_RS_DATABASE=dify - -# analyticdb configurations, only available when VECTOR_STORE is `analyticdb` -ANALYTICDB_KEY_ID=your-ak -ANALYTICDB_KEY_SECRET=your-sk -ANALYTICDB_REGION_ID=cn-hangzhou -ANALYTICDB_INSTANCE_ID=gp-ab123456 -ANALYTICDB_ACCOUNT=testaccount -ANALYTICDB_PASSWORD=testpassword -ANALYTICDB_NAMESPACE=dify -ANALYTICDB_NAMESPACE_PASSWORD=difypassword -ANALYTICDB_HOST=gp-test.aliyuncs.com -ANALYTICDB_PORT=5432 -ANALYTICDB_MIN_CONNECTION=1 -ANALYTICDB_MAX_CONNECTION=5 - -# TiDB vector configurations, only available when VECTOR_STORE is `tidb_vector` -TIDB_VECTOR_HOST=tidb -TIDB_VECTOR_PORT=4000 -TIDB_VECTOR_USER= -TIDB_VECTOR_PASSWORD= -TIDB_VECTOR_DATABASE=dify - -# Matrixone vector configurations. -MATRIXONE_HOST=matrixone -MATRIXONE_PORT=6001 -MATRIXONE_USER=dump -MATRIXONE_PASSWORD=111 -MATRIXONE_DATABASE=dify - -# Tidb on qdrant configuration, only available when VECTOR_STORE is `tidb_on_qdrant` -TIDB_ON_QDRANT_URL=http://127.0.0.1 -TIDB_ON_QDRANT_API_KEY=dify -TIDB_ON_QDRANT_CLIENT_TIMEOUT=20 -TIDB_ON_QDRANT_GRPC_ENABLED=false -TIDB_ON_QDRANT_GRPC_PORT=6334 -TIDB_PUBLIC_KEY=dify -TIDB_PRIVATE_KEY=dify -TIDB_API_URL=http://127.0.0.1 -TIDB_IAM_API_URL=http://127.0.0.1 -TIDB_REGION=regions/aws-us-east-1 -TIDB_PROJECT_ID=dify -TIDB_SPEND_LIMIT=100 - -# Chroma configuration, only available when VECTOR_STORE is `chroma` -CHROMA_HOST=127.0.0.1 -CHROMA_PORT=8000 -CHROMA_TENANT=default_tenant -CHROMA_DATABASE=default_database -CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthClientProvider -CHROMA_AUTH_CREDENTIALS= - -# Oracle configuration, only available when VECTOR_STORE is `oracle` -ORACLE_USER=dify -ORACLE_PASSWORD=dify -ORACLE_DSN=oracle:1521/FREEPDB1 -ORACLE_CONFIG_DIR=/app/api/storage/wallet -ORACLE_WALLET_LOCATION=/app/api/storage/wallet -ORACLE_WALLET_PASSWORD=dify -ORACLE_IS_AUTONOMOUS=false - -# AlibabaCloud MySQL configuration, only available when VECTOR_STORE is `alibabcloud_mysql` -ALIBABACLOUD_MYSQL_HOST=127.0.0.1 -ALIBABACLOUD_MYSQL_PORT=3306 -ALIBABACLOUD_MYSQL_USER=root -ALIBABACLOUD_MYSQL_PASSWORD=difyai123456 -ALIBABACLOUD_MYSQL_DATABASE=dify -ALIBABACLOUD_MYSQL_MAX_CONNECTION=5 -ALIBABACLOUD_MYSQL_HNSW_M=6 - -# relyt configurations, only available when VECTOR_STORE is `relyt` -RELYT_HOST=db -RELYT_PORT=5432 -RELYT_USER=postgres -RELYT_PASSWORD=difyai123456 -RELYT_DATABASE=postgres - -# open search configuration, only available when VECTOR_STORE is `opensearch` -OPENSEARCH_HOST=opensearch -OPENSEARCH_PORT=9200 -OPENSEARCH_SECURE=true -OPENSEARCH_VERIFY_CERTS=true -OPENSEARCH_AUTH_METHOD=basic -OPENSEARCH_USER=admin -OPENSEARCH_PASSWORD=admin -# If using AWS managed IAM, e.g. Managed Cluster or OpenSearch Serverless -OPENSEARCH_AWS_REGION=ap-southeast-1 -OPENSEARCH_AWS_SERVICE=aoss - -# tencent vector configurations, only available when VECTOR_STORE is `tencent` -TENCENT_VECTOR_DB_URL=http://127.0.0.1 -TENCENT_VECTOR_DB_API_KEY=dify -TENCENT_VECTOR_DB_TIMEOUT=30 -TENCENT_VECTOR_DB_USERNAME=dify -TENCENT_VECTOR_DB_DATABASE=dify -TENCENT_VECTOR_DB_SHARD=1 -TENCENT_VECTOR_DB_REPLICAS=2 -TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH=false - -# ElasticSearch configuration, only available when VECTOR_STORE is `elasticsearch` -ELASTICSEARCH_HOST=0.0.0.0 -ELASTICSEARCH_PORT=9200 -ELASTICSEARCH_USERNAME=elastic -ELASTICSEARCH_PASSWORD=elastic -KIBANA_PORT=5601 - -# Using ElasticSearch Cloud Serverless, or not. -ELASTICSEARCH_USE_CLOUD=false -ELASTICSEARCH_CLOUD_URL=YOUR-ELASTICSEARCH_CLOUD_URL -ELASTICSEARCH_API_KEY=YOUR-ELASTICSEARCH_API_KEY - -ELASTICSEARCH_VERIFY_CERTS=False -ELASTICSEARCH_CA_CERTS= -ELASTICSEARCH_REQUEST_TIMEOUT=100000 -ELASTICSEARCH_RETRY_ON_TIMEOUT=True -ELASTICSEARCH_MAX_RETRIES=10 - -# baidu vector configurations, only available when VECTOR_STORE is `baidu` -BAIDU_VECTOR_DB_ENDPOINT=http://127.0.0.1:5287 -BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS=30000 -BAIDU_VECTOR_DB_ACCOUNT=root -BAIDU_VECTOR_DB_API_KEY=dify -BAIDU_VECTOR_DB_DATABASE=dify -BAIDU_VECTOR_DB_SHARD=1 -BAIDU_VECTOR_DB_REPLICAS=3 -BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER=DEFAULT_ANALYZER -BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE=COARSE_MODE - -# VikingDB configurations, only available when VECTOR_STORE is `vikingdb` -VIKINGDB_ACCESS_KEY=your-ak -VIKINGDB_SECRET_KEY=your-sk -VIKINGDB_REGION=cn-shanghai -VIKINGDB_HOST=api-vikingdb.xxx.volces.com -VIKINGDB_SCHEMA=http -VIKINGDB_CONNECTION_TIMEOUT=30 -VIKINGDB_SOCKET_TIMEOUT=30 - -# Lindorm configuration, only available when VECTOR_STORE is `lindorm` -LINDORM_URL=http://localhost:30070 -LINDORM_USERNAME=admin -LINDORM_PASSWORD=admin -LINDORM_USING_UGC=True -LINDORM_QUERY_TIMEOUT=1 - -# opengauss configurations, only available when VECTOR_STORE is `opengauss` -OPENGAUSS_HOST=opengauss -OPENGAUSS_PORT=6600 -OPENGAUSS_USER=postgres -OPENGAUSS_PASSWORD=Dify@123 -OPENGAUSS_DATABASE=dify -OPENGAUSS_MIN_CONNECTION=1 -OPENGAUSS_MAX_CONNECTION=5 -OPENGAUSS_ENABLE_PQ=false - -# huawei cloud search service vector configurations, only available when VECTOR_STORE is `huawei_cloud` -HUAWEI_CLOUD_HOSTS=https://127.0.0.1:9200 -HUAWEI_CLOUD_USER=admin -HUAWEI_CLOUD_PASSWORD=admin - -# Upstash Vector configuration, only available when VECTOR_STORE is `upstash` -UPSTASH_VECTOR_URL=https://xxx-vector.upstash.io -UPSTASH_VECTOR_TOKEN=dify - -# TableStore Vector configuration -# (only used when VECTOR_STORE is tablestore) -TABLESTORE_ENDPOINT=https://instance-name.cn-hangzhou.ots.aliyuncs.com -TABLESTORE_INSTANCE_NAME=instance-name -TABLESTORE_ACCESS_KEY_ID=xxx -TABLESTORE_ACCESS_KEY_SECRET=xxx -TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE=false - -# Clickzetta configuration, only available when VECTOR_STORE is `clickzetta` -CLICKZETTA_USERNAME= -CLICKZETTA_PASSWORD= -CLICKZETTA_INSTANCE= -CLICKZETTA_SERVICE=api.clickzetta.com -CLICKZETTA_WORKSPACE=quick_start -CLICKZETTA_VCLUSTER=default_ap -CLICKZETTA_SCHEMA=dify -CLICKZETTA_BATCH_SIZE=100 -CLICKZETTA_ENABLE_INVERTED_INDEX=true -CLICKZETTA_ANALYZER_TYPE=chinese -CLICKZETTA_ANALYZER_MODE=smart -CLICKZETTA_VECTOR_DISTANCE_FUNCTION=cosine_distance - -# InterSystems IRIS configuration, only available when VECTOR_STORE is `iris` -IRIS_HOST=iris -IRIS_SUPER_SERVER_PORT=1972 -IRIS_WEB_SERVER_PORT=52773 -IRIS_USER=_SYSTEM -IRIS_PASSWORD=Dify@1234 -IRIS_DATABASE=USER -IRIS_SCHEMA=dify -IRIS_CONNECTION_URL= -IRIS_MIN_CONNECTION=1 -IRIS_MAX_CONNECTION=3 -IRIS_TEXT_INDEX=true -IRIS_TEXT_INDEX_LANGUAGE=en -IRIS_TIMEZONE=UTC - -# ------------------------------ -# Knowledge Configuration -# ------------------------------ - -# Upload file size limit, default 15M. -UPLOAD_FILE_SIZE_LIMIT=15 - -# The maximum number of files that can be uploaded at a time, default 5. -UPLOAD_FILE_BATCH_LIMIT=5 - -# Comma-separated list of file extensions blocked from upload for security reasons. -# Extensions should be lowercase without dots (e.g., exe,bat,sh,dll). -# Empty by default to allow all file types. -# Recommended: exe,bat,cmd,com,scr,vbs,ps1,msi,dll -UPLOAD_FILE_EXTENSION_BLACKLIST= - -# Maximum number of files allowed in a single chunk attachment, default 10. -SINGLE_CHUNK_ATTACHMENT_LIMIT=10 - -# Maximum number of files allowed in a image batch upload operation -IMAGE_FILE_BATCH_LIMIT=10 - -# Maximum allowed image file size for attachments in megabytes, default 2. -ATTACHMENT_IMAGE_FILE_SIZE_LIMIT=2 - -# Timeout for downloading image attachments in seconds, default 60. -ATTACHMENT_IMAGE_DOWNLOAD_TIMEOUT=60 - - -# ETL type, support: `dify`, `Unstructured` -# `dify` Dify's proprietary file extraction scheme -# `Unstructured` Unstructured.io file extraction scheme -ETL_TYPE=dify - -# Unstructured API path and API key, needs to be configured when ETL_TYPE is Unstructured -# Or using Unstructured for document extractor node for pptx. -# For example: http://unstructured:8000/general/v0/general -UNSTRUCTURED_API_URL= -UNSTRUCTURED_API_KEY= -SCARF_NO_ANALYTICS=true - -# ------------------------------ -# Model Configuration -# ------------------------------ - -# The maximum number of tokens allowed for prompt generation. -# This setting controls the upper limit of tokens that can be used by the LLM -# when generating a prompt in the prompt generation tool. -# Default: 512 tokens. -PROMPT_GENERATION_MAX_TOKENS=512 - -# The maximum number of tokens allowed for code generation. -# This setting controls the upper limit of tokens that can be used by the LLM -# when generating code in the code generation tool. -# Default: 1024 tokens. -CODE_GENERATION_MAX_TOKENS=1024 - -# Enable or disable plugin based token counting. If disabled, token counting will return 0. -# This can improve performance by skipping token counting operations. -# Default: false (disabled). -PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false - -# ------------------------------ -# Multi-modal Configuration -# ------------------------------ - -# The format of the image/video/audio/document sent when the multi-modal model is input, -# the default is base64, optional url. -# The delay of the call in url mode will be lower than that in base64 mode. -# It is generally recommended to use the more compatible base64 mode. -# If configured as url, you need to configure FILES_URL as an externally accessible address so that the multi-modal model can access the image/video/audio/document. -MULTIMODAL_SEND_FORMAT=base64 -# Upload image file size limit, default 10M. -UPLOAD_IMAGE_FILE_SIZE_LIMIT=10 -# Upload video file size limit, default 100M. -UPLOAD_VIDEO_FILE_SIZE_LIMIT=100 -# Upload audio file size limit, default 50M. -UPLOAD_AUDIO_FILE_SIZE_LIMIT=50 - -# ------------------------------ -# Sentry Configuration -# Used for application monitoring and error log tracking. -# ------------------------------ -SENTRY_DSN= - -# API Service Sentry DSN address, default is empty, when empty, -# all monitoring information is not reported to Sentry. -# If not set, Sentry error reporting will be disabled. -API_SENTRY_DSN= -# API Service The reporting ratio of Sentry events, if it is 0.01, it is 1%. -API_SENTRY_TRACES_SAMPLE_RATE=1.0 -# API Service The reporting ratio of Sentry profiles, if it is 0.01, it is 1%. -API_SENTRY_PROFILES_SAMPLE_RATE=1.0 - -# Web Service Sentry DSN address, default is empty, when empty, -# all monitoring information is not reported to Sentry. -# If not set, Sentry error reporting will be disabled. -WEB_SENTRY_DSN= - -# Plugin_daemon Service Sentry DSN address, default is empty, when empty, -# all monitoring information is not reported to Sentry. -# If not set, Sentry error reporting will be disabled. -PLUGIN_SENTRY_ENABLED=false -PLUGIN_SENTRY_DSN= - -# ------------------------------ -# Notion Integration Configuration -# Variables can be obtained by applying for Notion integration: https://www.notion.so/my-integrations -# ------------------------------ - -# Configure as "public" or "internal". -# Since Notion's OAuth redirect URL only supports HTTPS, -# if deploying locally, please use Notion's internal integration. -NOTION_INTEGRATION_TYPE=public -# Notion OAuth client secret (used for public integration type) -NOTION_CLIENT_SECRET= -# Notion OAuth client id (used for public integration type) -NOTION_CLIENT_ID= -# Notion internal integration secret. -# If the value of NOTION_INTEGRATION_TYPE is "internal", -# you need to configure this variable. -NOTION_INTERNAL_SECRET= - -# ------------------------------ -# Mail related configuration -# ------------------------------ - -# Mail type, support: resend, smtp, sendgrid -MAIL_TYPE=resend - -# Default send from email address, if not specified -# If using SendGrid, use the 'from' field for authentication if necessary. -MAIL_DEFAULT_SEND_FROM= - -# API-Key for the Resend email provider, used when MAIL_TYPE is `resend`. -RESEND_API_URL=https://api.resend.com -RESEND_API_KEY=your-resend-api-key - - -# SMTP server configuration, used when MAIL_TYPE is `smtp` -SMTP_SERVER= -SMTP_PORT=465 -SMTP_USERNAME= -SMTP_PASSWORD= -SMTP_USE_TLS=true -SMTP_OPPORTUNISTIC_TLS=false -# Optional: override the local hostname used for SMTP HELO/EHLO -SMTP_LOCAL_HOSTNAME= - -# Sendgid configuration -SENDGRID_API_KEY= - -# ------------------------------ -# Others Configuration -# ------------------------------ - -# Maximum length of segmentation tokens for indexing -INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 - -# Member invitation link valid time (hours), -# Default: 72. -INVITE_EXPIRY_HOURS=72 - -# Reset password token valid time (minutes), -RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5 -EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES=5 -CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5 -OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5 - -# The sandbox service endpoint. +# Sandbox and SSRF proxy CODE_EXECUTION_ENDPOINT=http://sandbox:8194 CODE_EXECUTION_API_KEY=dify-sandbox CODE_EXECUTION_SSL_VERIFY=True CODE_EXECUTION_POOL_MAX_CONNECTIONS=100 CODE_EXECUTION_POOL_MAX_KEEPALIVE_CONNECTIONS=20 CODE_EXECUTION_POOL_KEEPALIVE_EXPIRY=5.0 -CODE_MAX_NUMBER=9223372036854775807 -CODE_MIN_NUMBER=-9223372036854775808 -CODE_MAX_DEPTH=5 -CODE_MAX_PRECISION=20 -CODE_MAX_STRING_LENGTH=400000 -CODE_MAX_STRING_ARRAY_LENGTH=30 -CODE_MAX_OBJECT_ARRAY_LENGTH=30 -CODE_MAX_NUMBER_ARRAY_LENGTH=1000 CODE_EXECUTION_CONNECT_TIMEOUT=10 CODE_EXECUTION_READ_TIMEOUT=60 CODE_EXECUTION_WRITE_TIMEOUT=10 -TEMPLATE_TRANSFORM_MAX_LENGTH=400000 - -# Workflow runtime configuration -WORKFLOW_MAX_EXECUTION_STEPS=500 -WORKFLOW_MAX_EXECUTION_TIME=1200 -WORKFLOW_CALL_MAX_DEPTH=5 -MAX_VARIABLE_SIZE=204800 -WORKFLOW_FILE_UPLOAD_LIMIT=10 - -# GraphEngine Worker Pool Configuration -# Minimum number of workers per GraphEngine instance (default: 1) -GRAPH_ENGINE_MIN_WORKERS=1 -# Maximum number of workers per GraphEngine instance (default: 10) -GRAPH_ENGINE_MAX_WORKERS=10 -# Queue depth threshold that triggers worker scale up (default: 3) -GRAPH_ENGINE_SCALE_UP_THRESHOLD=3 -# Seconds of idle time before scaling down workers (default: 5.0) -GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME=5.0 - -# Workflow storage configuration -# Options: rdbms, hybrid -# rdbms: Use only the relational database (default) -# hybrid: Save new data to object storage, read from both object storage and RDBMS -WORKFLOW_NODE_EXECUTION_STORAGE=rdbms - -# Repository configuration -# Core workflow execution repository implementation -# Options: -# - core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository (default) -# - core.repositories.celery_workflow_execution_repository.CeleryWorkflowExecutionRepository -# - extensions.logstore.repositories.logstore_workflow_execution_repository.LogstoreWorkflowExecutionRepository -CORE_WORKFLOW_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository - -# Core workflow node execution repository implementation -# Options: -# - core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository (default) -# - core.repositories.celery_workflow_node_execution_repository.CeleryWorkflowNodeExecutionRepository -# - extensions.logstore.repositories.logstore_workflow_node_execution_repository.LogstoreWorkflowNodeExecutionRepository -CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository - -# API workflow run repository implementation -# Options: -# - repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository (default) -# - extensions.logstore.repositories.logstore_api_workflow_run_repository.LogstoreAPIWorkflowRunRepository -API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository - -# API workflow node execution repository implementation -# Options: -# - repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository (default) -# - extensions.logstore.repositories.logstore_api_workflow_node_execution_repository.LogstoreAPIWorkflowNodeExecutionRepository -API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository - -# Workflow log cleanup configuration -# Enable automatic cleanup of workflow run logs to manage database size -WORKFLOW_LOG_CLEANUP_ENABLED=false -# Number of days to retain workflow run logs (default: 30 days) -WORKFLOW_LOG_RETENTION_DAYS=30 -# Batch size for workflow log cleanup operations (default: 100) -WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100 -# Comma-separated list of workflow IDs to clean logs for -WORKFLOW_LOG_CLEANUP_SPECIFIC_WORKFLOW_IDS= - -# Aliyun SLS Logstore Configuration -# Aliyun Access Key ID -ALIYUN_SLS_ACCESS_KEY_ID= -# Aliyun Access Key Secret -ALIYUN_SLS_ACCESS_KEY_SECRET= -# Aliyun SLS Endpoint (e.g., cn-hangzhou.log.aliyuncs.com) -ALIYUN_SLS_ENDPOINT= -# Aliyun SLS Region (e.g., cn-hangzhou) -ALIYUN_SLS_REGION= -# Aliyun SLS Project Name -ALIYUN_SLS_PROJECT_NAME= -# Number of days to retain workflow run logs (default: 365 days, 3650 for permanent storage) -ALIYUN_SLS_LOGSTORE_TTL=365 -# Enable dual-write to both SLS LogStore and SQL database (default: false) -LOGSTORE_DUAL_WRITE_ENABLED=false -# Enable dual-read fallback to SQL database when LogStore returns no results (default: true) -# Useful for migration scenarios where historical data exists only in SQL database -LOGSTORE_DUAL_READ_ENABLED=true -# Control flag for whether to write the `graph` field to LogStore. -# If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field; -# otherwise write an empty {} instead. Defaults to writing the `graph` field. -LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true - -# HTTP request node in workflow configuration -HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 -HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 -HTTP_REQUEST_NODE_SSL_VERIFY=True - -# HTTP request node timeout configuration -# Maximum timeout values (in seconds) that users can set in HTTP request nodes -# - Connect timeout: Time to wait for establishing connection (default: 10s) -# - Read timeout: Time to wait for receiving response data (default: 600s, 10 minutes) -# - Write timeout: Time to wait for sending request data (default: 600s, 10 minutes) -HTTP_REQUEST_MAX_CONNECT_TIMEOUT=10 -HTTP_REQUEST_MAX_READ_TIMEOUT=600 -HTTP_REQUEST_MAX_WRITE_TIMEOUT=600 - -# Base64 encoded CA certificate data for custom certificate verification (PEM format, optional) -# HTTP_REQUEST_NODE_SSL_CERT_DATA=LS0tLS1CRUdJTi... -# Base64 encoded client certificate data for mutual TLS authentication (PEM format, optional) -# HTTP_REQUEST_NODE_SSL_CLIENT_CERT_DATA=LS0tLS1CRUdJTi... -# Base64 encoded client private key data for mutual TLS authentication (PEM format, optional) -# HTTP_REQUEST_NODE_SSL_CLIENT_KEY_DATA=LS0tLS1CRUdJTi... - -# Webhook request configuration -WEBHOOK_REQUEST_BODY_MAX_SIZE=10485760 - -# Respect X-* headers to redirect clients -RESPECT_XFORWARD_HEADERS_ENABLED=false - -# SSRF Proxy server HTTP URL -SSRF_PROXY_HTTP_URL=http://ssrf_proxy:3128 -# SSRF Proxy server HTTPS URL -SSRF_PROXY_HTTPS_URL=http://ssrf_proxy:3128 - -# Maximum loop count in the workflow -LOOP_NODE_MAX_COUNT=100 - -# The maximum number of tools that can be used in the agent. -MAX_TOOLS_NUM=10 - -# Maximum number of Parallelism branches in the workflow -MAX_PARALLEL_LIMIT=10 - -# The maximum number of iterations for agent setting -MAX_ITERATIONS_NUM=99 - -# ------------------------------ -# Environment Variables for web Service -# ------------------------------ - -# The timeout for the text generation in millisecond -TEXT_GENERATION_TIMEOUT_MS=60000 - -# Allow rendering unsafe URLs which have "data:" scheme. -ALLOW_UNSAFE_DATA_SCHEME=false - -# Maximum number of tree depth in the workflow -MAX_TREE_DEPTH=50 - -# ------------------------------ -# Environment Variables for database Service -# ------------------------------ -# Postgres data directory -PGDATA=/var/lib/postgresql/data/pgdata - -# MySQL Default Configuration -MYSQL_HOST_VOLUME=./volumes/mysql/data - -# ------------------------------ -# Environment Variables for sandbox Service -# ------------------------------ - -# The API key for the sandbox service SANDBOX_API_KEY=dify-sandbox -# The mode in which the Gin framework runs SANDBOX_GIN_MODE=release -# The timeout for the worker in seconds SANDBOX_WORKER_TIMEOUT=15 -# Enable network for the sandbox service SANDBOX_ENABLE_NETWORK=true -# HTTP proxy URL for SSRF protection SANDBOX_HTTP_PROXY=http://ssrf_proxy:3128 -# HTTPS proxy URL for SSRF protection SANDBOX_HTTPS_PROXY=http://ssrf_proxy:3128 -# The port on which the sandbox service runs SANDBOX_PORT=8194 - -# ------------------------------ -# Environment Variables for weaviate Service -# (only used when VECTOR_STORE is weaviate) -# ------------------------------ -WEAVIATE_PERSISTENCE_DATA_PATH=/var/lib/weaviate -WEAVIATE_QUERY_DEFAULTS_LIMIT=25 -WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=true -WEAVIATE_DEFAULT_VECTORIZER_MODULE=none -WEAVIATE_CLUSTER_HOSTNAME=node1 -WEAVIATE_AUTHENTICATION_APIKEY_ENABLED=true -WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih -WEAVIATE_AUTHENTICATION_APIKEY_USERS=hello@dify.ai -WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED=true -WEAVIATE_AUTHORIZATION_ADMINLIST_USERS=hello@dify.ai -WEAVIATE_DISABLE_TELEMETRY=false -WEAVIATE_ENABLE_TOKENIZER_GSE=false -WEAVIATE_ENABLE_TOKENIZER_KAGOME_JA=false -WEAVIATE_ENABLE_TOKENIZER_KAGOME_KR=false - -# ------------------------------ -# Environment Variables for Chroma -# (only used when VECTOR_STORE is chroma) -# ------------------------------ - -# Authentication credentials for Chroma server -CHROMA_SERVER_AUTHN_CREDENTIALS=difyai123456 -# Authentication provider for Chroma server -CHROMA_SERVER_AUTHN_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider -# Persistence setting for Chroma server -CHROMA_IS_PERSISTENT=TRUE - -# ------------------------------ -# Environment Variables for Oracle Service -# (only used when VECTOR_STORE is oracle) -# ------------------------------ -ORACLE_PWD=Dify123456 -ORACLE_CHARACTERSET=AL32UTF8 - -# ------------------------------ -# Environment Variables for milvus Service -# (only used when VECTOR_STORE is milvus) -# ------------------------------ -# ETCD configuration for auto compaction mode -ETCD_AUTO_COMPACTION_MODE=revision -# ETCD configuration for auto compaction retention in terms of number of revisions -ETCD_AUTO_COMPACTION_RETENTION=1000 -# ETCD configuration for backend quota in bytes -ETCD_QUOTA_BACKEND_BYTES=4294967296 -# ETCD configuration for the number of changes before triggering a snapshot -ETCD_SNAPSHOT_COUNT=50000 -# MinIO access key for authentication -MINIO_ACCESS_KEY=minioadmin -# MinIO secret key for authentication -MINIO_SECRET_KEY=minioadmin -# ETCD service endpoints -ETCD_ENDPOINTS=etcd:2379 -# MinIO service address -MINIO_ADDRESS=minio:9000 -# Enable or disable security authorization -MILVUS_AUTHORIZATION_ENABLED=true - -# ------------------------------ -# Environment Variables for pgvector / pgvector-rs Service -# (only used when VECTOR_STORE is pgvector / pgvector-rs) -# ------------------------------ -PGVECTOR_PGUSER=postgres -# The password for the default postgres user. -PGVECTOR_POSTGRES_PASSWORD=difyai123456 -# The name of the default postgres database. -PGVECTOR_POSTGRES_DB=dify -# postgres data directory -PGVECTOR_PGDATA=/var/lib/postgresql/data/pgdata - -# ------------------------------ -# Environment Variables for opensearch -# (only used when VECTOR_STORE is opensearch) -# ------------------------------ -OPENSEARCH_DISCOVERY_TYPE=single-node -OPENSEARCH_BOOTSTRAP_MEMORY_LOCK=true -OPENSEARCH_JAVA_OPTS_MIN=512m -OPENSEARCH_JAVA_OPTS_MAX=1024m -OPENSEARCH_INITIAL_ADMIN_PASSWORD=Qazwsxedc!@#123 -OPENSEARCH_MEMLOCK_SOFT=-1 -OPENSEARCH_MEMLOCK_HARD=-1 -OPENSEARCH_NOFILE_SOFT=65536 -OPENSEARCH_NOFILE_HARD=65536 - -# ------------------------------ -# Environment Variables for Nginx reverse proxy -# ------------------------------ -NGINX_SERVER_NAME=_ -NGINX_HTTPS_ENABLED=false -# HTTP port -NGINX_PORT=80 -# SSL settings are only applied when HTTPS_ENABLED is true -NGINX_SSL_PORT=443 -# if HTTPS_ENABLED is true, you're required to add your own SSL certificates/keys to the `./nginx/ssl` directory -# and modify the env vars below accordingly. -NGINX_SSL_CERT_FILENAME=dify.crt -NGINX_SSL_CERT_KEY_FILENAME=dify.key -NGINX_SSL_PROTOCOLS=TLSv1.2 TLSv1.3 - -# Nginx performance tuning -NGINX_WORKER_PROCESSES=auto -NGINX_CLIENT_MAX_BODY_SIZE=100M -NGINX_KEEPALIVE_TIMEOUT=65 - -# Proxy settings -NGINX_PROXY_READ_TIMEOUT=3600s -NGINX_PROXY_SEND_TIMEOUT=3600s - -# Set true to accept requests for /.well-known/acme-challenge/ -NGINX_ENABLE_CERTBOT_CHALLENGE=false - -# ------------------------------ -# Certbot Configuration -# ------------------------------ - -# Email address (required to get certificates from Let's Encrypt) -CERTBOT_EMAIL=your_email@example.com - -# Domain name -CERTBOT_DOMAIN=your_domain.com - -# certbot command options -# i.e: --force-renewal --dry-run --test-cert --debug -CERTBOT_OPTIONS= - -# ------------------------------ -# Environment Variables for SSRF Proxy -# ------------------------------ +PIP_MIRROR_URL= +SSRF_PROXY_HTTP_URL=http://ssrf_proxy:3128 +SSRF_PROXY_HTTPS_URL=http://ssrf_proxy:3128 SSRF_HTTP_PORT=3128 SSRF_COREDUMP_DIR=/var/spool/squid SSRF_REVERSE_PROXY_PORT=8194 @@ -1330,55 +205,7 @@ SSRF_POOL_MAX_CONNECTIONS=100 SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS=20 SSRF_POOL_KEEPALIVE_EXPIRY=5.0 -# ------------------------------ -# docker env var for specifying vector db and metadata db type at startup -# (based on the vector db and metadata db type, the corresponding docker -# compose profile will be used) -# if you want to use unstructured, add ',unstructured' to the end -# ------------------------------ -COMPOSE_PROFILES=${VECTOR_STORE:-weaviate},${DB_TYPE:-postgresql} - -# ------------------------------ -# Docker Compose Service Expose Host Port Configurations -# ------------------------------ -EXPOSE_NGINX_PORT=80 -EXPOSE_NGINX_SSL_PORT=443 - -# ---------------------------------------------------------------------------- -# ModelProvider & Tool Position Configuration -# Used to specify the model providers and tools that can be used in the app. -# ---------------------------------------------------------------------------- - -# Pin, include, and exclude tools -# Use comma-separated values with no spaces between items. -# Example: POSITION_TOOL_PINS=bing,google -POSITION_TOOL_PINS= -POSITION_TOOL_INCLUDES= -POSITION_TOOL_EXCLUDES= - -# Pin, include, and exclude model providers -# Use comma-separated values with no spaces between items. -# Example: POSITION_PROVIDER_PINS=openai,openllm -POSITION_PROVIDER_PINS= -POSITION_PROVIDER_INCLUDES= -POSITION_PROVIDER_EXCLUDES= - -# CSP https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP -CSP_WHITELIST= - -# Enable or disable create tidb service job -CREATE_TIDB_SERVICE_JOB_ENABLED=false - -# Maximum number of submitted thread count in a ThreadPool for parallel node execution -MAX_SUBMIT_COUNT=100 - -# The maximum number of top-k value for RAG. -TOP_K_MAX_VALUE=10 - -# ------------------------------ -# Plugin Daemon Configuration -# ------------------------------ - +# Plugin daemon DB_PLUGIN_DATABASE=dify_plugin EXPOSE_PLUGIN_DAEMON_PORT=5002 PLUGIN_DAEMON_PORT=5002 @@ -1387,174 +214,45 @@ PLUGIN_DAEMON_URL=http://plugin_daemon:5002 PLUGIN_MAX_PACKAGE_SIZE=52428800 PLUGIN_MODEL_SCHEMA_CACHE_TTL=3600 PLUGIN_PPROF_ENABLED=false - PLUGIN_DEBUGGING_HOST=0.0.0.0 PLUGIN_DEBUGGING_PORT=5003 EXPOSE_PLUGIN_DEBUGGING_HOST=localhost EXPOSE_PLUGIN_DEBUGGING_PORT=5003 - -# If this key is changed, DIFY_INNER_API_KEY in plugin_daemon service must also be updated or agent node will fail. PLUGIN_DIFY_INNER_API_KEY=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1 PLUGIN_DIFY_INNER_API_URL=http://api:5001 - -ENDPOINT_URL_TEMPLATE=http://localhost/e/{hook_id} - -MARKETPLACE_ENABLED=true -MARKETPLACE_API_URL=https://marketplace.dify.ai - FORCE_VERIFYING_SIGNATURE=true -ENFORCE_LANGGENIUS_PLUGIN_SIGNATURES=true - PLUGIN_STDIO_BUFFER_SIZE=1024 PLUGIN_STDIO_MAX_BUFFER_SIZE=5242880 - PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120 -# Plugin Daemon side timeout (configure to match the API side below) PLUGIN_MAX_EXECUTION_TIMEOUT=600 -# API side timeout (configure to match the Plugin Daemon side above) -PLUGIN_DAEMON_TIMEOUT=600.0 -# PIP_MIRROR_URL=https://pypi.tuna.tsinghua.edu.cn/simple -PIP_MIRROR_URL= - -# https://github.com/langgenius/dify-plugin-daemon/blob/main/.env.example -# Plugin storage type, local aws_s3 tencent_cos azure_blob aliyun_oss volcengine_tos PLUGIN_STORAGE_TYPE=local PLUGIN_STORAGE_LOCAL_ROOT=/app/storage PLUGIN_WORKING_PATH=/app/storage/cwd PLUGIN_INSTALLED_PATH=plugin PLUGIN_PACKAGE_CACHE_PATH=plugin_packages PLUGIN_MEDIA_CACHE_PATH=assets -# Plugin oss bucket PLUGIN_STORAGE_OSS_BUCKET= -# Plugin oss s3 credentials -PLUGIN_S3_USE_AWS=false -PLUGIN_S3_USE_AWS_MANAGED_IAM=false -PLUGIN_S3_ENDPOINT= -PLUGIN_S3_USE_PATH_STYLE=false -PLUGIN_AWS_ACCESS_KEY= -PLUGIN_AWS_SECRET_KEY= -PLUGIN_AWS_REGION= -# Plugin oss azure blob -PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME= -PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING= -# Plugin oss tencent cos -PLUGIN_TENCENT_COS_SECRET_KEY= -PLUGIN_TENCENT_COS_SECRET_ID= -PLUGIN_TENCENT_COS_REGION= -# Plugin oss aliyun oss -PLUGIN_ALIYUN_OSS_REGION= -PLUGIN_ALIYUN_OSS_ENDPOINT= -PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID= -PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET= -PLUGIN_ALIYUN_OSS_AUTH_VERSION=v4 -PLUGIN_ALIYUN_OSS_PATH= -# Plugin oss volcengine tos -PLUGIN_VOLCENGINE_TOS_ENDPOINT= -PLUGIN_VOLCENGINE_TOS_ACCESS_KEY= -PLUGIN_VOLCENGINE_TOS_SECRET_KEY= -PLUGIN_VOLCENGINE_TOS_REGION= - -# ------------------------------ -# OTLP Collector Configuration -# ------------------------------ -ENABLE_OTEL=false -OTLP_TRACE_ENDPOINT= -OTLP_METRIC_ENDPOINT= -OTLP_BASE_ENDPOINT=http://localhost:4318 -OTLP_API_KEY= -OTEL_EXPORTER_OTLP_PROTOCOL= -OTEL_EXPORTER_TYPE=otlp -OTEL_SAMPLING_RATE=0.1 -OTEL_BATCH_EXPORT_SCHEDULE_DELAY=5000 -OTEL_MAX_QUEUE_SIZE=2048 -OTEL_MAX_EXPORT_BATCH_SIZE=512 -OTEL_METRIC_EXPORT_INTERVAL=60000 -OTEL_BATCH_EXPORT_TIMEOUT=10000 -OTEL_METRIC_EXPORT_TIMEOUT=30000 - -# Prevent Clickjacking -ALLOW_EMBED=false - -# Dataset queue monitor configuration -QUEUE_MONITOR_THRESHOLD=200 -# You can configure multiple ones, separated by commas. eg: test1@dify.ai,test2@dify.ai -QUEUE_MONITOR_ALERT_EMAILS= -# Monitor interval in minutes, default is 30 minutes -QUEUE_MONITOR_INTERVAL=30 - -# Swagger UI configuration -SWAGGER_UI_ENABLED=false -SWAGGER_UI_PATH=/swagger-ui.html - -# Whether to encrypt dataset IDs when exporting DSL files (default: true) -# Set to false to export dataset IDs as plain text for easier cross-environment import -DSL_EXPORT_ENCRYPT_DATASET_ID=true - -# Maximum number of segments for dataset segments API (0 for unlimited) -DATASET_MAX_SEGMENTS_PER_REQUEST=0 - -# Celery schedule tasks configuration -ENABLE_CLEAN_EMBEDDING_CACHE_TASK=false -ENABLE_CLEAN_UNUSED_DATASETS_TASK=false -ENABLE_CREATE_TIDB_SERVERLESS_TASK=false -ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK=false -ENABLE_CLEAN_MESSAGES=false -ENABLE_WORKFLOW_RUN_CLEANUP_TASK=false -ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK=false -ENABLE_DATASETS_QUEUE_MONITOR=false -ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK=true -ENABLE_WORKFLOW_SCHEDULE_POLLER_TASK=true -WORKFLOW_SCHEDULE_POLLER_INTERVAL=1 -WORKFLOW_SCHEDULE_POLLER_BATCH_SIZE=100 -WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK=0 - -# Tenant isolated task queue configuration -TENANT_ISOLATED_TASK_CONCURRENCY=1 - -# Maximum allowed CSV file size for annotation import in megabytes -ANNOTATION_IMPORT_FILE_SIZE_LIMIT=2 -#Maximum number of annotation records allowed in a single import -ANNOTATION_IMPORT_MAX_RECORDS=10000 -# Minimum number of annotation records required in a single import -ANNOTATION_IMPORT_MIN_RECORDS=1 -ANNOTATION_IMPORT_RATE_LIMIT_PER_MINUTE=5 -ANNOTATION_IMPORT_RATE_LIMIT_PER_HOUR=20 -# Maximum number of concurrent annotation import tasks per tenant -ANNOTATION_IMPORT_MAX_CONCURRENT=5 - -# The API key of amplitude -AMPLITUDE_API_KEY= - -# Sandbox expired records clean configuration -SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD=21 -SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE=1000 -SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL=200 -SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS=30 - - -# Redis URL used for PubSub between API and -# celery worker -# defaults to url constructed from `REDIS_*` -# configurations -PUBSUB_REDIS_URL= -# Pub/sub channel type for streaming events. -# valid options are: -# -# - pubsub: for normal Pub/Sub -# - sharded: for sharded Pub/Sub -# -# It's highly recommended to use sharded Pub/Sub AND redis cluster -# for large deployments. -PUBSUB_REDIS_CHANNEL_TYPE=pubsub -# Whether to use Redis cluster mode while running -# PubSub. -# It's highly recommended to enable this for large deployments. -PUBSUB_REDIS_USE_CLUSTERS=false - -# Whether to Enable human input timeout check task -ENABLE_HUMAN_INPUT_TIMEOUT_TASK=true -# Human input timeout check interval in minutes -HUMAN_INPUT_TIMEOUT_TASK_INTERVAL=1 - +PLUGIN_SENTRY_ENABLED=false +PLUGIN_SENTRY_DSN= +MARKETPLACE_ENABLED=true +MARKETPLACE_API_URL=https://marketplace.dify.ai +MARKETPLACE_URL= -SANDBOX_EXPIRED_RECORDS_CLEAN_TASK_LOCK_TTL=90000 +# Nginx and Docker Compose +NGINX_SERVER_NAME=_ +NGINX_HTTPS_ENABLED=false +NGINX_PORT=80 +NGINX_SSL_PORT=443 +NGINX_SSL_CERT_FILENAME=dify.crt +NGINX_SSL_CERT_KEY_FILENAME=dify.key +NGINX_SSL_PROTOCOLS=TLSv1.2 TLSv1.3 +NGINX_WORKER_PROCESSES=auto +NGINX_CLIENT_MAX_BODY_SIZE=100M +NGINX_KEEPALIVE_TIMEOUT=65 +NGINX_PROXY_READ_TIMEOUT=3600s +NGINX_PROXY_SEND_TIMEOUT=3600s +NGINX_ENABLE_CERTBOT_CHALLENGE=false +NGINX_SOCKET_IO_UPSTREAM=api_websocket:5001 +EXPOSE_NGINX_PORT=80 +EXPOSE_NGINX_SSL_PORT=443 +COMPOSE_PROFILES=${VECTOR_STORE:-weaviate},${DB_TYPE:-postgresql},collaboration diff --git a/dify/code/.gitignore b/dify/code/.gitignore new file mode 100644 index 000000000..c3a47ad59 --- /dev/null +++ b/dify/code/.gitignore @@ -0,0 +1,3 @@ +# Ignore actual .env files (keep only .env.example files in git) +*.env +!*.env.example diff --git a/dify/code/README.md b/dify/code/README.md index 4c40317f3..2e21a2ce8 100644 --- a/dify/code/README.md +++ b/dify/code/README.md @@ -5,44 +5,46 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T ### What's Updated - **Certbot Container**: `docker-compose.yaml` now contains `certbot` for managing SSL certificates. This container automatically renews certificates and ensures secure HTTPS connections.\ - For more information, refer `docker/certbot/README.md`. + For more information, refer to `docker/certbot/README.md`. -- **Persistent Environment Variables**: Environment variables are now managed through a `.env` file, ensuring that your configurations persist across deployments. +- **Persistent Environment Variables**: Essential startup defaults are provided in `.env.example`, while local values are stored in `.env`, ensuring that your configurations persist across deployments. > What is `.env`?

- > The `.env` file is a crucial component in Docker and Docker Compose environments, serving as a centralized configuration file where you can define environment variables that are accessible to the containers at runtime. This file simplifies the management of environment settings across different stages of development, testing, and production, providing consistency and ease of configuration to deployments. + > The `.env` file is the local startup file. Copy it from `.env.example` for a default deployment. Optional advanced settings live in `envs/*.env.example` files. - **Unified Vector Database Services**: All vector database services are now managed from a single Docker Compose file `docker-compose.yaml`. You can switch between different vector databases by setting the `VECTOR_STORE` environment variable in your `.env` file. -- **Mandatory .env File**: A `.env` file is now required to run `docker compose up`. This file is crucial for configuring your deployment and for any custom settings to persist through upgrades. - ### How to Deploy Dify with `docker-compose.yaml` 1. **Prerequisites**: Ensure Docker and Docker Compose are installed on your system. -1. **Environment Setup**: +2. **Environment Setup**: - Navigate to the `docker` directory. - - Copy the `.env.example` file to a new file named `.env` by running `cp .env.example .env`. - - Customize the `.env` file as needed. Refer to the `.env.example` file for detailed configuration options. - - **Optional (Recommended for upgrades)**: - You may use the environment synchronization tool to help keep your `.env` file aligned with the latest `.env.example` updates, while preserving your custom settings. - This is especially useful when upgrading Dify or managing a large, customized `.env` file. + - Copy `.env.example` to `.env`. + - Customize `.env` when you need to change essential startup defaults. Copy optional files from `envs/` without the `.example` suffix when you need advanced settings. + - **Optional (for advanced deployments)**: + If you maintain a full `.env` file copied from `.env.example`, you may use the environment synchronization tool to keep it aligned with the latest `.env.example` updates while preserving your custom settings. See the [Environment Variables Synchronization](#environment-variables-synchronization) section below. -1. **Running the Services**: - - Execute `docker compose up` from the `docker` directory to start the services. - - To specify a vector database, set the `VECTOR_STORE` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`. -1. **SSL Certificate Setup**: - - Refer `docker/certbot/README.md` to set up SSL certificates using Certbot. -1. **OpenTelemetry Collector Setup**: - - Change `ENABLE_OTEL` to `true` in `.env`. - - Configure `OTLP_BASE_ENDPOINT` properly. +3. **Running the Services**: + - Execute `docker compose up -d` from the `docker` directory to start the services. + - To specify a vector database, set the `VECTOR_STORE` variable in your `.env` file to your desired vector database service, such as `milvus`, `weaviate`, or `opensearch`. See `envs/vectorstores/` for the full list of supported options. + ```bash + cp .env.example .env + docker compose up -d + ``` + +4. **SSL Certificate Setup**: + - Refer to `docker/certbot/README.md` to set up SSL certificates using Certbot. +5. **OpenTelemetry Collector Setup**: + - Copy `envs/core-services/shared.env.example` to `envs/core-services/shared.env`. + - Set `ENABLE_OTEL=true` and configure `OTLP_BASE_ENDPOINT`. Tune the other `OTEL_*` knobs in the same file if needed. ### How to Deploy Middleware for Developing Dify 1. **Middleware Setup**: - Use the `docker-compose.middleware.yaml` for setting up essential middleware services like databases and caches. - Navigate to the `docker` directory. - - Ensure the `middleware.env` file is created by running `cp middleware.env.example middleware.env` (refer to the `middleware.env.example` file). -1. **Running Middleware Services**: + - Ensure the `middleware.env` file is created by running `cp envs/middleware.env.example middleware.env` (refer to the `envs/middleware.env.example` file). +2. **Running Middleware Services**: - Navigate to the `docker` directory. - Execute `docker compose --env-file middleware.env -f docker-compose.middleware.yaml -p dify up -d` to start PostgreSQL/MySQL (per `DB_TYPE`) plus the bundled Weaviate instance. @@ -53,12 +55,18 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T For users migrating from the `docker-legacy` setup: 1. **Review Changes**: Familiarize yourself with the new `.env` configuration and Docker Compose setup. -1. **Transfer Customizations**: +2. **Transfer Customizations**: - If you have customized configurations such as `docker-compose.yaml`, `ssrf_proxy/squid.conf`, or `nginx/conf.d/default.conf`, you will need to reflect these changes in the `.env` file you create. -1. **Data Migration**: +3. **Data Migration**: - Ensure that data from services like databases and caches is backed up and migrated appropriately to the new structure if necessary. -### Overview of `.env` +### Overview of `.env`, `.env.example`, and `envs/` + +- `.env.example` contains the essential default configuration for Docker Compose deployments. +- `.env` contains local startup values copied from `.env.example` and any local changes. +- `envs/*.env.example` files contain optional advanced configuration grouped by theme. + +Docker Compose reads `envs/*.env` files when present, then reads `.env` last so values in `.env` take precedence. #### Key Modules and Customization @@ -68,58 +76,63 @@ For users migrating from the `docker-legacy` setup: #### Other notable variables -The `.env.example` file provided in the Docker setup is extensive and covers a wide range of configuration options. It is structured into several sections, each pertaining to different aspects of the application and its services. Here are some of the key sections and variables: +The root `.env.example` file contains the essential startup settings. Optional and provider-specific settings are grouped in `envs/*.env.example` files. Here are some of the key sections and variables: 1. **Common Variables**: - - `CONSOLE_API_URL`, `SERVICE_API_URL`: URLs for different API services. - - `APP_WEB_URL`: Frontend application URL. - - `FILES_URL`: Base URL for file downloads and previews. + - `CONSOLE_API_URL`, `CONSOLE_WEB_URL`, `SERVICE_API_URL`, `APP_API_URL`, `APP_WEB_URL`: URLs for the API and frontend services. + - `FILES_URL`, `INTERNAL_FILES_URL`: Public and internal base URLs for file downloads and previews. + - `ENDPOINT_URL_TEMPLATE`, `NEXT_PUBLIC_SOCKET_URL`, `TRIGGER_URL`: Additional service URLs. + + See `.env.example` for the full list. -1. **Server Configuration**: +2. **Server Configuration**: - `LOG_LEVEL`, `DEBUG`, `FLASK_DEBUG`: Logging and debug settings. - - `SECRET_KEY`: A key for encrypting session cookies and other sensitive data. + - `SECRET_KEY`: A key for signing sessions, JWTs, and file URLs. Leave it empty to let Dify generate a persistent key in the storage directory, or set a unique value yourself. -1. **Database Configuration**: +3. **Database Configuration**: - `DB_USERNAME`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`, `DB_DATABASE`: PostgreSQL database credentials and connection details. -1. **Redis Configuration**: +4. **Redis Configuration**: - `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`: Redis server connection settings. + - `REDIS_KEY_PREFIX`: Optional global namespace prefix for Redis keys, topics, streams, and Celery Redis transport artifacts. -1. **Celery Configuration**: +5. **Celery Configuration**: - `CELERY_BROKER_URL`: Configuration for Celery message broker. -1. **Storage Configuration**: +6. **Storage Configuration**: - - `STORAGE_TYPE`, `S3_BUCKET_NAME`, `AZURE_BLOB_ACCOUNT_NAME`: Settings for file storage options like local, S3, Azure Blob, etc. + - `STORAGE_TYPE`, `OPENDAL_SCHEME`, `OPENDAL_FS_ROOT`: Default local file storage settings. Optional storage backends are configured from the files under `envs/`. -1. **Vector Database Configuration**: +7. **Vector Database Configuration**: - - `VECTOR_STORE`: Type of vector database (e.g., `weaviate`, `milvus`). + - `VECTOR_STORE`: Type of vector database (e.g., `weaviate`, `milvus`). See `envs/vectorstores/` for the full list of supported options. - Specific settings for each vector store like `WEAVIATE_ENDPOINT`, `MILVUS_URI`. -1. **CORS Configuration**: +8. **CORS Configuration**: - `WEB_API_CORS_ALLOW_ORIGINS`, `CONSOLE_CORS_ALLOW_ORIGINS`: Settings for cross-origin resource sharing. -1. **OpenTelemetry Configuration**: +9. **OpenTelemetry Configuration**: - `ENABLE_OTEL`: Enable OpenTelemetry collector in api. - `OTLP_BASE_ENDPOINT`: Endpoint for your OTLP exporter. -1. **Other Service-Specific Environment Variables**: +10. **Other Service-Specific Environment Variables**: - - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`. + - Each service like `nginx`, `redis`, `db`, and vector databases have specific environment variables that are directly referenced in the `docker-compose.yaml`. ### Environment Variables Synchronization -When upgrading Dify or pulling the latest changes, new environment variables may be introduced in `.env.example`. +When upgrading Dify or pulling the latest changes, new environment variables may be introduced in `.env.example` or the optional files under `envs/`. + +If you use the default workflow, review `.env.example` and keep your `.env` aligned with essential startup values. -To help keep your existing `.env` file up to date **without losing your custom values**, an optional environment variables synchronization tool is provided. +If you maintain a customized `.env` file copied from `.env.example`, an optional environment variables synchronization tool is provided. > This tool performs a **one-way synchronization** from `.env.example` to `.env`. > Existing values in `.env` are never overwritten automatically. @@ -142,9 +155,9 @@ Before synchronization, the current `.env` file is saved to the `env-backup/` di **When to use** -- After upgrading Dify to a newer version +- After upgrading Dify to a newer version with a full `.env` file - When `.env.example` has been updated with new environment variables -- When managing a large or heavily customized `.env` file +- When managing a large or heavily customized `.env` file copied from `.env.example` **Usage** diff --git a/dify/code/dify-env-sync.py b/dify/code/dify-env-sync.py new file mode 100755 index 000000000..afa39d845 --- /dev/null +++ b/dify/code/dify-env-sync.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 + +# ================================================================ +# Dify Environment Variables Synchronization Script +# +# Features: +# - Synchronize latest settings from .env.example to .env +# - Preserve custom settings in existing .env +# - Add new environment variables +# - Detect removed environment variables +# - Create backup files +# ================================================================ + +import argparse +import re +import shutil +import sys +from datetime import datetime +from pathlib import Path + +# ANSI color codes +RED = "\033[0;31m" +GREEN = "\033[0;32m" +YELLOW = "\033[1;33m" +BLUE = "\033[0;34m" +NC = "\033[0m" # No Color + + +def supports_color() -> bool: + """Return True if the terminal supports ANSI color codes.""" + return hasattr(sys.stdout, "isatty") and sys.stdout.isatty() + + +def log_info(message: str) -> None: + """Print an informational message in blue.""" + if supports_color(): + print(f"{BLUE}[INFO]{NC} {message}") + else: + print(f"[INFO] {message}") + + +def log_success(message: str) -> None: + """Print a success message in green.""" + if supports_color(): + print(f"{GREEN}[SUCCESS]{NC} {message}") + else: + print(f"[SUCCESS] {message}") + + +def log_warning(message: str) -> None: + """Print a warning message in yellow to stderr.""" + if supports_color(): + print(f"{YELLOW}[WARNING]{NC} {message}", file=sys.stderr) + else: + print(f"[WARNING] {message}", file=sys.stderr) + + +def log_error(message: str) -> None: + """Print an error message in red to stderr.""" + if supports_color(): + print(f"{RED}[ERROR]{NC} {message}", file=sys.stderr) + else: + print(f"[ERROR] {message}", file=sys.stderr) + + +def parse_env_file(path: Path) -> dict[str, str]: + """Parse an .env-style file and return a mapping of key to raw value. + + Lines that are blank or start with '#' (after optional whitespace) are + skipped. Only lines containing '=' are considered variable definitions. + + Args: + path: Path to the .env file to parse. + + Returns: + Ordered dict mapping variable name to its value string. + """ + variables: dict[str, str] = {} + with path.open(encoding="utf-8") as fh: + for line in fh: + line = line.rstrip("\n") + # Skip blank lines and comment lines + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + if key: + variables[key] = value.strip() + return variables + + +def check_files(work_dir: Path) -> None: + """Verify required files exist; create .env from .env.example if absent. + + Args: + work_dir: Directory that must contain .env.example (and optionally .env). + + Raises: + SystemExit: If .env.example does not exist. + """ + log_info("Checking required files...") + + example_file = work_dir / ".env.example" + env_file = work_dir / ".env" + + if not example_file.exists(): + log_error(".env.example file not found") + sys.exit(1) + + if not env_file.exists(): + log_warning(".env file does not exist. Creating from .env.example.") + shutil.copy2(example_file, env_file) + log_success(".env file created") + + log_success("Required files verified") + + +def create_backup(work_dir: Path) -> None: + """Create a timestamped backup of the current .env file. + + Backups are placed in ``/env-backup/`` with the filename + ``.env.backup_``. + + Args: + work_dir: Directory containing the .env file to back up. + """ + env_file = work_dir / ".env" + if not env_file.exists(): + return + + backup_dir = work_dir / "env-backup" + if not backup_dir.exists(): + backup_dir.mkdir(parents=True) + log_info(f"Created backup directory: {backup_dir}") + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_file = backup_dir / f".env.backup_{timestamp}" + shutil.copy2(env_file, backup_file) + log_success(f"Backed up existing .env to {backup_file}") + + +def analyze_value_change(current: str, recommended: str) -> str | None: + """Analyse what kind of change occurred between two env values. + + Args: + current: Value currently set in .env. + recommended: Value present in .env.example. + + Returns: + A human-readable description string, or None when no analysis applies. + """ + use_colors = supports_color() + + def colorize(color: str, text: str) -> str: + return f"{color}{text}{NC}" if use_colors else text + + if not current and recommended: + return colorize(RED, " -> Setting from empty to recommended value") + if current and not recommended: + return colorize(RED, " -> Recommended value changed to empty") + + # Numeric comparison + if re.fullmatch(r"\d+", current) and re.fullmatch(r"\d+", recommended): + cur_int, rec_int = int(current), int(recommended) + if cur_int < rec_int: + return colorize(BLUE, f" -> Numeric increase ({current} < {recommended})") + if cur_int > rec_int: + return colorize(YELLOW, f" -> Numeric decrease ({current} > {recommended})") + return None + + # Boolean comparison + if current.lower() in {"true", "false"} and recommended.lower() in { + "true", + "false", + }: + if current.lower() != recommended.lower(): + return colorize(BLUE, f" -> Boolean value change ({current} -> {recommended})") + return None + + # URL / endpoint + if current.startswith(("http://", "https://")) or recommended.startswith(("http://", "https://")): + return colorize(BLUE, " -> URL/endpoint change") + + # File path + if current.startswith("/") or recommended.startswith("/"): + return colorize(BLUE, " -> File path change") + + # String length + if len(current) != len(recommended): + return colorize( + YELLOW, + f" -> String length change ({len(current)} -> {len(recommended)} characters)", + ) + + return None + + +def detect_differences(env_vars: dict[str, str], example_vars: dict[str, str]) -> dict[str, tuple[str, str]]: + """Find variables whose values differ between .env and .env.example. + + Only variables present in *both* files are compared; new or removed + variables are handled by separate functions. + + Args: + env_vars: Parsed key/value pairs from .env. + example_vars: Parsed key/value pairs from .env.example. + + Returns: + Mapping of key -> (env_value, example_value) for every key whose + values differ. + """ + log_info("Detecting differences between .env and .env.example...") + + diffs: dict[str, tuple[str, str]] = {} + for key, example_value in example_vars.items(): + if key in env_vars and env_vars[key] != example_value: + diffs[key] = (env_vars[key], example_value) + + if diffs: + log_success(f"Detected differences in {len(diffs)} environment variables") + show_differences_detail(diffs) + else: + log_info("No differences detected") + + return diffs + + +def show_differences_detail(diffs: dict[str, tuple[str, str]]) -> None: + """Print a formatted table of differing environment variables. + + Args: + diffs: Mapping of key -> (current_value, recommended_value). + """ + use_colors = supports_color() + + log_info("") + log_info("=== Environment Variable Differences ===") + + if not diffs: + log_info("No differences to display") + return + + for count, (key, (env_value, example_value)) in enumerate(diffs.items(), start=1): + print() + if use_colors: + print(f"{YELLOW}[{count}] {key}{NC}") + print(f" {GREEN}.env (current){NC} : {env_value}") + print(f" {BLUE}.env.example (recommended){NC} : {example_value}") + else: + print(f"[{count}] {key}") + print(f" .env (current) : {env_value}") + print(f" .env.example (recommended) : {example_value}") + + analysis = analyze_value_change(env_value, example_value) + if analysis: + print(analysis) + + print() + log_info("=== Difference Analysis Complete ===") + log_info("Note: Consider changing to the recommended values above.") + log_info("Current implementation preserves .env values.") + print() + + +def detect_removed_variables(env_vars: dict[str, str], example_vars: dict[str, str]) -> list[str]: + """Identify variables present in .env but absent from .env.example. + + Args: + env_vars: Parsed key/value pairs from .env. + example_vars: Parsed key/value pairs from .env.example. + + Returns: + Sorted list of variable names that no longer appear in .env.example. + """ + log_info("Detecting removed environment variables...") + + removed = sorted(set(env_vars) - set(example_vars)) + + if removed: + log_warning("The following environment variables have been removed from .env.example:") + for var in removed: + log_warning(f" - {var}") + log_warning("Consider manually removing these variables from .env") + else: + log_success("No removed environment variables found") + + return removed + + +def sync_env_file(work_dir: Path, env_vars: dict[str, str], diffs: dict[str, tuple[str, str]]) -> None: + """Rewrite .env based on .env.example while preserving custom values. + + The output file follows the exact line structure of .env.example + (preserving comments, blank lines, and ordering). For every variable + that exists in .env with a different value from the example, the + current .env value is kept. Variables that are new in .env.example + (not present in .env at all) are added with the example's default. + + Args: + work_dir: Directory containing .env and .env.example. + env_vars: Parsed key/value pairs from the original .env. + diffs: Keys whose .env values differ from .env.example (to preserve). + """ + log_info("Starting partial synchronization of .env file...") + + example_file = work_dir / ".env.example" + new_env_file = work_dir / ".env.new" + + # Keys whose current .env value should override the example default + preserved_keys: set[str] = set(diffs.keys()) + + preserved_count = 0 + updated_count = 0 + + env_var_pattern = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)\s*=") + + with ( + example_file.open(encoding="utf-8") as src, + new_env_file.open("w", encoding="utf-8") as dst, + ): + for line in src: + raw_line = line.rstrip("\n") + match = env_var_pattern.match(raw_line) + if match: + key = match.group(1) + if key in preserved_keys: + # Write the preserved value from .env + dst.write(f"{key}={env_vars[key]}\n") + log_info(f" Preserved: {key} (.env value)") + preserved_count += 1 + else: + # Use the example value (covers new vars and unchanged ones) + dst.write(line if line.endswith("\n") else raw_line + "\n") + updated_count += 1 + else: + # Blank line, comment, or non-variable line — keep as-is + dst.write(line if line.endswith("\n") else raw_line + "\n") + + # Atomically replace the original .env + try: + new_env_file.replace(work_dir / ".env") + except OSError as exc: + log_error(f"Failed to replace .env file: {exc}") + new_env_file.unlink(missing_ok=True) + sys.exit(1) + + log_success("Successfully created new .env file") + log_success("Partial synchronization of .env file completed") + log_info(f" Preserved .env values: {preserved_count}") + log_info(f" Updated to .env.example values: {updated_count}") + + +def show_statistics(work_dir: Path) -> None: + """Print a summary of variable counts from both env files. + + Args: + work_dir: Directory containing .env and .env.example. + """ + log_info("Synchronization statistics:") + + example_file = work_dir / ".env.example" + env_file = work_dir / ".env" + + example_count = len(parse_env_file(example_file)) if example_file.exists() else 0 + env_count = len(parse_env_file(env_file)) if env_file.exists() else 0 + + log_info(f" .env.example environment variables: {example_count}") + log_info(f" .env environment variables: {env_count}") + + +def build_arg_parser() -> argparse.ArgumentParser: + """Build and return the CLI argument parser. + + Returns: + Configured ArgumentParser instance. + """ + parser = argparse.ArgumentParser( + prog="dify-env-sync", + description=( + "Synchronize .env with .env.example: add new variables, " + "preserve custom values, and report removed variables." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "Examples:\n" + " # Run from the docker/ directory (default)\n" + " python dify-env-sync.py\n\n" + " # Specify a custom working directory\n" + " python dify-env-sync.py --dir /path/to/docker\n" + ), + ) + parser.add_argument( + "--dir", + metavar="DIRECTORY", + default=".", + help="Working directory containing .env and .env.example (default: current directory)", + ) + parser.add_argument( + "--no-backup", + action="store_true", + default=False, + help="Skip creating a timestamped backup of the existing .env file", + ) + return parser + + +def main() -> None: + """Orchestrate the complete environment variable synchronization process.""" + parser = build_arg_parser() + args = parser.parse_args() + + work_dir = Path(args.dir).resolve() + + log_info("=== Dify Environment Variables Synchronization Script ===") + log_info(f"Execution started: {datetime.now()}") + log_info(f"Working directory: {work_dir}") + + # 1. Verify prerequisites + check_files(work_dir) + + # 2. Backup existing .env + if not args.no_backup: + create_backup(work_dir) + + # 3. Parse both files + env_vars = parse_env_file(work_dir / ".env") + example_vars = parse_env_file(work_dir / ".env.example") + + # 4. Report differences (values that changed in the example) + diffs = detect_differences(env_vars, example_vars) + + # 5. Report variables removed from the example + detect_removed_variables(env_vars, example_vars) + + # 6. Rewrite .env + sync_env_file(work_dir, env_vars, diffs) + + # 7. Print summary statistics + show_statistics(work_dir) + + log_success("=== Synchronization process completed successfully ===") + log_info(f"Execution finished: {datetime.now()}") + + +if __name__ == "__main__": + main() diff --git a/dify/code/docker-compose-template.yaml b/dify/code/docker-compose-template.yaml index fcd480014..9d5018f6e 100644 --- a/dify/code/docker-compose-template.yaml +++ b/dify/code/docker-compose-template.yaml @@ -1,4 +1,202 @@ -x-shared-env: &shared-api-worker-env +# Shared configuration using YAML anchors and env_file +x-shared-api-worker-config: &shared-api-worker-config + env_file: + - path: ./envs/core-services/shared.env + required: false + - path: ./envs/core-services/api.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/db-postgres.env + required: false + - path: ./envs/databases/db-mysql.env + required: false + - path: ./envs/databases/redis.env + required: false + - path: ./envs/vectorstores/weaviate.env + required: false + - path: ./envs/vectorstores/qdrant.env + required: false + - path: ./envs/vectorstores/oceanbase.env + required: false + - path: ./envs/vectorstores/seekdb.env + required: false + - path: ./envs/vectorstores/couchbase.env + required: false + - path: ./envs/vectorstores/pgvector.env + required: false + - path: ./envs/vectorstores/vastbase.env + required: false + - path: ./envs/vectorstores/pgvecto-rs.env + required: false + - path: ./envs/vectorstores/chroma.env + required: false + - path: ./envs/vectorstores/iris.env + required: false + - path: ./envs/vectorstores/oracle.env + required: false + - path: ./envs/vectorstores/opengauss.env + required: false + - path: ./envs/vectorstores/myscale.env + required: false + - path: ./envs/vectorstores/matrixone.env + required: false + - path: ./envs/vectorstores/elasticsearch.env + required: false + - path: ./envs/vectorstores/opensearch.env + required: false + - path: ./envs/vectorstores/milvus.env + required: false + - path: ./envs/infrastructure/nginx.env + required: false + - path: ./envs/infrastructure/certbot.env + required: false + - path: ./envs/infrastructure/ssrf-proxy.env + required: false + - path: ./envs/infrastructure/etcd.env + required: false + - path: ./envs/infrastructure/minio.env + required: false + - path: ./envs/infrastructure/milvus-standalone.env + required: false + - ./.env + networks: + - ssrf_proxy_network + - default + restart: always + +x-shared-worker-config: &shared-worker-config + env_file: + - path: ./envs/core-services/shared.env + required: false + - path: ./envs/core-services/worker.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/db-postgres.env + required: false + - path: ./envs/databases/db-mysql.env + required: false + - path: ./envs/databases/redis.env + required: false + - path: ./envs/vectorstores/weaviate.env + required: false + - path: ./envs/vectorstores/qdrant.env + required: false + - path: ./envs/vectorstores/oceanbase.env + required: false + - path: ./envs/vectorstores/seekdb.env + required: false + - path: ./envs/vectorstores/couchbase.env + required: false + - path: ./envs/vectorstores/pgvector.env + required: false + - path: ./envs/vectorstores/vastbase.env + required: false + - path: ./envs/vectorstores/pgvecto-rs.env + required: false + - path: ./envs/vectorstores/chroma.env + required: false + - path: ./envs/vectorstores/iris.env + required: false + - path: ./envs/vectorstores/oracle.env + required: false + - path: ./envs/vectorstores/opengauss.env + required: false + - path: ./envs/vectorstores/myscale.env + required: false + - path: ./envs/vectorstores/matrixone.env + required: false + - path: ./envs/vectorstores/elasticsearch.env + required: false + - path: ./envs/vectorstores/opensearch.env + required: false + - path: ./envs/vectorstores/milvus.env + required: false + - path: ./envs/infrastructure/nginx.env + required: false + - path: ./envs/infrastructure/certbot.env + required: false + - path: ./envs/infrastructure/ssrf-proxy.env + required: false + - path: ./envs/infrastructure/etcd.env + required: false + - path: ./envs/infrastructure/minio.env + required: false + - path: ./envs/infrastructure/milvus-standalone.env + required: false + - ./.env + networks: + - ssrf_proxy_network + - default + restart: always + +x-shared-worker-beat-config: &shared-worker-beat-config + env_file: + - path: ./envs/core-services/shared.env + required: false + - path: ./envs/core-services/worker-beat.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/db-postgres.env + required: false + - path: ./envs/databases/db-mysql.env + required: false + - path: ./envs/databases/redis.env + required: false + - path: ./envs/vectorstores/weaviate.env + required: false + - path: ./envs/vectorstores/qdrant.env + required: false + - path: ./envs/vectorstores/oceanbase.env + required: false + - path: ./envs/vectorstores/seekdb.env + required: false + - path: ./envs/vectorstores/couchbase.env + required: false + - path: ./envs/vectorstores/pgvector.env + required: false + - path: ./envs/vectorstores/vastbase.env + required: false + - path: ./envs/vectorstores/pgvecto-rs.env + required: false + - path: ./envs/vectorstores/chroma.env + required: false + - path: ./envs/vectorstores/iris.env + required: false + - path: ./envs/vectorstores/oracle.env + required: false + - path: ./envs/vectorstores/opengauss.env + required: false + - path: ./envs/vectorstores/myscale.env + required: false + - path: ./envs/vectorstores/matrixone.env + required: false + - path: ./envs/vectorstores/elasticsearch.env + required: false + - path: ./envs/vectorstores/opensearch.env + required: false + - path: ./envs/vectorstores/milvus.env + required: false + - path: ./envs/infrastructure/nginx.env + required: false + - path: ./envs/infrastructure/certbot.env + required: false + - path: ./envs/infrastructure/ssrf-proxy.env + required: false + - path: ./envs/infrastructure/etcd.env + required: false + - path: ./envs/infrastructure/minio.env + required: false + - path: ./envs/infrastructure/milvus-standalone.env + required: false + - ./.env + networks: + - ssrf_proxy_network + - default + restart: always + services: # Init container to fix permissions init_permissions: @@ -21,12 +219,9 @@ services: # API service api: - image: langgenius/dify-api:1.13.0 - restart: always + <<: *shared-api-worker-config + image: langgenius/dify-api:1.14.1 environment: - # Use the shared environment variables. - <<: *shared-api-worker-env - # Startup mode, 'api' starts the API server. MODE: api SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} @@ -56,6 +251,37 @@ services: volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5001/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + networks: + - ssrf_proxy_network + - default + + # WebSocket service for workflow collaboration. + api_websocket: + <<: *shared-api-worker-config + image: langgenius/dify-api:1.14.1 + profiles: + - collaboration + environment: + MODE: api + SERVER_WORKER_AMOUNT: 1 + SERVER_WORKER_CLASS: ${API_WEBSOCKET_WORKER_CLASS:-geventwebsocket.gunicorn.workers.GeventWebSocketWorker} + SERVER_WORKER_CONNECTIONS: ${API_WEBSOCKET_WORKER_CONNECTIONS:-1000} + GUNICORN_TIMEOUT: ${API_WEBSOCKET_GUNICORN_TIMEOUT:-360} + depends_on: + db_postgres: + condition: service_healthy + required: false + db_mysql: + condition: service_healthy + required: false + redis: + condition: service_started networks: - ssrf_proxy_network - default @@ -63,12 +289,9 @@ services: # worker service # The Celery worker for processing all queues (dataset, workflow, mail, etc.) worker: - image: langgenius/dify-api:1.13.0 - restart: always + <<: *shared-worker-config + image: langgenius/dify-api:1.14.1 environment: - # Use the shared environment variables. - <<: *shared-api-worker-env - # Startup mode, 'worker' starts the Celery worker for processing all queues. MODE: worker SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} @@ -95,6 +318,13 @@ services: volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage + healthcheck: + test: ["CMD-SHELL", "celery -A celery_healthcheck.celery inspect ping"] + interval: ${COMPOSE_WORKER_HEALTHCHECK_INTERVAL:-30s} + timeout: ${COMPOSE_WORKER_HEALTHCHECK_TIMEOUT:-30s} + retries: 3 + start_period: 60s + disable: ${COMPOSE_WORKER_HEALTHCHECK_DISABLED:-true} networks: - ssrf_proxy_network - default @@ -102,12 +332,9 @@ services: # worker_beat service # Celery beat for scheduling periodic tasks. worker_beat: - image: langgenius/dify-api:1.13.0 - restart: always + <<: *shared-worker-beat-config + image: langgenius/dify-api:1.14.1 environment: - # Use the shared environment variables. - <<: *shared-api-worker-env - # Startup mode, 'worker_beat' starts the Celery beat for scheduling periodic tasks. MODE: beat depends_on: init_permissions: @@ -126,29 +353,45 @@ services: required: false redis: condition: service_started + healthcheck: + test: ["CMD-SHELL", "celery -A celery_healthcheck.celery inspect ping"] + interval: ${COMPOSE_WORKER_HEALTHCHECK_INTERVAL:-30s} + timeout: ${COMPOSE_WORKER_HEALTHCHECK_TIMEOUT:-30s} + retries: 3 + start_period: 60s + disable: ${COMPOSE_WORKER_HEALTHCHECK_DISABLED:-true} networks: - ssrf_proxy_network - default # Frontend web application. web: - image: langgenius/dify-web:1.13.0 + image: langgenius/dify-web:1.14.1 restart: always + env_file: + - path: ./envs/core-services/web.env + required: false + - path: ./envs/security.env + required: false + - ./.env environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} APP_API_URL: ${APP_API_URL:-} AMPLITUDE_API_KEY: ${AMPLITUDE_API_KEY:-} NEXT_PUBLIC_COOKIE_DOMAIN: ${NEXT_PUBLIC_COOKIE_DOMAIN:-} + NEXT_PUBLIC_SOCKET_URL: ${NEXT_PUBLIC_SOCKET_URL:-ws://localhost} SENTRY_DSN: ${WEB_SENTRY_DSN:-} NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0} + EXPERIMENTAL_ENABLE_VINEXT: ${EXPERIMENTAL_ENABLE_VINEXT:-false} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} CSP_WHITELIST: ${CSP_WHITELIST:-} ALLOW_EMBED: ${ALLOW_EMBED:-false} + ALLOW_INLINE_STYLES: ${ALLOW_INLINE_STYLES:-false} ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false} MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} - TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-} - INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-} + TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-10} + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} @@ -205,7 +448,7 @@ services: MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:-difyai123456} MYSQL_DATABASE: ${DB_DATABASE:-dify} command: > - --max_connections=1000 + --max_connections=${MYSQL_MAX_CONNECTIONS:-1000} --innodb_buffer_pool_size=${MYSQL_INNODB_BUFFER_POOL_SIZE:-512M} --innodb_log_file_size=${MYSQL_INNODB_LOG_FILE_SIZE:-128M} --innodb_flush_log_at_trx_commit=${MYSQL_INNODB_FLUSH_LOG_AT_TRX_COMMIT:-2} @@ -245,8 +488,14 @@ services: # The DifySandbox sandbox: - image: langgenius/dify-sandbox:0.2.12 + image: langgenius/dify-sandbox:0.2.15 restart: always + env_file: + - path: ./envs/core-services/sandbox.env + required: false + - path: ./envs/security.env + required: false + - ./.env environment: # The DifySandbox configurations # Make sure you are changing this key for your deployment with a strong key. @@ -269,12 +518,28 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.5.3-local + image: langgenius/dify-plugin-daemon:0.6.1-local restart: always + env_file: + - path: ./envs/core-services/shared.env + required: false + - path: ./envs/core-services/plugin-daemon.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/db-postgres.env + required: false + - path: ./envs/databases/db-mysql.env + required: false + - path: ./envs/databases/redis.env + required: false + - ./.env + networks: + - ssrf_proxy_network + - default environment: - # Use the shared environment variables. - <<: *shared-api-worker-env DB_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} + DB_SSL_MODE: ${DB_SSL_MODE:-disable} SERVER_PORT: ${PLUGIN_DAEMON_PORT:-5002} SERVER_KEY: ${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi} MAX_PLUGIN_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} @@ -378,8 +643,8 @@ services: - ./certbot/update-cert.template.txt:/update-cert.template.txt - ./certbot/docker-entrypoint.sh:/docker-entrypoint.sh environment: - - CERTBOT_EMAIL=${CERTBOT_EMAIL} - - CERTBOT_DOMAIN=${CERTBOT_DOMAIN} + - CERTBOT_EMAIL=${CERTBOT_EMAIL:-} + - CERTBOT_DOMAIN=${CERTBOT_DOMAIN:-} - CERTBOT_OPTIONS=${CERTBOT_OPTIONS:-} entrypoint: ["/docker-entrypoint.sh"] command: ["tail", "-f", "/dev/null"] @@ -421,6 +686,7 @@ services: NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s} NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s} NGINX_ENABLE_CERTBOT_CHALLENGE: ${NGINX_ENABLE_CERTBOT_CHALLENGE:-false} + NGINX_SOCKET_IO_UPSTREAM: ${NGINX_SOCKET_IO_UPSTREAM:-api_websocket:5001} CERTBOT_DOMAIN: ${CERTBOT_DOMAIN:-} depends_on: - api diff --git a/dify/code/docker-compose.middleware.yaml b/dify/code/docker-compose.middleware.yaml index 4a739bbbe..170e17185 100644 --- a/dify/code/docker-compose.middleware.yaml +++ b/dify/code/docker-compose.middleware.yaml @@ -51,7 +51,7 @@ services: MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:-difyai123456} MYSQL_DATABASE: ${DB_DATABASE:-dify} command: > - --max_connections=1000 + --max_connections=${MYSQL_MAX_CONNECTIONS:-1000} --innodb_buffer_pool_size=${MYSQL_INNODB_BUFFER_POOL_SIZE:-512M} --innodb_log_file_size=${MYSQL_INNODB_LOG_FILE_SIZE:-128M} --innodb_flush_log_at_trx_commit=${MYSQL_INNODB_FLUSH_LOG_AT_TRX_COMMIT:-2} @@ -59,19 +59,25 @@ services: - ${MYSQL_HOST_VOLUME:-./volumes/mysql/data}:/var/lib/mysql ports: - "${EXPOSE_MYSQL_PORT:-3306}:3306" + # mysqladmin ping passes during mysql:8.0's TCP-listening stage even while + # the server is still finalising init, leading to "Lost connection during + # query" on the first real query. Verify with a real SELECT instead. healthcheck: test: [ "CMD", - "mysqladmin", - "ping", - "-u", - "root", + "mysql", + "-h", + "127.0.0.1", + "-uroot", "-p${DB_PASSWORD:-difyai123456}", + "-e", + "SELECT 1", ] interval: 1s timeout: 3s retries: 30 + start_period: 20s # The redis cache. redis: @@ -97,7 +103,7 @@ services: # The DifySandbox sandbox: - image: langgenius/dify-sandbox:0.2.12 + image: langgenius/dify-sandbox:0.2.15 restart: always env_file: - ./middleware.env @@ -123,10 +129,12 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.5.3-local + image: langgenius/dify-plugin-daemon:0.6.1-local restart: always env_file: - ./middleware.env + extra_hosts: + - "host.docker.internal:host-gateway" environment: # Use the shared environment variables. LOG_OUTPUT_FORMAT: ${LOG_OUTPUT_FORMAT:-text} diff --git a/dify/code/docker-compose.yaml b/dify/code/docker-compose.yaml index 836ee763a..25b2229ad 100644 --- a/dify/code/docker-compose.yaml +++ b/dify/code/docker-compose.yaml @@ -4,696 +4,204 @@ # or docker-compose-template.yaml and regenerate this file. # ================================================================== -x-shared-env: &shared-api-worker-env - CONSOLE_API_URL: ${CONSOLE_API_URL:-} - CONSOLE_WEB_URL: ${CONSOLE_WEB_URL:-} - SERVICE_API_URL: ${SERVICE_API_URL:-} - TRIGGER_URL: ${TRIGGER_URL:-http://localhost} - APP_API_URL: ${APP_API_URL:-} - APP_WEB_URL: ${APP_WEB_URL:-} - FILES_URL: ${FILES_URL:-} - INTERNAL_FILES_URL: ${INTERNAL_FILES_URL:-} - LANG: ${LANG:-C.UTF-8} - LC_ALL: ${LC_ALL:-C.UTF-8} - PYTHONIOENCODING: ${PYTHONIOENCODING:-utf-8} - UV_CACHE_DIR: ${UV_CACHE_DIR:-/tmp/.uv-cache} - LOG_LEVEL: ${LOG_LEVEL:-INFO} - LOG_OUTPUT_FORMAT: ${LOG_OUTPUT_FORMAT:-text} - LOG_FILE: ${LOG_FILE:-/app/logs/server.log} - LOG_FILE_MAX_SIZE: ${LOG_FILE_MAX_SIZE:-20} - LOG_FILE_BACKUP_COUNT: ${LOG_FILE_BACKUP_COUNT:-5} - LOG_DATEFORMAT: ${LOG_DATEFORMAT:-%Y-%m-%d %H:%M:%S} - LOG_TZ: ${LOG_TZ:-UTC} - DEBUG: ${DEBUG:-false} - FLASK_DEBUG: ${FLASK_DEBUG:-false} - ENABLE_REQUEST_LOGGING: ${ENABLE_REQUEST_LOGGING:-False} - SECRET_KEY: ${SECRET_KEY:-sk-9f73s3ljTXVcMT3Blb3ljTqtsKiGHXVcMT3BlbkFJLK7U} - INIT_PASSWORD: ${INIT_PASSWORD:-} - DEPLOY_ENV: ${DEPLOY_ENV:-PRODUCTION} - CHECK_UPDATE_URL: ${CHECK_UPDATE_URL:-https://updates.dify.ai} - OPENAI_API_BASE: ${OPENAI_API_BASE:-https://api.openai.com/v1} - MIGRATION_ENABLED: ${MIGRATION_ENABLED:-true} - FILES_ACCESS_TIMEOUT: ${FILES_ACCESS_TIMEOUT:-300} - ACCESS_TOKEN_EXPIRE_MINUTES: ${ACCESS_TOKEN_EXPIRE_MINUTES:-60} - REFRESH_TOKEN_EXPIRE_DAYS: ${REFRESH_TOKEN_EXPIRE_DAYS:-30} - APP_DEFAULT_ACTIVE_REQUESTS: ${APP_DEFAULT_ACTIVE_REQUESTS:-0} - APP_MAX_ACTIVE_REQUESTS: ${APP_MAX_ACTIVE_REQUESTS:-0} - APP_MAX_EXECUTION_TIME: ${APP_MAX_EXECUTION_TIME:-1200} - DIFY_BIND_ADDRESS: ${DIFY_BIND_ADDRESS:-0.0.0.0} - DIFY_PORT: ${DIFY_PORT:-5001} - SERVER_WORKER_AMOUNT: ${SERVER_WORKER_AMOUNT:-1} - SERVER_WORKER_CLASS: ${SERVER_WORKER_CLASS:-gevent} - SERVER_WORKER_CONNECTIONS: ${SERVER_WORKER_CONNECTIONS:-10} - CELERY_WORKER_CLASS: ${CELERY_WORKER_CLASS:-} - GUNICORN_TIMEOUT: ${GUNICORN_TIMEOUT:-360} - CELERY_WORKER_AMOUNT: ${CELERY_WORKER_AMOUNT:-} - CELERY_AUTO_SCALE: ${CELERY_AUTO_SCALE:-false} - CELERY_MAX_WORKERS: ${CELERY_MAX_WORKERS:-} - CELERY_MIN_WORKERS: ${CELERY_MIN_WORKERS:-} - API_TOOL_DEFAULT_CONNECT_TIMEOUT: ${API_TOOL_DEFAULT_CONNECT_TIMEOUT:-10} - API_TOOL_DEFAULT_READ_TIMEOUT: ${API_TOOL_DEFAULT_READ_TIMEOUT:-60} - ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} - ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} - ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} - NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX: ${NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX:-false} - DB_TYPE: ${DB_TYPE:-postgresql} - DB_USERNAME: ${DB_USERNAME:-postgres} - DB_PASSWORD: ${DB_PASSWORD:-difyai123456} - DB_HOST: ${DB_HOST:-db_postgres} - DB_PORT: ${DB_PORT:-5432} - DB_DATABASE: ${DB_DATABASE:-dify} - SQLALCHEMY_POOL_SIZE: ${SQLALCHEMY_POOL_SIZE:-30} - SQLALCHEMY_MAX_OVERFLOW: ${SQLALCHEMY_MAX_OVERFLOW:-10} - SQLALCHEMY_POOL_RECYCLE: ${SQLALCHEMY_POOL_RECYCLE:-3600} - SQLALCHEMY_ECHO: ${SQLALCHEMY_ECHO:-false} - SQLALCHEMY_POOL_PRE_PING: ${SQLALCHEMY_POOL_PRE_PING:-false} - SQLALCHEMY_POOL_USE_LIFO: ${SQLALCHEMY_POOL_USE_LIFO:-false} - SQLALCHEMY_POOL_TIMEOUT: ${SQLALCHEMY_POOL_TIMEOUT:-30} - POSTGRES_MAX_CONNECTIONS: ${POSTGRES_MAX_CONNECTIONS:-100} - POSTGRES_SHARED_BUFFERS: ${POSTGRES_SHARED_BUFFERS:-128MB} - POSTGRES_WORK_MEM: ${POSTGRES_WORK_MEM:-4MB} - POSTGRES_MAINTENANCE_WORK_MEM: ${POSTGRES_MAINTENANCE_WORK_MEM:-64MB} - POSTGRES_EFFECTIVE_CACHE_SIZE: ${POSTGRES_EFFECTIVE_CACHE_SIZE:-4096MB} - POSTGRES_STATEMENT_TIMEOUT: ${POSTGRES_STATEMENT_TIMEOUT:-0} - POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT: ${POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT:-0} - MYSQL_MAX_CONNECTIONS: ${MYSQL_MAX_CONNECTIONS:-1000} - MYSQL_INNODB_BUFFER_POOL_SIZE: ${MYSQL_INNODB_BUFFER_POOL_SIZE:-512M} - MYSQL_INNODB_LOG_FILE_SIZE: ${MYSQL_INNODB_LOG_FILE_SIZE:-128M} - MYSQL_INNODB_FLUSH_LOG_AT_TRX_COMMIT: ${MYSQL_INNODB_FLUSH_LOG_AT_TRX_COMMIT:-2} - REDIS_HOST: ${REDIS_HOST:-redis} - REDIS_PORT: ${REDIS_PORT:-6379} - REDIS_USERNAME: ${REDIS_USERNAME:-} - REDIS_PASSWORD: ${REDIS_PASSWORD:-difyai123456} - REDIS_USE_SSL: ${REDIS_USE_SSL:-false} - REDIS_SSL_CERT_REQS: ${REDIS_SSL_CERT_REQS:-CERT_NONE} - REDIS_SSL_CA_CERTS: ${REDIS_SSL_CA_CERTS:-} - REDIS_SSL_CERTFILE: ${REDIS_SSL_CERTFILE:-} - REDIS_SSL_KEYFILE: ${REDIS_SSL_KEYFILE:-} - REDIS_DB: ${REDIS_DB:-0} - REDIS_MAX_CONNECTIONS: ${REDIS_MAX_CONNECTIONS:-} - REDIS_USE_SENTINEL: ${REDIS_USE_SENTINEL:-false} - REDIS_SENTINELS: ${REDIS_SENTINELS:-} - REDIS_SENTINEL_SERVICE_NAME: ${REDIS_SENTINEL_SERVICE_NAME:-} - REDIS_SENTINEL_USERNAME: ${REDIS_SENTINEL_USERNAME:-} - REDIS_SENTINEL_PASSWORD: ${REDIS_SENTINEL_PASSWORD:-} - REDIS_SENTINEL_SOCKET_TIMEOUT: ${REDIS_SENTINEL_SOCKET_TIMEOUT:-0.1} - REDIS_USE_CLUSTERS: ${REDIS_USE_CLUSTERS:-false} - REDIS_CLUSTERS: ${REDIS_CLUSTERS:-} - REDIS_CLUSTERS_PASSWORD: ${REDIS_CLUSTERS_PASSWORD:-} - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://:difyai123456@redis:6379/1} - CELERY_BACKEND: ${CELERY_BACKEND:-redis} - BROKER_USE_SSL: ${BROKER_USE_SSL:-false} - CELERY_USE_SENTINEL: ${CELERY_USE_SENTINEL:-false} - CELERY_SENTINEL_MASTER_NAME: ${CELERY_SENTINEL_MASTER_NAME:-} - CELERY_SENTINEL_PASSWORD: ${CELERY_SENTINEL_PASSWORD:-} - CELERY_SENTINEL_SOCKET_TIMEOUT: ${CELERY_SENTINEL_SOCKET_TIMEOUT:-0.1} - CELERY_TASK_ANNOTATIONS: ${CELERY_TASK_ANNOTATIONS:-null} - WEB_API_CORS_ALLOW_ORIGINS: ${WEB_API_CORS_ALLOW_ORIGINS:-*} - CONSOLE_CORS_ALLOW_ORIGINS: ${CONSOLE_CORS_ALLOW_ORIGINS:-*} - COOKIE_DOMAIN: ${COOKIE_DOMAIN:-} - NEXT_PUBLIC_COOKIE_DOMAIN: ${NEXT_PUBLIC_COOKIE_DOMAIN:-} - NEXT_PUBLIC_BATCH_CONCURRENCY: ${NEXT_PUBLIC_BATCH_CONCURRENCY:-5} - STORAGE_TYPE: ${STORAGE_TYPE:-opendal} - OPENDAL_SCHEME: ${OPENDAL_SCHEME:-fs} - OPENDAL_FS_ROOT: ${OPENDAL_FS_ROOT:-storage} - CLICKZETTA_VOLUME_TYPE: ${CLICKZETTA_VOLUME_TYPE:-user} - CLICKZETTA_VOLUME_NAME: ${CLICKZETTA_VOLUME_NAME:-} - CLICKZETTA_VOLUME_TABLE_PREFIX: ${CLICKZETTA_VOLUME_TABLE_PREFIX:-dataset_} - CLICKZETTA_VOLUME_DIFY_PREFIX: ${CLICKZETTA_VOLUME_DIFY_PREFIX:-dify_km} - S3_ENDPOINT: ${S3_ENDPOINT:-} - S3_REGION: ${S3_REGION:-us-east-1} - S3_BUCKET_NAME: ${S3_BUCKET_NAME:-difyai} - S3_ACCESS_KEY: ${S3_ACCESS_KEY:-} - S3_SECRET_KEY: ${S3_SECRET_KEY:-} - S3_USE_AWS_MANAGED_IAM: ${S3_USE_AWS_MANAGED_IAM:-false} - ARCHIVE_STORAGE_ENABLED: ${ARCHIVE_STORAGE_ENABLED:-false} - ARCHIVE_STORAGE_ENDPOINT: ${ARCHIVE_STORAGE_ENDPOINT:-} - ARCHIVE_STORAGE_ARCHIVE_BUCKET: ${ARCHIVE_STORAGE_ARCHIVE_BUCKET:-} - ARCHIVE_STORAGE_EXPORT_BUCKET: ${ARCHIVE_STORAGE_EXPORT_BUCKET:-} - ARCHIVE_STORAGE_ACCESS_KEY: ${ARCHIVE_STORAGE_ACCESS_KEY:-} - ARCHIVE_STORAGE_SECRET_KEY: ${ARCHIVE_STORAGE_SECRET_KEY:-} - ARCHIVE_STORAGE_REGION: ${ARCHIVE_STORAGE_REGION:-auto} - AZURE_BLOB_ACCOUNT_NAME: ${AZURE_BLOB_ACCOUNT_NAME:-difyai} - AZURE_BLOB_ACCOUNT_KEY: ${AZURE_BLOB_ACCOUNT_KEY:-difyai} - AZURE_BLOB_CONTAINER_NAME: ${AZURE_BLOB_CONTAINER_NAME:-difyai-container} - AZURE_BLOB_ACCOUNT_URL: ${AZURE_BLOB_ACCOUNT_URL:-https://.blob.core.windows.net} - GOOGLE_STORAGE_BUCKET_NAME: ${GOOGLE_STORAGE_BUCKET_NAME:-your-bucket-name} - GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: ${GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64:-} - ALIYUN_OSS_BUCKET_NAME: ${ALIYUN_OSS_BUCKET_NAME:-your-bucket-name} - ALIYUN_OSS_ACCESS_KEY: ${ALIYUN_OSS_ACCESS_KEY:-your-access-key} - ALIYUN_OSS_SECRET_KEY: ${ALIYUN_OSS_SECRET_KEY:-your-secret-key} - ALIYUN_OSS_ENDPOINT: ${ALIYUN_OSS_ENDPOINT:-https://oss-ap-southeast-1-internal.aliyuncs.com} - ALIYUN_OSS_REGION: ${ALIYUN_OSS_REGION:-ap-southeast-1} - ALIYUN_OSS_AUTH_VERSION: ${ALIYUN_OSS_AUTH_VERSION:-v4} - ALIYUN_OSS_PATH: ${ALIYUN_OSS_PATH:-your-path} - ALIYUN_CLOUDBOX_ID: ${ALIYUN_CLOUDBOX_ID:-your-cloudbox-id} - TENCENT_COS_BUCKET_NAME: ${TENCENT_COS_BUCKET_NAME:-your-bucket-name} - TENCENT_COS_SECRET_KEY: ${TENCENT_COS_SECRET_KEY:-your-secret-key} - TENCENT_COS_SECRET_ID: ${TENCENT_COS_SECRET_ID:-your-secret-id} - TENCENT_COS_REGION: ${TENCENT_COS_REGION:-your-region} - TENCENT_COS_SCHEME: ${TENCENT_COS_SCHEME:-your-scheme} - TENCENT_COS_CUSTOM_DOMAIN: ${TENCENT_COS_CUSTOM_DOMAIN:-your-custom-domain} - OCI_ENDPOINT: ${OCI_ENDPOINT:-https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com} - OCI_BUCKET_NAME: ${OCI_BUCKET_NAME:-your-bucket-name} - OCI_ACCESS_KEY: ${OCI_ACCESS_KEY:-your-access-key} - OCI_SECRET_KEY: ${OCI_SECRET_KEY:-your-secret-key} - OCI_REGION: ${OCI_REGION:-us-ashburn-1} - HUAWEI_OBS_BUCKET_NAME: ${HUAWEI_OBS_BUCKET_NAME:-your-bucket-name} - HUAWEI_OBS_SECRET_KEY: ${HUAWEI_OBS_SECRET_KEY:-your-secret-key} - HUAWEI_OBS_ACCESS_KEY: ${HUAWEI_OBS_ACCESS_KEY:-your-access-key} - HUAWEI_OBS_SERVER: ${HUAWEI_OBS_SERVER:-your-server-url} - HUAWEI_OBS_PATH_STYLE: ${HUAWEI_OBS_PATH_STYLE:-false} - VOLCENGINE_TOS_BUCKET_NAME: ${VOLCENGINE_TOS_BUCKET_NAME:-your-bucket-name} - VOLCENGINE_TOS_SECRET_KEY: ${VOLCENGINE_TOS_SECRET_KEY:-your-secret-key} - VOLCENGINE_TOS_ACCESS_KEY: ${VOLCENGINE_TOS_ACCESS_KEY:-your-access-key} - VOLCENGINE_TOS_ENDPOINT: ${VOLCENGINE_TOS_ENDPOINT:-your-server-url} - VOLCENGINE_TOS_REGION: ${VOLCENGINE_TOS_REGION:-your-region} - BAIDU_OBS_BUCKET_NAME: ${BAIDU_OBS_BUCKET_NAME:-your-bucket-name} - BAIDU_OBS_SECRET_KEY: ${BAIDU_OBS_SECRET_KEY:-your-secret-key} - BAIDU_OBS_ACCESS_KEY: ${BAIDU_OBS_ACCESS_KEY:-your-access-key} - BAIDU_OBS_ENDPOINT: ${BAIDU_OBS_ENDPOINT:-your-server-url} - SUPABASE_BUCKET_NAME: ${SUPABASE_BUCKET_NAME:-your-bucket-name} - SUPABASE_API_KEY: ${SUPABASE_API_KEY:-your-access-key} - SUPABASE_URL: ${SUPABASE_URL:-your-server-url} - VECTOR_STORE: ${VECTOR_STORE:-weaviate} - VECTOR_INDEX_NAME_PREFIX: ${VECTOR_INDEX_NAME_PREFIX:-Vector_index} - WEAVIATE_ENDPOINT: ${WEAVIATE_ENDPOINT:-http://weaviate:8080} - WEAVIATE_API_KEY: ${WEAVIATE_API_KEY:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} - WEAVIATE_GRPC_ENDPOINT: ${WEAVIATE_GRPC_ENDPOINT:-grpc://weaviate:50051} - WEAVIATE_TOKENIZATION: ${WEAVIATE_TOKENIZATION:-word} - OCEANBASE_VECTOR_HOST: ${OCEANBASE_VECTOR_HOST:-oceanbase} - OCEANBASE_VECTOR_PORT: ${OCEANBASE_VECTOR_PORT:-2881} - OCEANBASE_VECTOR_USER: ${OCEANBASE_VECTOR_USER:-root@test} - OCEANBASE_VECTOR_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456} - OCEANBASE_VECTOR_DATABASE: ${OCEANBASE_VECTOR_DATABASE:-test} - OCEANBASE_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} - OCEANBASE_MEMORY_LIMIT: ${OCEANBASE_MEMORY_LIMIT:-6G} - OCEANBASE_ENABLE_HYBRID_SEARCH: ${OCEANBASE_ENABLE_HYBRID_SEARCH:-false} - OCEANBASE_FULLTEXT_PARSER: ${OCEANBASE_FULLTEXT_PARSER:-ik} - SEEKDB_MEMORY_LIMIT: ${SEEKDB_MEMORY_LIMIT:-2G} - QDRANT_URL: ${QDRANT_URL:-http://qdrant:6333} - QDRANT_API_KEY: ${QDRANT_API_KEY:-difyai123456} - QDRANT_CLIENT_TIMEOUT: ${QDRANT_CLIENT_TIMEOUT:-20} - QDRANT_GRPC_ENABLED: ${QDRANT_GRPC_ENABLED:-false} - QDRANT_GRPC_PORT: ${QDRANT_GRPC_PORT:-6334} - QDRANT_REPLICATION_FACTOR: ${QDRANT_REPLICATION_FACTOR:-1} - MILVUS_URI: ${MILVUS_URI:-http://host.docker.internal:19530} - MILVUS_DATABASE: ${MILVUS_DATABASE:-} - MILVUS_TOKEN: ${MILVUS_TOKEN:-} - MILVUS_USER: ${MILVUS_USER:-} - MILVUS_PASSWORD: ${MILVUS_PASSWORD:-} - MILVUS_ENABLE_HYBRID_SEARCH: ${MILVUS_ENABLE_HYBRID_SEARCH:-False} - MILVUS_ANALYZER_PARAMS: ${MILVUS_ANALYZER_PARAMS:-} - MYSCALE_HOST: ${MYSCALE_HOST:-myscale} - MYSCALE_PORT: ${MYSCALE_PORT:-8123} - MYSCALE_USER: ${MYSCALE_USER:-default} - MYSCALE_PASSWORD: ${MYSCALE_PASSWORD:-} - MYSCALE_DATABASE: ${MYSCALE_DATABASE:-dify} - MYSCALE_FTS_PARAMS: ${MYSCALE_FTS_PARAMS:-} - COUCHBASE_CONNECTION_STRING: ${COUCHBASE_CONNECTION_STRING:-couchbase://couchbase-server} - COUCHBASE_USER: ${COUCHBASE_USER:-Administrator} - COUCHBASE_PASSWORD: ${COUCHBASE_PASSWORD:-password} - COUCHBASE_BUCKET_NAME: ${COUCHBASE_BUCKET_NAME:-Embeddings} - COUCHBASE_SCOPE_NAME: ${COUCHBASE_SCOPE_NAME:-_default} - PGVECTOR_HOST: ${PGVECTOR_HOST:-pgvector} - PGVECTOR_PORT: ${PGVECTOR_PORT:-5432} - PGVECTOR_USER: ${PGVECTOR_USER:-postgres} - PGVECTOR_PASSWORD: ${PGVECTOR_PASSWORD:-difyai123456} - PGVECTOR_DATABASE: ${PGVECTOR_DATABASE:-dify} - PGVECTOR_MIN_CONNECTION: ${PGVECTOR_MIN_CONNECTION:-1} - PGVECTOR_MAX_CONNECTION: ${PGVECTOR_MAX_CONNECTION:-5} - PGVECTOR_PG_BIGM: ${PGVECTOR_PG_BIGM:-false} - PGVECTOR_PG_BIGM_VERSION: ${PGVECTOR_PG_BIGM_VERSION:-1.2-20240606} - VASTBASE_HOST: ${VASTBASE_HOST:-vastbase} - VASTBASE_PORT: ${VASTBASE_PORT:-5432} - VASTBASE_USER: ${VASTBASE_USER:-dify} - VASTBASE_PASSWORD: ${VASTBASE_PASSWORD:-Difyai123456} - VASTBASE_DATABASE: ${VASTBASE_DATABASE:-dify} - VASTBASE_MIN_CONNECTION: ${VASTBASE_MIN_CONNECTION:-1} - VASTBASE_MAX_CONNECTION: ${VASTBASE_MAX_CONNECTION:-5} - PGVECTO_RS_HOST: ${PGVECTO_RS_HOST:-pgvecto-rs} - PGVECTO_RS_PORT: ${PGVECTO_RS_PORT:-5432} - PGVECTO_RS_USER: ${PGVECTO_RS_USER:-postgres} - PGVECTO_RS_PASSWORD: ${PGVECTO_RS_PASSWORD:-difyai123456} - PGVECTO_RS_DATABASE: ${PGVECTO_RS_DATABASE:-dify} - ANALYTICDB_KEY_ID: ${ANALYTICDB_KEY_ID:-your-ak} - ANALYTICDB_KEY_SECRET: ${ANALYTICDB_KEY_SECRET:-your-sk} - ANALYTICDB_REGION_ID: ${ANALYTICDB_REGION_ID:-cn-hangzhou} - ANALYTICDB_INSTANCE_ID: ${ANALYTICDB_INSTANCE_ID:-gp-ab123456} - ANALYTICDB_ACCOUNT: ${ANALYTICDB_ACCOUNT:-testaccount} - ANALYTICDB_PASSWORD: ${ANALYTICDB_PASSWORD:-testpassword} - ANALYTICDB_NAMESPACE: ${ANALYTICDB_NAMESPACE:-dify} - ANALYTICDB_NAMESPACE_PASSWORD: ${ANALYTICDB_NAMESPACE_PASSWORD:-difypassword} - ANALYTICDB_HOST: ${ANALYTICDB_HOST:-gp-test.aliyuncs.com} - ANALYTICDB_PORT: ${ANALYTICDB_PORT:-5432} - ANALYTICDB_MIN_CONNECTION: ${ANALYTICDB_MIN_CONNECTION:-1} - ANALYTICDB_MAX_CONNECTION: ${ANALYTICDB_MAX_CONNECTION:-5} - TIDB_VECTOR_HOST: ${TIDB_VECTOR_HOST:-tidb} - TIDB_VECTOR_PORT: ${TIDB_VECTOR_PORT:-4000} - TIDB_VECTOR_USER: ${TIDB_VECTOR_USER:-} - TIDB_VECTOR_PASSWORD: ${TIDB_VECTOR_PASSWORD:-} - TIDB_VECTOR_DATABASE: ${TIDB_VECTOR_DATABASE:-dify} - MATRIXONE_HOST: ${MATRIXONE_HOST:-matrixone} - MATRIXONE_PORT: ${MATRIXONE_PORT:-6001} - MATRIXONE_USER: ${MATRIXONE_USER:-dump} - MATRIXONE_PASSWORD: ${MATRIXONE_PASSWORD:-111} - MATRIXONE_DATABASE: ${MATRIXONE_DATABASE:-dify} - TIDB_ON_QDRANT_URL: ${TIDB_ON_QDRANT_URL:-http://127.0.0.1} - TIDB_ON_QDRANT_API_KEY: ${TIDB_ON_QDRANT_API_KEY:-dify} - TIDB_ON_QDRANT_CLIENT_TIMEOUT: ${TIDB_ON_QDRANT_CLIENT_TIMEOUT:-20} - TIDB_ON_QDRANT_GRPC_ENABLED: ${TIDB_ON_QDRANT_GRPC_ENABLED:-false} - TIDB_ON_QDRANT_GRPC_PORT: ${TIDB_ON_QDRANT_GRPC_PORT:-6334} - TIDB_PUBLIC_KEY: ${TIDB_PUBLIC_KEY:-dify} - TIDB_PRIVATE_KEY: ${TIDB_PRIVATE_KEY:-dify} - TIDB_API_URL: ${TIDB_API_URL:-http://127.0.0.1} - TIDB_IAM_API_URL: ${TIDB_IAM_API_URL:-http://127.0.0.1} - TIDB_REGION: ${TIDB_REGION:-regions/aws-us-east-1} - TIDB_PROJECT_ID: ${TIDB_PROJECT_ID:-dify} - TIDB_SPEND_LIMIT: ${TIDB_SPEND_LIMIT:-100} - CHROMA_HOST: ${CHROMA_HOST:-127.0.0.1} - CHROMA_PORT: ${CHROMA_PORT:-8000} - CHROMA_TENANT: ${CHROMA_TENANT:-default_tenant} - CHROMA_DATABASE: ${CHROMA_DATABASE:-default_database} - CHROMA_AUTH_PROVIDER: ${CHROMA_AUTH_PROVIDER:-chromadb.auth.token_authn.TokenAuthClientProvider} - CHROMA_AUTH_CREDENTIALS: ${CHROMA_AUTH_CREDENTIALS:-} - ORACLE_USER: ${ORACLE_USER:-dify} - ORACLE_PASSWORD: ${ORACLE_PASSWORD:-dify} - ORACLE_DSN: ${ORACLE_DSN:-oracle:1521/FREEPDB1} - ORACLE_CONFIG_DIR: ${ORACLE_CONFIG_DIR:-/app/api/storage/wallet} - ORACLE_WALLET_LOCATION: ${ORACLE_WALLET_LOCATION:-/app/api/storage/wallet} - ORACLE_WALLET_PASSWORD: ${ORACLE_WALLET_PASSWORD:-dify} - ORACLE_IS_AUTONOMOUS: ${ORACLE_IS_AUTONOMOUS:-false} - ALIBABACLOUD_MYSQL_HOST: ${ALIBABACLOUD_MYSQL_HOST:-127.0.0.1} - ALIBABACLOUD_MYSQL_PORT: ${ALIBABACLOUD_MYSQL_PORT:-3306} - ALIBABACLOUD_MYSQL_USER: ${ALIBABACLOUD_MYSQL_USER:-root} - ALIBABACLOUD_MYSQL_PASSWORD: ${ALIBABACLOUD_MYSQL_PASSWORD:-difyai123456} - ALIBABACLOUD_MYSQL_DATABASE: ${ALIBABACLOUD_MYSQL_DATABASE:-dify} - ALIBABACLOUD_MYSQL_MAX_CONNECTION: ${ALIBABACLOUD_MYSQL_MAX_CONNECTION:-5} - ALIBABACLOUD_MYSQL_HNSW_M: ${ALIBABACLOUD_MYSQL_HNSW_M:-6} - RELYT_HOST: ${RELYT_HOST:-db} - RELYT_PORT: ${RELYT_PORT:-5432} - RELYT_USER: ${RELYT_USER:-postgres} - RELYT_PASSWORD: ${RELYT_PASSWORD:-difyai123456} - RELYT_DATABASE: ${RELYT_DATABASE:-postgres} - OPENSEARCH_HOST: ${OPENSEARCH_HOST:-opensearch} - OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200} - OPENSEARCH_SECURE: ${OPENSEARCH_SECURE:-true} - OPENSEARCH_VERIFY_CERTS: ${OPENSEARCH_VERIFY_CERTS:-true} - OPENSEARCH_AUTH_METHOD: ${OPENSEARCH_AUTH_METHOD:-basic} - OPENSEARCH_USER: ${OPENSEARCH_USER:-admin} - OPENSEARCH_PASSWORD: ${OPENSEARCH_PASSWORD:-admin} - OPENSEARCH_AWS_REGION: ${OPENSEARCH_AWS_REGION:-ap-southeast-1} - OPENSEARCH_AWS_SERVICE: ${OPENSEARCH_AWS_SERVICE:-aoss} - TENCENT_VECTOR_DB_URL: ${TENCENT_VECTOR_DB_URL:-http://127.0.0.1} - TENCENT_VECTOR_DB_API_KEY: ${TENCENT_VECTOR_DB_API_KEY:-dify} - TENCENT_VECTOR_DB_TIMEOUT: ${TENCENT_VECTOR_DB_TIMEOUT:-30} - TENCENT_VECTOR_DB_USERNAME: ${TENCENT_VECTOR_DB_USERNAME:-dify} - TENCENT_VECTOR_DB_DATABASE: ${TENCENT_VECTOR_DB_DATABASE:-dify} - TENCENT_VECTOR_DB_SHARD: ${TENCENT_VECTOR_DB_SHARD:-1} - TENCENT_VECTOR_DB_REPLICAS: ${TENCENT_VECTOR_DB_REPLICAS:-2} - TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH: ${TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH:-false} - ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-0.0.0.0} - ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} - ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_USERNAME:-elastic} - ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic} - KIBANA_PORT: ${KIBANA_PORT:-5601} - ELASTICSEARCH_USE_CLOUD: ${ELASTICSEARCH_USE_CLOUD:-false} - ELASTICSEARCH_CLOUD_URL: ${ELASTICSEARCH_CLOUD_URL:-YOUR-ELASTICSEARCH_CLOUD_URL} - ELASTICSEARCH_API_KEY: ${ELASTICSEARCH_API_KEY:-YOUR-ELASTICSEARCH_API_KEY} - ELASTICSEARCH_VERIFY_CERTS: ${ELASTICSEARCH_VERIFY_CERTS:-False} - ELASTICSEARCH_CA_CERTS: ${ELASTICSEARCH_CA_CERTS:-} - ELASTICSEARCH_REQUEST_TIMEOUT: ${ELASTICSEARCH_REQUEST_TIMEOUT:-100000} - ELASTICSEARCH_RETRY_ON_TIMEOUT: ${ELASTICSEARCH_RETRY_ON_TIMEOUT:-True} - ELASTICSEARCH_MAX_RETRIES: ${ELASTICSEARCH_MAX_RETRIES:-10} - BAIDU_VECTOR_DB_ENDPOINT: ${BAIDU_VECTOR_DB_ENDPOINT:-http://127.0.0.1:5287} - BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS: ${BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS:-30000} - BAIDU_VECTOR_DB_ACCOUNT: ${BAIDU_VECTOR_DB_ACCOUNT:-root} - BAIDU_VECTOR_DB_API_KEY: ${BAIDU_VECTOR_DB_API_KEY:-dify} - BAIDU_VECTOR_DB_DATABASE: ${BAIDU_VECTOR_DB_DATABASE:-dify} - BAIDU_VECTOR_DB_SHARD: ${BAIDU_VECTOR_DB_SHARD:-1} - BAIDU_VECTOR_DB_REPLICAS: ${BAIDU_VECTOR_DB_REPLICAS:-3} - BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER: ${BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER:-DEFAULT_ANALYZER} - BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE: ${BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE:-COARSE_MODE} - VIKINGDB_ACCESS_KEY: ${VIKINGDB_ACCESS_KEY:-your-ak} - VIKINGDB_SECRET_KEY: ${VIKINGDB_SECRET_KEY:-your-sk} - VIKINGDB_REGION: ${VIKINGDB_REGION:-cn-shanghai} - VIKINGDB_HOST: ${VIKINGDB_HOST:-api-vikingdb.xxx.volces.com} - VIKINGDB_SCHEMA: ${VIKINGDB_SCHEMA:-http} - VIKINGDB_CONNECTION_TIMEOUT: ${VIKINGDB_CONNECTION_TIMEOUT:-30} - VIKINGDB_SOCKET_TIMEOUT: ${VIKINGDB_SOCKET_TIMEOUT:-30} - LINDORM_URL: ${LINDORM_URL:-http://localhost:30070} - LINDORM_USERNAME: ${LINDORM_USERNAME:-admin} - LINDORM_PASSWORD: ${LINDORM_PASSWORD:-admin} - LINDORM_USING_UGC: ${LINDORM_USING_UGC:-True} - LINDORM_QUERY_TIMEOUT: ${LINDORM_QUERY_TIMEOUT:-1} - OPENGAUSS_HOST: ${OPENGAUSS_HOST:-opengauss} - OPENGAUSS_PORT: ${OPENGAUSS_PORT:-6600} - OPENGAUSS_USER: ${OPENGAUSS_USER:-postgres} - OPENGAUSS_PASSWORD: ${OPENGAUSS_PASSWORD:-Dify@123} - OPENGAUSS_DATABASE: ${OPENGAUSS_DATABASE:-dify} - OPENGAUSS_MIN_CONNECTION: ${OPENGAUSS_MIN_CONNECTION:-1} - OPENGAUSS_MAX_CONNECTION: ${OPENGAUSS_MAX_CONNECTION:-5} - OPENGAUSS_ENABLE_PQ: ${OPENGAUSS_ENABLE_PQ:-false} - HUAWEI_CLOUD_HOSTS: ${HUAWEI_CLOUD_HOSTS:-https://127.0.0.1:9200} - HUAWEI_CLOUD_USER: ${HUAWEI_CLOUD_USER:-admin} - HUAWEI_CLOUD_PASSWORD: ${HUAWEI_CLOUD_PASSWORD:-admin} - UPSTASH_VECTOR_URL: ${UPSTASH_VECTOR_URL:-https://xxx-vector.upstash.io} - UPSTASH_VECTOR_TOKEN: ${UPSTASH_VECTOR_TOKEN:-dify} - TABLESTORE_ENDPOINT: ${TABLESTORE_ENDPOINT:-https://instance-name.cn-hangzhou.ots.aliyuncs.com} - TABLESTORE_INSTANCE_NAME: ${TABLESTORE_INSTANCE_NAME:-instance-name} - TABLESTORE_ACCESS_KEY_ID: ${TABLESTORE_ACCESS_KEY_ID:-xxx} - TABLESTORE_ACCESS_KEY_SECRET: ${TABLESTORE_ACCESS_KEY_SECRET:-xxx} - TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE: ${TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE:-false} - CLICKZETTA_USERNAME: ${CLICKZETTA_USERNAME:-} - CLICKZETTA_PASSWORD: ${CLICKZETTA_PASSWORD:-} - CLICKZETTA_INSTANCE: ${CLICKZETTA_INSTANCE:-} - CLICKZETTA_SERVICE: ${CLICKZETTA_SERVICE:-api.clickzetta.com} - CLICKZETTA_WORKSPACE: ${CLICKZETTA_WORKSPACE:-quick_start} - CLICKZETTA_VCLUSTER: ${CLICKZETTA_VCLUSTER:-default_ap} - CLICKZETTA_SCHEMA: ${CLICKZETTA_SCHEMA:-dify} - CLICKZETTA_BATCH_SIZE: ${CLICKZETTA_BATCH_SIZE:-100} - CLICKZETTA_ENABLE_INVERTED_INDEX: ${CLICKZETTA_ENABLE_INVERTED_INDEX:-true} - CLICKZETTA_ANALYZER_TYPE: ${CLICKZETTA_ANALYZER_TYPE:-chinese} - CLICKZETTA_ANALYZER_MODE: ${CLICKZETTA_ANALYZER_MODE:-smart} - CLICKZETTA_VECTOR_DISTANCE_FUNCTION: ${CLICKZETTA_VECTOR_DISTANCE_FUNCTION:-cosine_distance} - IRIS_HOST: ${IRIS_HOST:-iris} - IRIS_SUPER_SERVER_PORT: ${IRIS_SUPER_SERVER_PORT:-1972} - IRIS_WEB_SERVER_PORT: ${IRIS_WEB_SERVER_PORT:-52773} - IRIS_USER: ${IRIS_USER:-_SYSTEM} - IRIS_PASSWORD: ${IRIS_PASSWORD:-Dify@1234} - IRIS_DATABASE: ${IRIS_DATABASE:-USER} - IRIS_SCHEMA: ${IRIS_SCHEMA:-dify} - IRIS_CONNECTION_URL: ${IRIS_CONNECTION_URL:-} - IRIS_MIN_CONNECTION: ${IRIS_MIN_CONNECTION:-1} - IRIS_MAX_CONNECTION: ${IRIS_MAX_CONNECTION:-3} - IRIS_TEXT_INDEX: ${IRIS_TEXT_INDEX:-true} - IRIS_TEXT_INDEX_LANGUAGE: ${IRIS_TEXT_INDEX_LANGUAGE:-en} - IRIS_TIMEZONE: ${IRIS_TIMEZONE:-UTC} - UPLOAD_FILE_SIZE_LIMIT: ${UPLOAD_FILE_SIZE_LIMIT:-15} - UPLOAD_FILE_BATCH_LIMIT: ${UPLOAD_FILE_BATCH_LIMIT:-5} - UPLOAD_FILE_EXTENSION_BLACKLIST: ${UPLOAD_FILE_EXTENSION_BLACKLIST:-} - SINGLE_CHUNK_ATTACHMENT_LIMIT: ${SINGLE_CHUNK_ATTACHMENT_LIMIT:-10} - IMAGE_FILE_BATCH_LIMIT: ${IMAGE_FILE_BATCH_LIMIT:-10} - ATTACHMENT_IMAGE_FILE_SIZE_LIMIT: ${ATTACHMENT_IMAGE_FILE_SIZE_LIMIT:-2} - ATTACHMENT_IMAGE_DOWNLOAD_TIMEOUT: ${ATTACHMENT_IMAGE_DOWNLOAD_TIMEOUT:-60} - ETL_TYPE: ${ETL_TYPE:-dify} - UNSTRUCTURED_API_URL: ${UNSTRUCTURED_API_URL:-} - UNSTRUCTURED_API_KEY: ${UNSTRUCTURED_API_KEY:-} - SCARF_NO_ANALYTICS: ${SCARF_NO_ANALYTICS:-true} - PROMPT_GENERATION_MAX_TOKENS: ${PROMPT_GENERATION_MAX_TOKENS:-512} - CODE_GENERATION_MAX_TOKENS: ${CODE_GENERATION_MAX_TOKENS:-1024} - PLUGIN_BASED_TOKEN_COUNTING_ENABLED: ${PLUGIN_BASED_TOKEN_COUNTING_ENABLED:-false} - MULTIMODAL_SEND_FORMAT: ${MULTIMODAL_SEND_FORMAT:-base64} - UPLOAD_IMAGE_FILE_SIZE_LIMIT: ${UPLOAD_IMAGE_FILE_SIZE_LIMIT:-10} - UPLOAD_VIDEO_FILE_SIZE_LIMIT: ${UPLOAD_VIDEO_FILE_SIZE_LIMIT:-100} - UPLOAD_AUDIO_FILE_SIZE_LIMIT: ${UPLOAD_AUDIO_FILE_SIZE_LIMIT:-50} - SENTRY_DSN: ${SENTRY_DSN:-} - API_SENTRY_DSN: ${API_SENTRY_DSN:-} - API_SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} - API_SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} - WEB_SENTRY_DSN: ${WEB_SENTRY_DSN:-} - PLUGIN_SENTRY_ENABLED: ${PLUGIN_SENTRY_ENABLED:-false} - PLUGIN_SENTRY_DSN: ${PLUGIN_SENTRY_DSN:-} - NOTION_INTEGRATION_TYPE: ${NOTION_INTEGRATION_TYPE:-public} - NOTION_CLIENT_SECRET: ${NOTION_CLIENT_SECRET:-} - NOTION_CLIENT_ID: ${NOTION_CLIENT_ID:-} - NOTION_INTERNAL_SECRET: ${NOTION_INTERNAL_SECRET:-} - MAIL_TYPE: ${MAIL_TYPE:-resend} - MAIL_DEFAULT_SEND_FROM: ${MAIL_DEFAULT_SEND_FROM:-} - RESEND_API_URL: ${RESEND_API_URL:-https://api.resend.com} - RESEND_API_KEY: ${RESEND_API_KEY:-your-resend-api-key} - SMTP_SERVER: ${SMTP_SERVER:-} - SMTP_PORT: ${SMTP_PORT:-465} - SMTP_USERNAME: ${SMTP_USERNAME:-} - SMTP_PASSWORD: ${SMTP_PASSWORD:-} - SMTP_USE_TLS: ${SMTP_USE_TLS:-true} - SMTP_OPPORTUNISTIC_TLS: ${SMTP_OPPORTUNISTIC_TLS:-false} - SMTP_LOCAL_HOSTNAME: ${SMTP_LOCAL_HOSTNAME:-} - SENDGRID_API_KEY: ${SENDGRID_API_KEY:-} - INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} - INVITE_EXPIRY_HOURS: ${INVITE_EXPIRY_HOURS:-72} - RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: ${RESET_PASSWORD_TOKEN_EXPIRY_MINUTES:-5} - EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES: ${EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES:-5} - CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES: ${CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES:-5} - OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES: ${OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES:-5} - CODE_EXECUTION_ENDPOINT: ${CODE_EXECUTION_ENDPOINT:-http://sandbox:8194} - CODE_EXECUTION_API_KEY: ${CODE_EXECUTION_API_KEY:-dify-sandbox} - CODE_EXECUTION_SSL_VERIFY: ${CODE_EXECUTION_SSL_VERIFY:-True} - CODE_EXECUTION_POOL_MAX_CONNECTIONS: ${CODE_EXECUTION_POOL_MAX_CONNECTIONS:-100} - CODE_EXECUTION_POOL_MAX_KEEPALIVE_CONNECTIONS: ${CODE_EXECUTION_POOL_MAX_KEEPALIVE_CONNECTIONS:-20} - CODE_EXECUTION_POOL_KEEPALIVE_EXPIRY: ${CODE_EXECUTION_POOL_KEEPALIVE_EXPIRY:-5.0} - CODE_MAX_NUMBER: ${CODE_MAX_NUMBER:-9223372036854775807} - CODE_MIN_NUMBER: ${CODE_MIN_NUMBER:--9223372036854775808} - CODE_MAX_DEPTH: ${CODE_MAX_DEPTH:-5} - CODE_MAX_PRECISION: ${CODE_MAX_PRECISION:-20} - CODE_MAX_STRING_LENGTH: ${CODE_MAX_STRING_LENGTH:-400000} - CODE_MAX_STRING_ARRAY_LENGTH: ${CODE_MAX_STRING_ARRAY_LENGTH:-30} - CODE_MAX_OBJECT_ARRAY_LENGTH: ${CODE_MAX_OBJECT_ARRAY_LENGTH:-30} - CODE_MAX_NUMBER_ARRAY_LENGTH: ${CODE_MAX_NUMBER_ARRAY_LENGTH:-1000} - CODE_EXECUTION_CONNECT_TIMEOUT: ${CODE_EXECUTION_CONNECT_TIMEOUT:-10} - CODE_EXECUTION_READ_TIMEOUT: ${CODE_EXECUTION_READ_TIMEOUT:-60} - CODE_EXECUTION_WRITE_TIMEOUT: ${CODE_EXECUTION_WRITE_TIMEOUT:-10} - TEMPLATE_TRANSFORM_MAX_LENGTH: ${TEMPLATE_TRANSFORM_MAX_LENGTH:-400000} - WORKFLOW_MAX_EXECUTION_STEPS: ${WORKFLOW_MAX_EXECUTION_STEPS:-500} - WORKFLOW_MAX_EXECUTION_TIME: ${WORKFLOW_MAX_EXECUTION_TIME:-1200} - WORKFLOW_CALL_MAX_DEPTH: ${WORKFLOW_CALL_MAX_DEPTH:-5} - MAX_VARIABLE_SIZE: ${MAX_VARIABLE_SIZE:-204800} - WORKFLOW_FILE_UPLOAD_LIMIT: ${WORKFLOW_FILE_UPLOAD_LIMIT:-10} - GRAPH_ENGINE_MIN_WORKERS: ${GRAPH_ENGINE_MIN_WORKERS:-1} - GRAPH_ENGINE_MAX_WORKERS: ${GRAPH_ENGINE_MAX_WORKERS:-10} - GRAPH_ENGINE_SCALE_UP_THRESHOLD: ${GRAPH_ENGINE_SCALE_UP_THRESHOLD:-3} - GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME: ${GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME:-5.0} - WORKFLOW_NODE_EXECUTION_STORAGE: ${WORKFLOW_NODE_EXECUTION_STORAGE:-rdbms} - CORE_WORKFLOW_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository} - CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY:-core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository} - API_WORKFLOW_RUN_REPOSITORY: ${API_WORKFLOW_RUN_REPOSITORY:-repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository} - API_WORKFLOW_NODE_EXECUTION_REPOSITORY: ${API_WORKFLOW_NODE_EXECUTION_REPOSITORY:-repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository} - WORKFLOW_LOG_CLEANUP_ENABLED: ${WORKFLOW_LOG_CLEANUP_ENABLED:-false} - WORKFLOW_LOG_RETENTION_DAYS: ${WORKFLOW_LOG_RETENTION_DAYS:-30} - WORKFLOW_LOG_CLEANUP_BATCH_SIZE: ${WORKFLOW_LOG_CLEANUP_BATCH_SIZE:-100} - WORKFLOW_LOG_CLEANUP_SPECIFIC_WORKFLOW_IDS: ${WORKFLOW_LOG_CLEANUP_SPECIFIC_WORKFLOW_IDS:-} - ALIYUN_SLS_ACCESS_KEY_ID: ${ALIYUN_SLS_ACCESS_KEY_ID:-} - ALIYUN_SLS_ACCESS_KEY_SECRET: ${ALIYUN_SLS_ACCESS_KEY_SECRET:-} - ALIYUN_SLS_ENDPOINT: ${ALIYUN_SLS_ENDPOINT:-} - ALIYUN_SLS_REGION: ${ALIYUN_SLS_REGION:-} - ALIYUN_SLS_PROJECT_NAME: ${ALIYUN_SLS_PROJECT_NAME:-} - ALIYUN_SLS_LOGSTORE_TTL: ${ALIYUN_SLS_LOGSTORE_TTL:-365} - LOGSTORE_DUAL_WRITE_ENABLED: ${LOGSTORE_DUAL_WRITE_ENABLED:-false} - LOGSTORE_DUAL_READ_ENABLED: ${LOGSTORE_DUAL_READ_ENABLED:-true} - LOGSTORE_ENABLE_PUT_GRAPH_FIELD: ${LOGSTORE_ENABLE_PUT_GRAPH_FIELD:-true} - HTTP_REQUEST_NODE_MAX_BINARY_SIZE: ${HTTP_REQUEST_NODE_MAX_BINARY_SIZE:-10485760} - HTTP_REQUEST_NODE_MAX_TEXT_SIZE: ${HTTP_REQUEST_NODE_MAX_TEXT_SIZE:-1048576} - HTTP_REQUEST_NODE_SSL_VERIFY: ${HTTP_REQUEST_NODE_SSL_VERIFY:-True} - HTTP_REQUEST_MAX_CONNECT_TIMEOUT: ${HTTP_REQUEST_MAX_CONNECT_TIMEOUT:-10} - HTTP_REQUEST_MAX_READ_TIMEOUT: ${HTTP_REQUEST_MAX_READ_TIMEOUT:-600} - HTTP_REQUEST_MAX_WRITE_TIMEOUT: ${HTTP_REQUEST_MAX_WRITE_TIMEOUT:-600} - WEBHOOK_REQUEST_BODY_MAX_SIZE: ${WEBHOOK_REQUEST_BODY_MAX_SIZE:-10485760} - RESPECT_XFORWARD_HEADERS_ENABLED: ${RESPECT_XFORWARD_HEADERS_ENABLED:-false} - SSRF_PROXY_HTTP_URL: ${SSRF_PROXY_HTTP_URL:-http://ssrf_proxy:3128} - SSRF_PROXY_HTTPS_URL: ${SSRF_PROXY_HTTPS_URL:-http://ssrf_proxy:3128} - LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} - MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} - MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} - MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-99} - TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} - ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false} - MAX_TREE_DEPTH: ${MAX_TREE_DEPTH:-50} - PGDATA: ${PGDATA:-/var/lib/postgresql/data/pgdata} - MYSQL_HOST_VOLUME: ${MYSQL_HOST_VOLUME:-./volumes/mysql/data} - SANDBOX_API_KEY: ${SANDBOX_API_KEY:-dify-sandbox} - SANDBOX_GIN_MODE: ${SANDBOX_GIN_MODE:-release} - SANDBOX_WORKER_TIMEOUT: ${SANDBOX_WORKER_TIMEOUT:-15} - SANDBOX_ENABLE_NETWORK: ${SANDBOX_ENABLE_NETWORK:-true} - SANDBOX_HTTP_PROXY: ${SANDBOX_HTTP_PROXY:-http://ssrf_proxy:3128} - SANDBOX_HTTPS_PROXY: ${SANDBOX_HTTPS_PROXY:-http://ssrf_proxy:3128} - SANDBOX_PORT: ${SANDBOX_PORT:-8194} - WEAVIATE_PERSISTENCE_DATA_PATH: ${WEAVIATE_PERSISTENCE_DATA_PATH:-/var/lib/weaviate} - WEAVIATE_QUERY_DEFAULTS_LIMIT: ${WEAVIATE_QUERY_DEFAULTS_LIMIT:-25} - WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: ${WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED:-true} - WEAVIATE_DEFAULT_VECTORIZER_MODULE: ${WEAVIATE_DEFAULT_VECTORIZER_MODULE:-none} - WEAVIATE_CLUSTER_HOSTNAME: ${WEAVIATE_CLUSTER_HOSTNAME:-node1} - WEAVIATE_AUTHENTICATION_APIKEY_ENABLED: ${WEAVIATE_AUTHENTICATION_APIKEY_ENABLED:-true} - WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS: ${WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS:-WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih} - WEAVIATE_AUTHENTICATION_APIKEY_USERS: ${WEAVIATE_AUTHENTICATION_APIKEY_USERS:-hello@dify.ai} - WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED: ${WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED:-true} - WEAVIATE_AUTHORIZATION_ADMINLIST_USERS: ${WEAVIATE_AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai} - WEAVIATE_DISABLE_TELEMETRY: ${WEAVIATE_DISABLE_TELEMETRY:-false} - WEAVIATE_ENABLE_TOKENIZER_GSE: ${WEAVIATE_ENABLE_TOKENIZER_GSE:-false} - WEAVIATE_ENABLE_TOKENIZER_KAGOME_JA: ${WEAVIATE_ENABLE_TOKENIZER_KAGOME_JA:-false} - WEAVIATE_ENABLE_TOKENIZER_KAGOME_KR: ${WEAVIATE_ENABLE_TOKENIZER_KAGOME_KR:-false} - CHROMA_SERVER_AUTHN_CREDENTIALS: ${CHROMA_SERVER_AUTHN_CREDENTIALS:-difyai123456} - CHROMA_SERVER_AUTHN_PROVIDER: ${CHROMA_SERVER_AUTHN_PROVIDER:-chromadb.auth.token_authn.TokenAuthenticationServerProvider} - CHROMA_IS_PERSISTENT: ${CHROMA_IS_PERSISTENT:-TRUE} - ORACLE_PWD: ${ORACLE_PWD:-Dify123456} - ORACLE_CHARACTERSET: ${ORACLE_CHARACTERSET:-AL32UTF8} - ETCD_AUTO_COMPACTION_MODE: ${ETCD_AUTO_COMPACTION_MODE:-revision} - ETCD_AUTO_COMPACTION_RETENTION: ${ETCD_AUTO_COMPACTION_RETENTION:-1000} - ETCD_QUOTA_BACKEND_BYTES: ${ETCD_QUOTA_BACKEND_BYTES:-4294967296} - ETCD_SNAPSHOT_COUNT: ${ETCD_SNAPSHOT_COUNT:-50000} - MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin} - MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin} - ETCD_ENDPOINTS: ${ETCD_ENDPOINTS:-etcd:2379} - MINIO_ADDRESS: ${MINIO_ADDRESS:-minio:9000} - MILVUS_AUTHORIZATION_ENABLED: ${MILVUS_AUTHORIZATION_ENABLED:-true} - PGVECTOR_PGUSER: ${PGVECTOR_PGUSER:-postgres} - PGVECTOR_POSTGRES_PASSWORD: ${PGVECTOR_POSTGRES_PASSWORD:-difyai123456} - PGVECTOR_POSTGRES_DB: ${PGVECTOR_POSTGRES_DB:-dify} - PGVECTOR_PGDATA: ${PGVECTOR_PGDATA:-/var/lib/postgresql/data/pgdata} - OPENSEARCH_DISCOVERY_TYPE: ${OPENSEARCH_DISCOVERY_TYPE:-single-node} - OPENSEARCH_BOOTSTRAP_MEMORY_LOCK: ${OPENSEARCH_BOOTSTRAP_MEMORY_LOCK:-true} - OPENSEARCH_JAVA_OPTS_MIN: ${OPENSEARCH_JAVA_OPTS_MIN:-512m} - OPENSEARCH_JAVA_OPTS_MAX: ${OPENSEARCH_JAVA_OPTS_MAX:-1024m} - OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-Qazwsxedc!@#123} - OPENSEARCH_MEMLOCK_SOFT: ${OPENSEARCH_MEMLOCK_SOFT:--1} - OPENSEARCH_MEMLOCK_HARD: ${OPENSEARCH_MEMLOCK_HARD:--1} - OPENSEARCH_NOFILE_SOFT: ${OPENSEARCH_NOFILE_SOFT:-65536} - OPENSEARCH_NOFILE_HARD: ${OPENSEARCH_NOFILE_HARD:-65536} - NGINX_SERVER_NAME: ${NGINX_SERVER_NAME:-_} - NGINX_HTTPS_ENABLED: ${NGINX_HTTPS_ENABLED:-false} - NGINX_PORT: ${NGINX_PORT:-80} - NGINX_SSL_PORT: ${NGINX_SSL_PORT:-443} - NGINX_SSL_CERT_FILENAME: ${NGINX_SSL_CERT_FILENAME:-dify.crt} - NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key} - NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.2 TLSv1.3} - NGINX_WORKER_PROCESSES: ${NGINX_WORKER_PROCESSES:-auto} - NGINX_CLIENT_MAX_BODY_SIZE: ${NGINX_CLIENT_MAX_BODY_SIZE:-100M} - NGINX_KEEPALIVE_TIMEOUT: ${NGINX_KEEPALIVE_TIMEOUT:-65} - NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s} - NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s} - NGINX_ENABLE_CERTBOT_CHALLENGE: ${NGINX_ENABLE_CERTBOT_CHALLENGE:-false} - CERTBOT_EMAIL: ${CERTBOT_EMAIL:-your_email@example.com} - CERTBOT_DOMAIN: ${CERTBOT_DOMAIN:-your_domain.com} - CERTBOT_OPTIONS: ${CERTBOT_OPTIONS:-} - SSRF_HTTP_PORT: ${SSRF_HTTP_PORT:-3128} - SSRF_COREDUMP_DIR: ${SSRF_COREDUMP_DIR:-/var/spool/squid} - SSRF_REVERSE_PROXY_PORT: ${SSRF_REVERSE_PROXY_PORT:-8194} - SSRF_SANDBOX_HOST: ${SSRF_SANDBOX_HOST:-sandbox} - SSRF_DEFAULT_TIME_OUT: ${SSRF_DEFAULT_TIME_OUT:-5} - SSRF_DEFAULT_CONNECT_TIME_OUT: ${SSRF_DEFAULT_CONNECT_TIME_OUT:-5} - SSRF_DEFAULT_READ_TIME_OUT: ${SSRF_DEFAULT_READ_TIME_OUT:-5} - SSRF_DEFAULT_WRITE_TIME_OUT: ${SSRF_DEFAULT_WRITE_TIME_OUT:-5} - SSRF_POOL_MAX_CONNECTIONS: ${SSRF_POOL_MAX_CONNECTIONS:-100} - SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS: ${SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS:-20} - SSRF_POOL_KEEPALIVE_EXPIRY: ${SSRF_POOL_KEEPALIVE_EXPIRY:-5.0} - EXPOSE_NGINX_PORT: ${EXPOSE_NGINX_PORT:-80} - EXPOSE_NGINX_SSL_PORT: ${EXPOSE_NGINX_SSL_PORT:-443} - POSITION_TOOL_PINS: ${POSITION_TOOL_PINS:-} - POSITION_TOOL_INCLUDES: ${POSITION_TOOL_INCLUDES:-} - POSITION_TOOL_EXCLUDES: ${POSITION_TOOL_EXCLUDES:-} - POSITION_PROVIDER_PINS: ${POSITION_PROVIDER_PINS:-} - POSITION_PROVIDER_INCLUDES: ${POSITION_PROVIDER_INCLUDES:-} - POSITION_PROVIDER_EXCLUDES: ${POSITION_PROVIDER_EXCLUDES:-} - CSP_WHITELIST: ${CSP_WHITELIST:-} - CREATE_TIDB_SERVICE_JOB_ENABLED: ${CREATE_TIDB_SERVICE_JOB_ENABLED:-false} - MAX_SUBMIT_COUNT: ${MAX_SUBMIT_COUNT:-100} - TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-10} - DB_PLUGIN_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} - EXPOSE_PLUGIN_DAEMON_PORT: ${EXPOSE_PLUGIN_DAEMON_PORT:-5002} - PLUGIN_DAEMON_PORT: ${PLUGIN_DAEMON_PORT:-5002} - PLUGIN_DAEMON_KEY: ${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi} - PLUGIN_DAEMON_URL: ${PLUGIN_DAEMON_URL:-http://plugin_daemon:5002} - PLUGIN_MAX_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} - PLUGIN_MODEL_SCHEMA_CACHE_TTL: ${PLUGIN_MODEL_SCHEMA_CACHE_TTL:-3600} - PLUGIN_PPROF_ENABLED: ${PLUGIN_PPROF_ENABLED:-false} - PLUGIN_DEBUGGING_HOST: ${PLUGIN_DEBUGGING_HOST:-0.0.0.0} - PLUGIN_DEBUGGING_PORT: ${PLUGIN_DEBUGGING_PORT:-5003} - EXPOSE_PLUGIN_DEBUGGING_HOST: ${EXPOSE_PLUGIN_DEBUGGING_HOST:-localhost} - EXPOSE_PLUGIN_DEBUGGING_PORT: ${EXPOSE_PLUGIN_DEBUGGING_PORT:-5003} - PLUGIN_DIFY_INNER_API_KEY: ${PLUGIN_DIFY_INNER_API_KEY:-QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1} - PLUGIN_DIFY_INNER_API_URL: ${PLUGIN_DIFY_INNER_API_URL:-http://api:5001} - ENDPOINT_URL_TEMPLATE: ${ENDPOINT_URL_TEMPLATE:-http://localhost/e/{hook_id}} - MARKETPLACE_ENABLED: ${MARKETPLACE_ENABLED:-true} - MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} - FORCE_VERIFYING_SIGNATURE: ${FORCE_VERIFYING_SIGNATURE:-true} - ENFORCE_LANGGENIUS_PLUGIN_SIGNATURES: ${ENFORCE_LANGGENIUS_PLUGIN_SIGNATURES:-true} - PLUGIN_STDIO_BUFFER_SIZE: ${PLUGIN_STDIO_BUFFER_SIZE:-1024} - PLUGIN_STDIO_MAX_BUFFER_SIZE: ${PLUGIN_STDIO_MAX_BUFFER_SIZE:-5242880} - PLUGIN_PYTHON_ENV_INIT_TIMEOUT: ${PLUGIN_PYTHON_ENV_INIT_TIMEOUT:-120} - PLUGIN_MAX_EXECUTION_TIMEOUT: ${PLUGIN_MAX_EXECUTION_TIMEOUT:-600} - PLUGIN_DAEMON_TIMEOUT: ${PLUGIN_DAEMON_TIMEOUT:-600.0} - PIP_MIRROR_URL: ${PIP_MIRROR_URL:-} - PLUGIN_STORAGE_TYPE: ${PLUGIN_STORAGE_TYPE:-local} - PLUGIN_STORAGE_LOCAL_ROOT: ${PLUGIN_STORAGE_LOCAL_ROOT:-/app/storage} - PLUGIN_WORKING_PATH: ${PLUGIN_WORKING_PATH:-/app/storage/cwd} - PLUGIN_INSTALLED_PATH: ${PLUGIN_INSTALLED_PATH:-plugin} - PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} - PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} - PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} - PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} - PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} - PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} - PLUGIN_AWS_ACCESS_KEY: ${PLUGIN_AWS_ACCESS_KEY:-} - PLUGIN_AWS_SECRET_KEY: ${PLUGIN_AWS_SECRET_KEY:-} - PLUGIN_AWS_REGION: ${PLUGIN_AWS_REGION:-} - PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME: ${PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME:-} - PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING: ${PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING:-} - PLUGIN_TENCENT_COS_SECRET_KEY: ${PLUGIN_TENCENT_COS_SECRET_KEY:-} - PLUGIN_TENCENT_COS_SECRET_ID: ${PLUGIN_TENCENT_COS_SECRET_ID:-} - PLUGIN_TENCENT_COS_REGION: ${PLUGIN_TENCENT_COS_REGION:-} - PLUGIN_ALIYUN_OSS_REGION: ${PLUGIN_ALIYUN_OSS_REGION:-} - PLUGIN_ALIYUN_OSS_ENDPOINT: ${PLUGIN_ALIYUN_OSS_ENDPOINT:-} - PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID:-} - PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET: ${PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET:-} - PLUGIN_ALIYUN_OSS_AUTH_VERSION: ${PLUGIN_ALIYUN_OSS_AUTH_VERSION:-v4} - PLUGIN_ALIYUN_OSS_PATH: ${PLUGIN_ALIYUN_OSS_PATH:-} - PLUGIN_VOLCENGINE_TOS_ENDPOINT: ${PLUGIN_VOLCENGINE_TOS_ENDPOINT:-} - PLUGIN_VOLCENGINE_TOS_ACCESS_KEY: ${PLUGIN_VOLCENGINE_TOS_ACCESS_KEY:-} - PLUGIN_VOLCENGINE_TOS_SECRET_KEY: ${PLUGIN_VOLCENGINE_TOS_SECRET_KEY:-} - PLUGIN_VOLCENGINE_TOS_REGION: ${PLUGIN_VOLCENGINE_TOS_REGION:-} - ENABLE_OTEL: ${ENABLE_OTEL:-false} - OTLP_TRACE_ENDPOINT: ${OTLP_TRACE_ENDPOINT:-} - OTLP_METRIC_ENDPOINT: ${OTLP_METRIC_ENDPOINT:-} - OTLP_BASE_ENDPOINT: ${OTLP_BASE_ENDPOINT:-http://localhost:4318} - OTLP_API_KEY: ${OTLP_API_KEY:-} - OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-} - OTEL_EXPORTER_TYPE: ${OTEL_EXPORTER_TYPE:-otlp} - OTEL_SAMPLING_RATE: ${OTEL_SAMPLING_RATE:-0.1} - OTEL_BATCH_EXPORT_SCHEDULE_DELAY: ${OTEL_BATCH_EXPORT_SCHEDULE_DELAY:-5000} - OTEL_MAX_QUEUE_SIZE: ${OTEL_MAX_QUEUE_SIZE:-2048} - OTEL_MAX_EXPORT_BATCH_SIZE: ${OTEL_MAX_EXPORT_BATCH_SIZE:-512} - OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-60000} - OTEL_BATCH_EXPORT_TIMEOUT: ${OTEL_BATCH_EXPORT_TIMEOUT:-10000} - OTEL_METRIC_EXPORT_TIMEOUT: ${OTEL_METRIC_EXPORT_TIMEOUT:-30000} - ALLOW_EMBED: ${ALLOW_EMBED:-false} - QUEUE_MONITOR_THRESHOLD: ${QUEUE_MONITOR_THRESHOLD:-200} - QUEUE_MONITOR_ALERT_EMAILS: ${QUEUE_MONITOR_ALERT_EMAILS:-} - QUEUE_MONITOR_INTERVAL: ${QUEUE_MONITOR_INTERVAL:-30} - SWAGGER_UI_ENABLED: ${SWAGGER_UI_ENABLED:-false} - SWAGGER_UI_PATH: ${SWAGGER_UI_PATH:-/swagger-ui.html} - DSL_EXPORT_ENCRYPT_DATASET_ID: ${DSL_EXPORT_ENCRYPT_DATASET_ID:-true} - DATASET_MAX_SEGMENTS_PER_REQUEST: ${DATASET_MAX_SEGMENTS_PER_REQUEST:-0} - ENABLE_CLEAN_EMBEDDING_CACHE_TASK: ${ENABLE_CLEAN_EMBEDDING_CACHE_TASK:-false} - ENABLE_CLEAN_UNUSED_DATASETS_TASK: ${ENABLE_CLEAN_UNUSED_DATASETS_TASK:-false} - ENABLE_CREATE_TIDB_SERVERLESS_TASK: ${ENABLE_CREATE_TIDB_SERVERLESS_TASK:-false} - ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK: ${ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK:-false} - ENABLE_CLEAN_MESSAGES: ${ENABLE_CLEAN_MESSAGES:-false} - ENABLE_WORKFLOW_RUN_CLEANUP_TASK: ${ENABLE_WORKFLOW_RUN_CLEANUP_TASK:-false} - ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK: ${ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK:-false} - ENABLE_DATASETS_QUEUE_MONITOR: ${ENABLE_DATASETS_QUEUE_MONITOR:-false} - ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK: ${ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK:-true} - ENABLE_WORKFLOW_SCHEDULE_POLLER_TASK: ${ENABLE_WORKFLOW_SCHEDULE_POLLER_TASK:-true} - WORKFLOW_SCHEDULE_POLLER_INTERVAL: ${WORKFLOW_SCHEDULE_POLLER_INTERVAL:-1} - WORKFLOW_SCHEDULE_POLLER_BATCH_SIZE: ${WORKFLOW_SCHEDULE_POLLER_BATCH_SIZE:-100} - WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK: ${WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK:-0} - TENANT_ISOLATED_TASK_CONCURRENCY: ${TENANT_ISOLATED_TASK_CONCURRENCY:-1} - ANNOTATION_IMPORT_FILE_SIZE_LIMIT: ${ANNOTATION_IMPORT_FILE_SIZE_LIMIT:-2} - ANNOTATION_IMPORT_MAX_RECORDS: ${ANNOTATION_IMPORT_MAX_RECORDS:-10000} - ANNOTATION_IMPORT_MIN_RECORDS: ${ANNOTATION_IMPORT_MIN_RECORDS:-1} - ANNOTATION_IMPORT_RATE_LIMIT_PER_MINUTE: ${ANNOTATION_IMPORT_RATE_LIMIT_PER_MINUTE:-5} - ANNOTATION_IMPORT_RATE_LIMIT_PER_HOUR: ${ANNOTATION_IMPORT_RATE_LIMIT_PER_HOUR:-20} - ANNOTATION_IMPORT_MAX_CONCURRENT: ${ANNOTATION_IMPORT_MAX_CONCURRENT:-5} - AMPLITUDE_API_KEY: ${AMPLITUDE_API_KEY:-} - SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD: ${SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD:-21} - SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE: ${SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE:-1000} - SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL: ${SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL:-200} - SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS: ${SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS:-30} - PUBSUB_REDIS_URL: ${PUBSUB_REDIS_URL:-} - PUBSUB_REDIS_CHANNEL_TYPE: ${PUBSUB_REDIS_CHANNEL_TYPE:-pubsub} - PUBSUB_REDIS_USE_CLUSTERS: ${PUBSUB_REDIS_USE_CLUSTERS:-false} - ENABLE_HUMAN_INPUT_TIMEOUT_TASK: ${ENABLE_HUMAN_INPUT_TIMEOUT_TASK:-true} - HUMAN_INPUT_TIMEOUT_TASK_INTERVAL: ${HUMAN_INPUT_TIMEOUT_TASK_INTERVAL:-1} - SANDBOX_EXPIRED_RECORDS_CLEAN_TASK_LOCK_TTL: ${SANDBOX_EXPIRED_RECORDS_CLEAN_TASK_LOCK_TTL:-90000} +# Shared configuration using YAML anchors and env_file +x-shared-api-worker-config: &shared-api-worker-config + env_file: + - path: ./envs/core-services/shared.env + required: false + - path: ./envs/core-services/api.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/db-postgres.env + required: false + - path: ./envs/databases/db-mysql.env + required: false + - path: ./envs/databases/redis.env + required: false + - path: ./envs/vectorstores/weaviate.env + required: false + - path: ./envs/vectorstores/qdrant.env + required: false + - path: ./envs/vectorstores/oceanbase.env + required: false + - path: ./envs/vectorstores/seekdb.env + required: false + - path: ./envs/vectorstores/couchbase.env + required: false + - path: ./envs/vectorstores/pgvector.env + required: false + - path: ./envs/vectorstores/vastbase.env + required: false + - path: ./envs/vectorstores/pgvecto-rs.env + required: false + - path: ./envs/vectorstores/chroma.env + required: false + - path: ./envs/vectorstores/iris.env + required: false + - path: ./envs/vectorstores/oracle.env + required: false + - path: ./envs/vectorstores/opengauss.env + required: false + - path: ./envs/vectorstores/myscale.env + required: false + - path: ./envs/vectorstores/matrixone.env + required: false + - path: ./envs/vectorstores/elasticsearch.env + required: false + - path: ./envs/vectorstores/opensearch.env + required: false + - path: ./envs/vectorstores/milvus.env + required: false + - path: ./envs/infrastructure/nginx.env + required: false + - path: ./envs/infrastructure/certbot.env + required: false + - path: ./envs/infrastructure/ssrf-proxy.env + required: false + - path: ./envs/infrastructure/etcd.env + required: false + - path: ./envs/infrastructure/minio.env + required: false + - path: ./envs/infrastructure/milvus-standalone.env + required: false + - ./.env + networks: + - ssrf_proxy_network + - default + restart: always + +x-shared-worker-config: &shared-worker-config + env_file: + - path: ./envs/core-services/shared.env + required: false + - path: ./envs/core-services/worker.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/db-postgres.env + required: false + - path: ./envs/databases/db-mysql.env + required: false + - path: ./envs/databases/redis.env + required: false + - path: ./envs/vectorstores/weaviate.env + required: false + - path: ./envs/vectorstores/qdrant.env + required: false + - path: ./envs/vectorstores/oceanbase.env + required: false + - path: ./envs/vectorstores/seekdb.env + required: false + - path: ./envs/vectorstores/couchbase.env + required: false + - path: ./envs/vectorstores/pgvector.env + required: false + - path: ./envs/vectorstores/vastbase.env + required: false + - path: ./envs/vectorstores/pgvecto-rs.env + required: false + - path: ./envs/vectorstores/chroma.env + required: false + - path: ./envs/vectorstores/iris.env + required: false + - path: ./envs/vectorstores/oracle.env + required: false + - path: ./envs/vectorstores/opengauss.env + required: false + - path: ./envs/vectorstores/myscale.env + required: false + - path: ./envs/vectorstores/matrixone.env + required: false + - path: ./envs/vectorstores/elasticsearch.env + required: false + - path: ./envs/vectorstores/opensearch.env + required: false + - path: ./envs/vectorstores/milvus.env + required: false + - path: ./envs/infrastructure/nginx.env + required: false + - path: ./envs/infrastructure/certbot.env + required: false + - path: ./envs/infrastructure/ssrf-proxy.env + required: false + - path: ./envs/infrastructure/etcd.env + required: false + - path: ./envs/infrastructure/minio.env + required: false + - path: ./envs/infrastructure/milvus-standalone.env + required: false + - ./.env + networks: + - ssrf_proxy_network + - default + restart: always + +x-shared-worker-beat-config: &shared-worker-beat-config + env_file: + - path: ./envs/core-services/shared.env + required: false + - path: ./envs/core-services/worker-beat.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/db-postgres.env + required: false + - path: ./envs/databases/db-mysql.env + required: false + - path: ./envs/databases/redis.env + required: false + - path: ./envs/vectorstores/weaviate.env + required: false + - path: ./envs/vectorstores/qdrant.env + required: false + - path: ./envs/vectorstores/oceanbase.env + required: false + - path: ./envs/vectorstores/seekdb.env + required: false + - path: ./envs/vectorstores/couchbase.env + required: false + - path: ./envs/vectorstores/pgvector.env + required: false + - path: ./envs/vectorstores/vastbase.env + required: false + - path: ./envs/vectorstores/pgvecto-rs.env + required: false + - path: ./envs/vectorstores/chroma.env + required: false + - path: ./envs/vectorstores/iris.env + required: false + - path: ./envs/vectorstores/oracle.env + required: false + - path: ./envs/vectorstores/opengauss.env + required: false + - path: ./envs/vectorstores/myscale.env + required: false + - path: ./envs/vectorstores/matrixone.env + required: false + - path: ./envs/vectorstores/elasticsearch.env + required: false + - path: ./envs/vectorstores/opensearch.env + required: false + - path: ./envs/vectorstores/milvus.env + required: false + - path: ./envs/infrastructure/nginx.env + required: false + - path: ./envs/infrastructure/certbot.env + required: false + - path: ./envs/infrastructure/ssrf-proxy.env + required: false + - path: ./envs/infrastructure/etcd.env + required: false + - path: ./envs/infrastructure/minio.env + required: false + - path: ./envs/infrastructure/milvus-standalone.env + required: false + - ./.env + networks: + - ssrf_proxy_network + - default + restart: always services: # Init container to fix permissions @@ -717,12 +225,9 @@ services: # API service api: - image: langgenius/dify-api:1.13.0 - restart: always + <<: *shared-api-worker-config + image: langgenius/dify-api:1.14.1 environment: - # Use the shared environment variables. - <<: *shared-api-worker-env - # Startup mode, 'api' starts the API server. MODE: api SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} @@ -752,6 +257,37 @@ services: volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:5001/health" ] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + networks: + - ssrf_proxy_network + - default + + # WebSocket service for workflow collaboration. + api_websocket: + <<: *shared-api-worker-config + image: langgenius/dify-api:1.14.1 + profiles: + - collaboration + environment: + MODE: api + SERVER_WORKER_AMOUNT: 1 + SERVER_WORKER_CLASS: ${API_WEBSOCKET_WORKER_CLASS:-geventwebsocket.gunicorn.workers.GeventWebSocketWorker} + SERVER_WORKER_CONNECTIONS: ${API_WEBSOCKET_WORKER_CONNECTIONS:-1000} + GUNICORN_TIMEOUT: ${API_WEBSOCKET_GUNICORN_TIMEOUT:-360} + depends_on: + db_postgres: + condition: service_healthy + required: false + db_mysql: + condition: service_healthy + required: false + redis: + condition: service_started networks: - ssrf_proxy_network - default @@ -759,12 +295,9 @@ services: # worker service # The Celery worker for processing all queues (dataset, workflow, mail, etc.) worker: - image: langgenius/dify-api:1.13.0 - restart: always + <<: *shared-worker-config + image: langgenius/dify-api:1.14.1 environment: - # Use the shared environment variables. - <<: *shared-api-worker-env - # Startup mode, 'worker' starts the Celery worker for processing all queues. MODE: worker SENTRY_DSN: ${API_SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} @@ -791,6 +324,13 @@ services: volumes: # Mount the storage directory to the container, for storing user files. - ./volumes/app/storage:/app/api/storage + healthcheck: + test: [ "CMD-SHELL", "celery -A celery_healthcheck.celery inspect ping" ] + interval: ${COMPOSE_WORKER_HEALTHCHECK_INTERVAL:-30s} + timeout: ${COMPOSE_WORKER_HEALTHCHECK_TIMEOUT:-30s} + retries: 3 + start_period: 60s + disable: ${COMPOSE_WORKER_HEALTHCHECK_DISABLED:-true} networks: - ssrf_proxy_network - default @@ -798,12 +338,9 @@ services: # worker_beat service # Celery beat for scheduling periodic tasks. worker_beat: - image: langgenius/dify-api:1.13.0 - restart: always + <<: *shared-worker-beat-config + image: langgenius/dify-api:1.14.1 environment: - # Use the shared environment variables. - <<: *shared-api-worker-env - # Startup mode, 'worker_beat' starts the Celery beat for scheduling periodic tasks. MODE: beat depends_on: init_permissions: @@ -822,29 +359,45 @@ services: required: false redis: condition: service_started + healthcheck: + test: [ "CMD-SHELL", "celery -A celery_healthcheck.celery inspect ping" ] + interval: ${COMPOSE_WORKER_HEALTHCHECK_INTERVAL:-30s} + timeout: ${COMPOSE_WORKER_HEALTHCHECK_TIMEOUT:-30s} + retries: 3 + start_period: 60s + disable: ${COMPOSE_WORKER_HEALTHCHECK_DISABLED:-true} networks: - ssrf_proxy_network - default # Frontend web application. web: - image: langgenius/dify-web:1.13.0 + image: langgenius/dify-web:1.14.1 restart: always + env_file: + - path: ./envs/core-services/web.env + required: false + - path: ./envs/security.env + required: false + - ./.env environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} APP_API_URL: ${APP_API_URL:-} AMPLITUDE_API_KEY: ${AMPLITUDE_API_KEY:-} NEXT_PUBLIC_COOKIE_DOMAIN: ${NEXT_PUBLIC_COOKIE_DOMAIN:-} + NEXT_PUBLIC_SOCKET_URL: ${NEXT_PUBLIC_SOCKET_URL:-ws://localhost} SENTRY_DSN: ${WEB_SENTRY_DSN:-} NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0} + EXPERIMENTAL_ENABLE_VINEXT: ${EXPERIMENTAL_ENABLE_VINEXT:-false} TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000} CSP_WHITELIST: ${CSP_WHITELIST:-} ALLOW_EMBED: ${ALLOW_EMBED:-false} + ALLOW_INLINE_STYLES: ${ALLOW_INLINE_STYLES:-false} ALLOW_UNSAFE_DATA_SCHEME: ${ALLOW_UNSAFE_DATA_SCHEME:-false} MARKETPLACE_API_URL: ${MARKETPLACE_API_URL:-https://marketplace.dify.ai} MARKETPLACE_URL: ${MARKETPLACE_URL:-https://marketplace.dify.ai} - TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-} - INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-} + TOP_K_MAX_VALUE: ${TOP_K_MAX_VALUE:-10} + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: ${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH:-4000} LOOP_NODE_MAX_COUNT: ${LOOP_NODE_MAX_COUNT:-100} MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} @@ -901,7 +454,7 @@ services: MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:-difyai123456} MYSQL_DATABASE: ${DB_DATABASE:-dify} command: > - --max_connections=1000 + --max_connections=${MYSQL_MAX_CONNECTIONS:-1000} --innodb_buffer_pool_size=${MYSQL_INNODB_BUFFER_POOL_SIZE:-512M} --innodb_log_file_size=${MYSQL_INNODB_LOG_FILE_SIZE:-128M} --innodb_flush_log_at_trx_commit=${MYSQL_INNODB_FLUSH_LOG_AT_TRX_COMMIT:-2} @@ -941,8 +494,14 @@ services: # The DifySandbox sandbox: - image: langgenius/dify-sandbox:0.2.12 + image: langgenius/dify-sandbox:0.2.15 restart: always + env_file: + - path: ./envs/core-services/sandbox.env + required: false + - path: ./envs/security.env + required: false + - ./.env environment: # The DifySandbox configurations # Make sure you are changing this key for your deployment with a strong key. @@ -965,12 +524,28 @@ services: # plugin daemon plugin_daemon: - image: langgenius/dify-plugin-daemon:0.5.3-local + image: langgenius/dify-plugin-daemon:0.6.1-local restart: always + env_file: + - path: ./envs/core-services/shared.env + required: false + - path: ./envs/core-services/plugin-daemon.env + required: false + - path: ./envs/security.env + required: false + - path: ./envs/databases/db-postgres.env + required: false + - path: ./envs/databases/db-mysql.env + required: false + - path: ./envs/databases/redis.env + required: false + - ./.env + networks: + - ssrf_proxy_network + - default environment: - # Use the shared environment variables. - <<: *shared-api-worker-env DB_DATABASE: ${DB_PLUGIN_DATABASE:-dify_plugin} + DB_SSL_MODE: ${DB_SSL_MODE:-disable} SERVER_PORT: ${PLUGIN_DAEMON_PORT:-5002} SERVER_KEY: ${PLUGIN_DAEMON_KEY:-lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi} MAX_PLUGIN_PACKAGE_SIZE: ${PLUGIN_MAX_PACKAGE_SIZE:-52428800} @@ -1074,8 +649,8 @@ services: - ./certbot/update-cert.template.txt:/update-cert.template.txt - ./certbot/docker-entrypoint.sh:/docker-entrypoint.sh environment: - - CERTBOT_EMAIL=${CERTBOT_EMAIL} - - CERTBOT_DOMAIN=${CERTBOT_DOMAIN} + - CERTBOT_EMAIL=${CERTBOT_EMAIL:-} + - CERTBOT_DOMAIN=${CERTBOT_DOMAIN:-} - CERTBOT_OPTIONS=${CERTBOT_OPTIONS:-} entrypoint: [ "/docker-entrypoint.sh" ] command: [ "tail", "-f", "/dev/null" ] @@ -1119,6 +694,7 @@ services: NGINX_PROXY_READ_TIMEOUT: ${NGINX_PROXY_READ_TIMEOUT:-3600s} NGINX_PROXY_SEND_TIMEOUT: ${NGINX_PROXY_SEND_TIMEOUT:-3600s} NGINX_ENABLE_CERTBOT_CHALLENGE: ${NGINX_ENABLE_CERTBOT_CHALLENGE:-false} + NGINX_SOCKET_IO_UPSTREAM: ${NGINX_SOCKET_IO_UPSTREAM:-api_websocket:5001} CERTBOT_DOMAIN: ${CERTBOT_DOMAIN:-} depends_on: - api diff --git a/dify/code/envs/core-services/api.env.example b/dify/code/envs/core-services/api.env.example new file mode 100644 index 000000000..1a3fc7a4a --- /dev/null +++ b/dify/code/envs/core-services/api.env.example @@ -0,0 +1,13 @@ +# ------------------------------ +# Api Configuration +# ------------------------------ + +MODE=api +SENTRY_DSN= +SENTRY_TRACES_SAMPLE_RATE=1.0 +SENTRY_PROFILES_SAMPLE_RATE=1.0 +PLUGIN_REMOTE_INSTALL_HOST=localhost +PLUGIN_REMOTE_INSTALL_PORT=5003 +PLUGIN_MAX_PACKAGE_SIZE=52428800 +PLUGIN_DAEMON_TIMEOUT=600.0 +INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1 diff --git a/dify/code/envs/core-services/plugin-daemon.env.example b/dify/code/envs/core-services/plugin-daemon.env.example new file mode 100644 index 000000000..c3b1bef97 --- /dev/null +++ b/dify/code/envs/core-services/plugin-daemon.env.example @@ -0,0 +1,23 @@ +# ------------------------------ +# Plugin Daemon Configuration +# ------------------------------ + +DB_PLUGIN_DATABASE=dify_plugin +PLUGIN_DAEMON_URL=http://plugin_daemon:5002 +PLUGIN_PPROF_ENABLED=false +PLUGIN_DIFY_INNER_API_URL=http://api:5001 +FORCE_VERIFYING_SIGNATURE=true +PLUGIN_STDIO_BUFFER_SIZE=1024 +PLUGIN_STDIO_MAX_BUFFER_SIZE=5242880 +PLUGIN_PYTHON_ENV_INIT_TIMEOUT=120 +PLUGIN_MAX_EXECUTION_TIMEOUT=600 +PLUGIN_DEBUGGING_HOST=0.0.0.0 +PLUGIN_DEBUGGING_PORT=5003 +PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi +PLUGIN_DIFY_INNER_API_KEY=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1 +PLUGIN_DAEMON_PORT=5002 +CELERY_WORKER_CLASS= +PLUGIN_STORAGE_TYPE=local +PLUGIN_STORAGE_LOCAL_ROOT=/app/storage +PLUGIN_WORKING_PATH=/app/storage/cwd +PLUGIN_STORAGE_OSS_BUCKET= diff --git a/dify/code/envs/core-services/sandbox.env.example b/dify/code/envs/core-services/sandbox.env.example new file mode 100644 index 000000000..5d4ee6614 --- /dev/null +++ b/dify/code/envs/core-services/sandbox.env.example @@ -0,0 +1,17 @@ +# ------------------------------ +# Sandbox Configuration +# ------------------------------ + +SANDBOX_HTTP_PROXY=http://ssrf_proxy:3128 +SANDBOX_HTTPS_PROXY=http://ssrf_proxy:3128 +SANDBOX_PORT=8194 +PIP_MIRROR_URL= +SANDBOX_API_KEY=dify-sandbox +SANDBOX_GIN_MODE=release +SANDBOX_WORKER_TIMEOUT=15 +SANDBOX_ENABLE_NETWORK=true +SANDBOX_EXPIRED_RECORDS_CLEAN_GRACEFUL_PERIOD=21 +SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_SIZE=1000 +SANDBOX_EXPIRED_RECORDS_CLEAN_BATCH_MAX_INTERVAL=200 +SANDBOX_EXPIRED_RECORDS_RETENTION_DAYS=30 +SANDBOX_EXPIRED_RECORDS_CLEAN_TASK_LOCK_TTL=90000 diff --git a/dify/code/envs/core-services/shared.env.example b/dify/code/envs/core-services/shared.env.example new file mode 100644 index 000000000..fca0b57d0 --- /dev/null +++ b/dify/code/envs/core-services/shared.env.example @@ -0,0 +1,475 @@ +# ------------------------------ +# Shared API/Worker Configuration +# ------------------------------ + +CONSOLE_WEB_URL= +SERVICE_API_URL= +TRIGGER_URL=http://localhost +APP_WEB_URL= +FILES_URL= +INTERNAL_FILES_URL= +LANG=C.UTF-8 +LC_ALL=C.UTF-8 +PYTHONIOENCODING=utf-8 +UV_CACHE_DIR=/tmp/.uv-cache +CHECK_UPDATE_URL=https://updates.dify.ai +OPENAI_API_BASE=https://api.openai.com/v1 +MIGRATION_ENABLED=true +FILES_ACCESS_TIMEOUT=300 +# Remove `collaboration` from COMPOSE_PROFILES to stop the dedicated websocket service. +ENABLE_COLLABORATION_MODE=true +CELERY_BROKER_URL=redis://:difyai123456@redis:6379/1 +CELERY_TASK_ANNOTATIONS=null +AZURE_BLOB_ACCOUNT_URL=https://.blob.core.windows.net +SUPABASE_URL=your-server-url +TIDB_ON_QDRANT_URL=http://127.0.0.1 +TIDB_ON_QDRANT_API_KEY=dify +TIDB_API_URL=http://127.0.0.1 +TIDB_IAM_API_URL=http://127.0.0.1 +TIDB_REGION=regions/aws-us-east-1 +TIDB_PROJECT_ID=dify +TIDB_SPEND_LIMIT=100 +TENCENT_VECTOR_DB_URL=http://127.0.0.1 +TENCENT_VECTOR_DB_API_KEY=dify +LINDORM_URL=http://localhost:30070 +LINDORM_USERNAME=admin +UPSTASH_VECTOR_URL=https://xxx-vector.upstash.io +UPLOAD_FILE_SIZE_LIMIT=15 +UPLOAD_FILE_BATCH_LIMIT=5 +UPLOAD_FILE_EXTENSION_BLACKLIST= +SINGLE_CHUNK_ATTACHMENT_LIMIT=10 +IMAGE_FILE_BATCH_LIMIT=10 +ATTACHMENT_IMAGE_FILE_SIZE_LIMIT=2 +ATTACHMENT_IMAGE_DOWNLOAD_TIMEOUT=60 +ETL_TYPE=dify +UNSTRUCTURED_API_URL= +MULTIMODAL_SEND_FORMAT=base64 +UPLOAD_IMAGE_FILE_SIZE_LIMIT=10 +UPLOAD_VIDEO_FILE_SIZE_LIMIT=100 +UPLOAD_AUDIO_FILE_SIZE_LIMIT=50 +API_SENTRY_DSN= +API_SENTRY_TRACES_SAMPLE_RATE=1.0 +API_SENTRY_PROFILES_SAMPLE_RATE=1.0 +WEB_SENTRY_DSN= +PLUGIN_SENTRY_ENABLED=false +PLUGIN_SENTRY_DSN= +NOTION_INTEGRATION_TYPE=public +RESEND_API_URL=https://api.resend.com +SSRF_PROXY_HTTP_URL=http://ssrf_proxy:3128 +SSRF_PROXY_HTTPS_URL=http://ssrf_proxy:3128 +PGDATA=/var/lib/postgresql/data/pgdata +PLUGIN_MAX_PACKAGE_SIZE=52428800 +PLUGIN_MODEL_SCHEMA_CACHE_TTL=3600 +ENDPOINT_URL_TEMPLATE=http://localhost/e/{hook_id} +LOG_LEVEL=INFO +LOG_OUTPUT_FORMAT=text +LOG_FILE=/app/logs/server.log +LOG_FILE_MAX_SIZE=20 +LOG_FILE_BACKUP_COUNT=5 +LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S +LOG_TZ=UTC +DEBUG=false +FLASK_DEBUG=false +ENABLE_REQUEST_LOGGING=False +OPS_TRACE_RETRYABLE_DISPATCH_MAX_RETRIES=60 +OPS_TRACE_RETRYABLE_DISPATCH_DELAY_SECONDS=5 +WORKFLOW_LOG_CLEANUP_ENABLED=false +WORKFLOW_LOG_RETENTION_DAYS=30 +WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100 +WORKFLOW_LOG_CLEANUP_SPECIFIC_WORKFLOW_IDS= +EXPOSE_PLUGIN_DEBUGGING_HOST=localhost +EXPOSE_PLUGIN_DEBUGGING_PORT=5003 +DEPLOY_ENV=PRODUCTION +ACCESS_TOKEN_EXPIRE_MINUTES=60 +REFRESH_TOKEN_EXPIRE_DAYS=30 +APP_DEFAULT_ACTIVE_REQUESTS=0 +APP_MAX_ACTIVE_REQUESTS=0 +APP_MAX_EXECUTION_TIME=1200 +DIFY_BIND_ADDRESS=0.0.0.0 +DIFY_PORT=5001 +SERVER_WORKER_AMOUNT=1 +SERVER_WORKER_CLASS=gevent +SERVER_WORKER_CONNECTIONS=10 +API_WEBSOCKET_WORKER_CLASS=geventwebsocket.gunicorn.workers.GeventWebSocketWorker +API_WEBSOCKET_WORKER_CONNECTIONS=1000 +API_WEBSOCKET_GUNICORN_TIMEOUT=360 +CELERY_SENTINEL_PASSWORD= +S3_ACCESS_KEY= +S3_SECRET_KEY= +ARCHIVE_STORAGE_ACCESS_KEY= +ARCHIVE_STORAGE_SECRET_KEY= +AZURE_BLOB_ACCOUNT_KEY=difyai +ALIYUN_OSS_ACCESS_KEY=your-access-key +ALIYUN_OSS_SECRET_KEY=your-secret-key +TENCENT_COS_SECRET_KEY=your-secret-key +TENCENT_COS_SECRET_ID=your-secret-id +OCI_ACCESS_KEY=your-access-key +OCI_SECRET_KEY=your-secret-key +HUAWEI_OBS_SECRET_KEY=your-secret-key +HUAWEI_OBS_ACCESS_KEY=your-access-key +VOLCENGINE_TOS_SECRET_KEY=your-secret-key +VOLCENGINE_TOS_ACCESS_KEY=your-access-key +BAIDU_OBS_SECRET_KEY=your-secret-key +BAIDU_OBS_ACCESS_KEY=your-access-key +SUPABASE_API_KEY=your-access-key +ALIBABACLOUD_MYSQL_PASSWORD=difyai123456 +RELYT_PASSWORD=difyai123456 +LINDORM_PASSWORD=admin +LINDORM_USING_UGC=True +LINDORM_QUERY_TIMEOUT=1 +HUAWEI_CLOUD_PASSWORD=admin +UPSTASH_VECTOR_TOKEN=dify +TABLESTORE_ACCESS_KEY_ID=xxx +TABLESTORE_ACCESS_KEY_SECRET=xxx +TABLESTORE_NORMALIZE_FULLTEXT_BM25_SCORE=false +CLICKZETTA_PASSWORD= +CLICKZETTA_INSTANCE= +CLICKZETTA_SERVICE=api.clickzetta.com +CLICKZETTA_WORKSPACE=quick_start +CLICKZETTA_VCLUSTER=default_ap +CLICKZETTA_SCHEMA=dify +CLICKZETTA_BATCH_SIZE=100 +CLICKZETTA_ENABLE_INVERTED_INDEX=true +CLICKZETTA_ANALYZER_TYPE=chinese +CLICKZETTA_ANALYZER_MODE=smart +UNSTRUCTURED_API_KEY= +SCARF_NO_ANALYTICS=true +PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false +NOTION_CLIENT_SECRET= +NOTION_CLIENT_ID= +NOTION_INTERNAL_SECRET= +MAIL_TYPE=resend +MAIL_DEFAULT_SEND_FROM= +RESEND_API_KEY=your-resend-api-key +SMTP_SERVER= +SMTP_PORT=465 +SMTP_USERNAME= +SMTP_PASSWORD= +SMTP_USE_TLS=true +SMTP_OPPORTUNISTIC_TLS=false +SMTP_LOCAL_HOSTNAME= +SENDGRID_API_KEY= +INVITE_EXPIRY_HOURS=72 +RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5 +EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES=5 +CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5 +OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5 +CODE_EXECUTION_ENDPOINT=http://sandbox:8194 +CODE_EXECUTION_API_KEY=dify-sandbox +CODE_EXECUTION_SSL_VERIFY=True +CODE_EXECUTION_POOL_MAX_CONNECTIONS=100 +CODE_EXECUTION_POOL_MAX_KEEPALIVE_CONNECTIONS=20 +CODE_EXECUTION_POOL_KEEPALIVE_EXPIRY=5.0 +CODE_MAX_NUMBER=9223372036854775807 +CODE_MIN_NUMBER=-9223372036854775808 +CODE_MAX_DEPTH=5 +CODE_MAX_PRECISION=20 +CODE_MAX_STRING_LENGTH=400000 +CODE_MAX_STRING_ARRAY_LENGTH=30 +CODE_MAX_OBJECT_ARRAY_LENGTH=30 +CODE_MAX_NUMBER_ARRAY_LENGTH=1000 +CODE_EXECUTION_CONNECT_TIMEOUT=10 +CODE_EXECUTION_READ_TIMEOUT=60 +CODE_EXECUTION_WRITE_TIMEOUT=10 +TEMPLATE_TRANSFORM_MAX_LENGTH=400000 +WORKFLOW_MAX_EXECUTION_STEPS=500 +WORKFLOW_MAX_EXECUTION_TIME=1200 +WORKFLOW_CALL_MAX_DEPTH=5 +MAX_VARIABLE_SIZE=204800 +WORKFLOW_FILE_UPLOAD_LIMIT=10 +GRAPH_ENGINE_MIN_WORKERS=3 +GRAPH_ENGINE_MAX_WORKERS=10 +GRAPH_ENGINE_SCALE_UP_THRESHOLD=3 +GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME=5.0 +ALIYUN_SLS_ACCESS_KEY_ID= +ALIYUN_SLS_ACCESS_KEY_SECRET= +WEBHOOK_REQUEST_BODY_MAX_SIZE=10485760 +RESPECT_XFORWARD_HEADERS_ENABLED=false +SSRF_HTTP_PORT=3128 +SSRF_COREDUMP_DIR=/var/spool/squid +SSRF_REVERSE_PROXY_PORT=8194 +SSRF_SANDBOX_HOST=sandbox +SSRF_DEFAULT_TIME_OUT=5 +SSRF_DEFAULT_CONNECT_TIME_OUT=5 +SSRF_DEFAULT_READ_TIME_OUT=5 +SSRF_DEFAULT_WRITE_TIME_OUT=5 +SSRF_POOL_MAX_CONNECTIONS=100 +SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS=20 +SSRF_POOL_KEEPALIVE_EXPIRY=5.0 +PLUGIN_AWS_ACCESS_KEY= +PLUGIN_AWS_SECRET_KEY= +PLUGIN_AWS_REGION= +PLUGIN_TENCENT_COS_SECRET_KEY= +PLUGIN_TENCENT_COS_SECRET_ID= +PLUGIN_ALIYUN_OSS_ACCESS_KEY_ID= +PLUGIN_ALIYUN_OSS_ACCESS_KEY_SECRET= +PLUGIN_VOLCENGINE_TOS_ACCESS_KEY= +PLUGIN_VOLCENGINE_TOS_SECRET_KEY= +OTLP_API_KEY= +OTEL_EXPORTER_OTLP_PROTOCOL= +OTEL_EXPORTER_TYPE=otlp +OTEL_SAMPLING_RATE=0.1 +OTEL_BATCH_EXPORT_SCHEDULE_DELAY=5000 +OTEL_MAX_QUEUE_SIZE=2048 +OTEL_MAX_EXPORT_BATCH_SIZE=512 +OTEL_METRIC_EXPORT_INTERVAL=60000 +OTEL_BATCH_EXPORT_TIMEOUT=10000 +OTEL_METRIC_EXPORT_TIMEOUT=30000 +QUEUE_MONITOR_THRESHOLD=200 +QUEUE_MONITOR_ALERT_EMAILS= +QUEUE_MONITOR_INTERVAL=30 +SWAGGER_UI_ENABLED=false +SWAGGER_UI_PATH=/swagger-ui.html +DSL_EXPORT_ENCRYPT_DATASET_ID=true +DATASET_MAX_SEGMENTS_PER_REQUEST=0 +ENABLE_CLEAN_EMBEDDING_CACHE_TASK=false +ENABLE_CLEAN_UNUSED_DATASETS_TASK=false +ENABLE_CREATE_TIDB_SERVERLESS_TASK=false +ENABLE_UPDATE_TIDB_SERVERLESS_STATUS_TASK=false +ENABLE_CLEAN_MESSAGES=false +ENABLE_WORKFLOW_RUN_CLEANUP_TASK=false +ENABLE_MAIL_CLEAN_DOCUMENT_NOTIFY_TASK=false +ENABLE_DATASETS_QUEUE_MONITOR=false +ENABLE_CHECK_UPGRADABLE_PLUGIN_TASK=true +ENABLE_WORKFLOW_SCHEDULE_POLLER_TASK=true +WORKFLOW_SCHEDULE_POLLER_INTERVAL=1 +WORKFLOW_SCHEDULE_POLLER_BATCH_SIZE=100 +WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK=0 +TENANT_ISOLATED_TASK_CONCURRENCY=1 +ANNOTATION_IMPORT_FILE_SIZE_LIMIT=2 +ANNOTATION_IMPORT_MAX_RECORDS=10000 +ANNOTATION_IMPORT_MIN_RECORDS=1 +ANNOTATION_IMPORT_RATE_LIMIT_PER_MINUTE=5 +ANNOTATION_IMPORT_RATE_LIMIT_PER_HOUR=20 +ANNOTATION_IMPORT_MAX_CONCURRENT=5 +CREATORS_PLATFORM_FEATURES_ENABLED=true +CREATORS_PLATFORM_API_URL=https://creators.dify.ai +CREATORS_PLATFORM_OAUTH_CLIENT_ID= +TIDB_VECTOR_DATABASE=dify +ALIBABACLOUD_MYSQL_HOST=127.0.0.1 +ALIBABACLOUD_MYSQL_PORT=3306 +ALIBABACLOUD_MYSQL_USER=root +ALIBABACLOUD_MYSQL_DATABASE=dify +ALIBABACLOUD_MYSQL_MAX_CONNECTION=5 +ALIBABACLOUD_MYSQL_HNSW_M=6 +RELYT_DATABASE=postgres +TENCENT_VECTOR_DB_DATABASE=dify +BAIDU_VECTOR_DB_DATABASE=dify +EXPOSE_PLUGIN_DAEMON_PORT=5002 +GUNICORN_TIMEOUT=360 +CELERY_WORKER_AMOUNT= +CELERY_AUTO_SCALE=false +CELERY_MAX_WORKERS= +CELERY_MIN_WORKERS= +API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 +API_TOOL_DEFAULT_READ_TIMEOUT=60 +CELERY_BACKEND=redis +CELERY_USE_SENTINEL=false +CELERY_SENTINEL_MASTER_NAME= +CELERY_SENTINEL_SOCKET_TIMEOUT=0.1 +WEB_API_CORS_ALLOW_ORIGINS=* +CONSOLE_CORS_ALLOW_ORIGINS=* +COOKIE_DOMAIN= +OPENDAL_SCHEME=fs +OPENDAL_FS_ROOT=storage +CLICKZETTA_VOLUME_TYPE=user +CLICKZETTA_VOLUME_NAME= +CLICKZETTA_VOLUME_TABLE_PREFIX=dataset_ +CLICKZETTA_VOLUME_DIFY_PREFIX=dify_km +S3_ENDPOINT= +S3_REGION=us-east-1 +S3_BUCKET_NAME=difyai +S3_ADDRESS_STYLE=auto +S3_USE_AWS_MANAGED_IAM=false +ARCHIVE_STORAGE_ENABLED=false +ARCHIVE_STORAGE_ENDPOINT= +ARCHIVE_STORAGE_ARCHIVE_BUCKET= +ARCHIVE_STORAGE_EXPORT_BUCKET= +ARCHIVE_STORAGE_REGION=auto +AZURE_BLOB_ACCOUNT_NAME=difyai +AZURE_BLOB_CONTAINER_NAME=difyai-container +GOOGLE_STORAGE_BUCKET_NAME=your-bucket-name +GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64= +ALIYUN_OSS_BUCKET_NAME=your-bucket-name +ALIYUN_OSS_ENDPOINT=https://oss-ap-southeast-1-internal.aliyuncs.com +ALIYUN_OSS_REGION=ap-southeast-1 +ALIYUN_OSS_AUTH_VERSION=v4 +ALIYUN_OSS_PATH=your-path +ALIYUN_CLOUDBOX_ID=your-cloudbox-id +TENCENT_COS_BUCKET_NAME=your-bucket-name +TENCENT_COS_REGION=your-region +TENCENT_COS_SCHEME=your-scheme +TENCENT_COS_CUSTOM_DOMAIN=your-custom-domain +OCI_ENDPOINT=https://your-object-storage-namespace.compat.objectstorage.us-ashburn-1.oraclecloud.com +OCI_BUCKET_NAME=your-bucket-name +OCI_REGION=us-ashburn-1 +HUAWEI_OBS_BUCKET_NAME=your-bucket-name +HUAWEI_OBS_SERVER=your-server-url +HUAWEI_OBS_PATH_STYLE=false +VOLCENGINE_TOS_BUCKET_NAME=your-bucket-name +VOLCENGINE_TOS_ENDPOINT=your-server-url +VOLCENGINE_TOS_REGION=your-region +BAIDU_OBS_BUCKET_NAME=your-bucket-name +BAIDU_OBS_ENDPOINT=your-server-url +SUPABASE_BUCKET_NAME=your-bucket-name +TENCENT_VECTOR_DB_TIMEOUT=30 +TENCENT_VECTOR_DB_USERNAME=dify +TENCENT_VECTOR_DB_SHARD=1 +TENCENT_VECTOR_DB_REPLICAS=2 +TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH=false +BAIDU_VECTOR_DB_ENDPOINT=http://127.0.0.1:5287 +BAIDU_VECTOR_DB_CONNECTION_TIMEOUT_MS=30000 +BAIDU_VECTOR_DB_ACCOUNT=root +BAIDU_VECTOR_DB_API_KEY=dify +BAIDU_VECTOR_DB_SHARD=1 +BAIDU_VECTOR_DB_REPLICAS=3 +BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER=DEFAULT_ANALYZER +BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE=COARSE_MODE +BAIDU_VECTOR_DB_AUTO_BUILD_ROW_COUNT_INCREMENT=500 +BAIDU_VECTOR_DB_AUTO_BUILD_ROW_COUNT_INCREMENT_RATIO=0.05 +BAIDU_VECTOR_DB_REBUILD_INDEX_TIMEOUT_IN_SECONDS=300 +HUAWEI_CLOUD_HOSTS=https://127.0.0.1:9200 +HUAWEI_CLOUD_USER=admin +WORKFLOW_NODE_EXECUTION_STORAGE=rdbms +CORE_WORKFLOW_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository +CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY=core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository +API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository +API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node_execution_repository.DifyAPISQLAlchemyWorkflowNodeExecutionRepository +ALIYUN_SLS_ENDPOINT= +ALIYUN_SLS_REGION= +ALIYUN_SLS_PROJECT_NAME= +ALIYUN_SLS_LOGSTORE_TTL=365 +LOGSTORE_DUAL_WRITE_ENABLED=false +LOGSTORE_DUAL_READ_ENABLED=true +LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true +HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760 +HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576 +HTTP_REQUEST_NODE_SSL_VERIFY=True +HTTP_REQUEST_MAX_CONNECT_TIMEOUT=10 +HTTP_REQUEST_MAX_READ_TIMEOUT=600 +HTTP_REQUEST_MAX_WRITE_TIMEOUT=600 +PLUGIN_INSTALLED_PATH=plugin +PLUGIN_PACKAGE_CACHE_PATH=plugin_packages +PLUGIN_MEDIA_CACHE_PATH=assets +PLUGIN_S3_USE_AWS=false +PLUGIN_S3_USE_AWS_MANAGED_IAM=false +PLUGIN_S3_ENDPOINT= +PLUGIN_S3_USE_PATH_STYLE=false +PLUGIN_AZURE_BLOB_STORAGE_CONTAINER_NAME= +PLUGIN_AZURE_BLOB_STORAGE_CONNECTION_STRING= +PLUGIN_TENCENT_COS_REGION= +PLUGIN_ALIYUN_OSS_REGION= +PLUGIN_ALIYUN_OSS_ENDPOINT= +PLUGIN_ALIYUN_OSS_AUTH_VERSION=v4 +PLUGIN_ALIYUN_OSS_PATH= +PLUGIN_VOLCENGINE_TOS_ENDPOINT= +PLUGIN_VOLCENGINE_TOS_REGION= +ENABLE_OTEL=false +OTLP_TRACE_ENDPOINT= +OTLP_METRIC_ENDPOINT= +# Prefix used to create collection name in vector database +OTLP_BASE_ENDPOINT=http://localhost:4318 +WEAVIATE_GRPC_ENDPOINT=grpc://weaviate:50051 +ANALYTICDB_KEY_ID=your-ak +ANALYTICDB_KEY_SECRET=your-sk +ANALYTICDB_REGION_ID=cn-hangzhou +ANALYTICDB_INSTANCE_ID=gp-ab123456 +ANALYTICDB_ACCOUNT=testaccount +ANALYTICDB_PASSWORD=testpassword +ANALYTICDB_NAMESPACE=dify +ANALYTICDB_NAMESPACE_PASSWORD=difypassword +ANALYTICDB_HOST=gp-test.aliyuncs.com +ANALYTICDB_PORT=5432 +ANALYTICDB_MIN_CONNECTION=1 +ANALYTICDB_MAX_CONNECTION=5 +TIDB_VECTOR_HOST=tidb +TIDB_VECTOR_PORT=4000 +TIDB_VECTOR_USER= +TIDB_VECTOR_PASSWORD= +TIDB_ON_QDRANT_CLIENT_TIMEOUT=20 +TIDB_ON_QDRANT_GRPC_ENABLED=false +TIDB_ON_QDRANT_GRPC_PORT=6334 +TIDB_PUBLIC_KEY=dify +TIDB_PRIVATE_KEY=dify +RELYT_HOST=db +RELYT_PORT=5432 +RELYT_USER=postgres +VIKINGDB_ACCESS_KEY=your-ak +VIKINGDB_SECRET_KEY=your-sk +VIKINGDB_REGION=cn-shanghai +VIKINGDB_HOST=api-vikingdb.xxx.volces.com +VIKINGDB_SCHEME=http +VIKINGDB_CONNECTION_TIMEOUT=30 +VIKINGDB_SOCKET_TIMEOUT=30 +TABLESTORE_ENDPOINT=https://instance-name.cn-hangzhou.ots.aliyuncs.com +TABLESTORE_INSTANCE_NAME=instance-name +CLICKZETTA_USERNAME= +CLICKZETTA_VECTOR_DISTANCE_FUNCTION=cosine_distance +COMPOSE_PROFILES=${VECTOR_STORE:-weaviate},${DB_TYPE:-postgresql},collaboration +EXPOSE_NGINX_PORT=80 +EXPOSE_NGINX_SSL_PORT=443 +POSITION_TOOL_PINS= +POSITION_TOOL_INCLUDES= +POSITION_TOOL_EXCLUDES= +POSITION_PROVIDER_PINS= +POSITION_PROVIDER_INCLUDES= +POSITION_PROVIDER_EXCLUDES= +CREATE_TIDB_SERVICE_JOB_ENABLED=false +MAX_SUBMIT_COUNT=100 + +# Vector Store Configuration +STORAGE_TYPE=opendal +VECTOR_STORE=weaviate +VECTOR_INDEX_NAME_PREFIX=Vector_index +WEAVIATE_ENDPOINT=http://weaviate:8080 +WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih +WEAVIATE_TOKENIZATION=word +OCEANBASE_VECTOR_HOST=oceanbase +OCEANBASE_VECTOR_PORT=2881 +OCEANBASE_VECTOR_USER=root@test +OCEANBASE_VECTOR_PASSWORD=difyai123456 +OCEANBASE_VECTOR_DATABASE=test +OCEANBASE_ENABLE_HYBRID_SEARCH=false +OCEANBASE_FULLTEXT_PARSER=ik +SEEKDB_MEMORY_LIMIT=2G +QDRANT_URL=http://qdrant:6333 +QDRANT_API_KEY=difyai123456 +QDRANT_CLIENT_TIMEOUT=20 +QDRANT_GRPC_ENABLED=false +QDRANT_GRPC_PORT=6334 +QDRANT_REPLICATION_FACTOR=1 +MILVUS_URI=http://host.docker.internal:19530 +MILVUS_TOKEN= +MILVUS_USER= +MILVUS_PASSWORD= +MILVUS_ANALYZER_PARAMS= +PGVECTOR_HOST=pgvector +PGVECTOR_PORT=5432 +PGVECTOR_USER=postgres +PGVECTOR_PASSWORD=difyai123456 +PGVECTOR_DATABASE=dify +PGVECTOR_MIN_CONNECTION=1 +PGVECTOR_MAX_CONNECTION=5 +PGVECTOR_PG_BIGM=false +PGVECTOR_PG_BIGM_VERSION=1.2-20240606 + +# Hologres Configuration +HOLOGRES_HOST= +HOLOGRES_PORT=80 +HOLOGRES_DATABASE= +HOLOGRES_ACCESS_KEY_ID= +HOLOGRES_ACCESS_KEY_SECRET= +HOLOGRES_SCHEMA=public +HOLOGRES_TOKENIZER=jieba +HOLOGRES_DISTANCE_METHOD=Cosine +HOLOGRES_BASE_QUANTIZATION_TYPE=rabitq +HOLOGRES_MAX_DEGREE=64 +HOLOGRES_EF_CONSTRUCTION=400 + +# Milvus API Configuration +MILVUS_DATABASE= +MILVUS_ENABLE_HYBRID_SEARCH=False + +# Human Input Task Configuration +ENABLE_HUMAN_INPUT_TIMEOUT_TASK=true +HUMAN_INPUT_TIMEOUT_TASK_INTERVAL=1 diff --git a/dify/code/envs/core-services/web.env.example b/dify/code/envs/core-services/web.env.example new file mode 100644 index 000000000..d366cd87b --- /dev/null +++ b/dify/code/envs/core-services/web.env.example @@ -0,0 +1,30 @@ +# ------------------------------ +# Web Configuration +# ------------------------------ + +CONSOLE_API_URL= +APP_API_URL= +SENTRY_DSN= +NEXT_PUBLIC_SOCKET_URL=ws://localhost +EXPERIMENTAL_ENABLE_VINEXT=false +LOOP_NODE_MAX_COUNT=100 +MAX_TOOLS_NUM=10 +MAX_PARALLEL_LIMIT=10 +MAX_ITERATIONS_NUM=99 +TEXT_GENERATION_TIMEOUT_MS=60000 +ALLOW_INLINE_STYLES=false +ALLOW_UNSAFE_DATA_SCHEME=false +MAX_TREE_DEPTH=50 +MARKETPLACE_ENABLED=true +MARKETPLACE_API_URL=https://marketplace.dify.ai +INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000 +ALLOW_EMBED=false +AMPLITUDE_API_KEY= +ENABLE_WEBSITE_JINAREADER=true +ENABLE_WEBSITE_FIRECRAWL=true +ENABLE_WEBSITE_WATERCRAWL=true +NEXT_PUBLIC_ENABLE_SINGLE_DOLLAR_LATEX=false +NEXT_PUBLIC_COOKIE_DOMAIN= +NEXT_PUBLIC_BATCH_CONCURRENCY=5 +CSP_WHITELIST= +TOP_K_MAX_VALUE=10 diff --git a/dify/code/envs/core-services/worker-beat.env.example b/dify/code/envs/core-services/worker-beat.env.example new file mode 100644 index 000000000..380fe02b6 --- /dev/null +++ b/dify/code/envs/core-services/worker-beat.env.example @@ -0,0 +1,8 @@ +# ------------------------------ +# Worker Beat Configuration +# ------------------------------ + +MODE=beat +COMPOSE_WORKER_HEALTHCHECK_DISABLED=true +COMPOSE_WORKER_HEALTHCHECK_INTERVAL=30s +COMPOSE_WORKER_HEALTHCHECK_TIMEOUT=30s diff --git a/dify/code/envs/core-services/worker.env.example b/dify/code/envs/core-services/worker.env.example new file mode 100644 index 000000000..58cf4ea90 --- /dev/null +++ b/dify/code/envs/core-services/worker.env.example @@ -0,0 +1,13 @@ +# ------------------------------ +# Worker Configuration +# ------------------------------ + +MODE=worker +SENTRY_DSN= +SENTRY_TRACES_SAMPLE_RATE=1.0 +SENTRY_PROFILES_SAMPLE_RATE=1.0 +PLUGIN_MAX_PACKAGE_SIZE=52428800 +INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1 +COMPOSE_WORKER_HEALTHCHECK_DISABLED=true +COMPOSE_WORKER_HEALTHCHECK_INTERVAL=30s +COMPOSE_WORKER_HEALTHCHECK_TIMEOUT=30s diff --git a/dify/code/envs/databases/db-mysql.env.example b/dify/code/envs/databases/db-mysql.env.example new file mode 100644 index 000000000..b3ea6801f --- /dev/null +++ b/dify/code/envs/databases/db-mysql.env.example @@ -0,0 +1,9 @@ +# ------------------------------ +# Db Mysql Configuration +# ------------------------------ + +MYSQL_INNODB_LOG_FILE_SIZE=128M +MYSQL_INNODB_FLUSH_LOG_AT_TRX_COMMIT=2 +MYSQL_MAX_CONNECTIONS=1000 +MYSQL_INNODB_BUFFER_POOL_SIZE=512M +MYSQL_HOST_VOLUME=./volumes/mysql/data diff --git a/dify/code/envs/databases/db-postgres.env.example b/dify/code/envs/databases/db-postgres.env.example new file mode 100644 index 000000000..14cefb6be --- /dev/null +++ b/dify/code/envs/databases/db-postgres.env.example @@ -0,0 +1,26 @@ +# ------------------------------ +# Db Postgres Configuration +# ------------------------------ + +PGDATA=/var/lib/postgresql/data/pgdata +DB_TYPE=postgresql +DB_USERNAME=postgres +DB_PASSWORD=difyai123456 +DB_HOST=db_postgres +DB_PORT=5432 +DB_DATABASE=dify +SQLALCHEMY_POOL_SIZE=30 +SQLALCHEMY_MAX_OVERFLOW=10 +SQLALCHEMY_POOL_RECYCLE=3600 +SQLALCHEMY_ECHO=false +SQLALCHEMY_POOL_PRE_PING=false +SQLALCHEMY_POOL_USE_LIFO=false +SQLALCHEMY_POOL_TIMEOUT=30 +SQLALCHEMY_POOL_RESET_ON_RETURN=rollback +POSTGRES_MAX_CONNECTIONS=100 +POSTGRES_SHARED_BUFFERS=128MB +POSTGRES_WORK_MEM=4MB +POSTGRES_MAINTENANCE_WORK_MEM=64MB +POSTGRES_EFFECTIVE_CACHE_SIZE=4096MB +POSTGRES_STATEMENT_TIMEOUT=0 +POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=0 diff --git a/dify/code/envs/databases/redis.env.example b/dify/code/envs/databases/redis.env.example new file mode 100644 index 000000000..74bcb6525 --- /dev/null +++ b/dify/code/envs/databases/redis.env.example @@ -0,0 +1,35 @@ +# ------------------------------ +# Redis Configuration +# ------------------------------ + +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_USERNAME= +REDIS_PASSWORD=difyai123456 +REDIS_USE_SSL=false +REDIS_SSL_CERT_REQS=CERT_NONE +REDIS_SSL_CA_CERTS= +REDIS_SSL_CERTFILE= +REDIS_SSL_KEYFILE= +REDIS_DB=0 +REDIS_KEY_PREFIX= +REDIS_MAX_CONNECTIONS= +REDIS_USE_SENTINEL=false +REDIS_SENTINELS= +REDIS_SENTINEL_SERVICE_NAME= +REDIS_SENTINEL_USERNAME= +REDIS_SENTINEL_PASSWORD= +REDIS_SENTINEL_SOCKET_TIMEOUT=0.1 +REDIS_USE_CLUSTERS=false +REDIS_CLUSTERS= +REDIS_CLUSTERS_PASSWORD= +REDIS_RETRY_RETRIES=3 +REDIS_RETRY_BACKOFF_BASE=1.0 +REDIS_RETRY_BACKOFF_CAP=10.0 +REDIS_SOCKET_TIMEOUT=5.0 +REDIS_SOCKET_CONNECT_TIMEOUT=5.0 +REDIS_HEALTH_CHECK_INTERVAL=30 +EVENT_BUS_REDIS_URL= +EVENT_BUS_REDIS_CHANNEL_TYPE=pubsub +EVENT_BUS_REDIS_USE_CLUSTERS=false +BROKER_USE_SSL=false diff --git a/dify/code/envs/infrastructure/certbot.env.example b/dify/code/envs/infrastructure/certbot.env.example new file mode 100644 index 000000000..c654fbe02 --- /dev/null +++ b/dify/code/envs/infrastructure/certbot.env.example @@ -0,0 +1,7 @@ +# ------------------------------ +# Certbot Configuration +# ------------------------------ + +CERTBOT_EMAIL=your_email@example.com +CERTBOT_DOMAIN=your_domain.com +CERTBOT_OPTIONS= diff --git a/dify/code/envs/infrastructure/etcd.env.example b/dify/code/envs/infrastructure/etcd.env.example new file mode 100644 index 000000000..4dca26671 --- /dev/null +++ b/dify/code/envs/infrastructure/etcd.env.example @@ -0,0 +1,4 @@ +# ------------------------------ +# Etcd Configuration +# ------------------------------ + diff --git a/dify/code/envs/infrastructure/milvus-standalone.env.example b/dify/code/envs/infrastructure/milvus-standalone.env.example new file mode 100644 index 000000000..7e87ed264 --- /dev/null +++ b/dify/code/envs/infrastructure/milvus-standalone.env.example @@ -0,0 +1,4 @@ +# ------------------------------ +# Milvus Standalone Configuration +# ------------------------------ + diff --git a/dify/code/envs/infrastructure/minio.env.example b/dify/code/envs/infrastructure/minio.env.example new file mode 100644 index 000000000..7c8e1fa35 --- /dev/null +++ b/dify/code/envs/infrastructure/minio.env.example @@ -0,0 +1,4 @@ +# ------------------------------ +# Minio Configuration +# ------------------------------ + diff --git a/dify/code/envs/infrastructure/nginx.env.example b/dify/code/envs/infrastructure/nginx.env.example new file mode 100644 index 000000000..fcb369a47 --- /dev/null +++ b/dify/code/envs/infrastructure/nginx.env.example @@ -0,0 +1,18 @@ +# ------------------------------ +# Nginx Configuration +# ------------------------------ + +NGINX_SERVER_NAME=_ +NGINX_HTTPS_ENABLED=false +NGINX_PORT=80 +NGINX_SSL_PORT=443 +NGINX_SSL_CERT_FILENAME=dify.crt +NGINX_SSL_CERT_KEY_FILENAME=dify.key +NGINX_SSL_PROTOCOLS=TLSv1.2 TLSv1.3 +NGINX_WORKER_PROCESSES=auto +NGINX_CLIENT_MAX_BODY_SIZE=100M +NGINX_KEEPALIVE_TIMEOUT=65 +NGINX_PROXY_READ_TIMEOUT=3600s +NGINX_PROXY_SEND_TIMEOUT=3600s +NGINX_ENABLE_CERTBOT_CHALLENGE=false +NGINX_SOCKET_IO_UPSTREAM=api_websocket:5001 diff --git a/dify/code/envs/infrastructure/ssrf-proxy.env.example b/dify/code/envs/infrastructure/ssrf-proxy.env.example new file mode 100644 index 000000000..210a78249 --- /dev/null +++ b/dify/code/envs/infrastructure/ssrf-proxy.env.example @@ -0,0 +1,17 @@ +# ------------------------------ +# Ssrf Proxy Configuration +# ------------------------------ + +SSRF_PROXY_HTTP_URL=http://ssrf_proxy:3128 +SSRF_PROXY_HTTPS_URL=http://ssrf_proxy:3128 +SSRF_HTTP_PORT=3128 +SSRF_COREDUMP_DIR=/var/spool/squid +SSRF_REVERSE_PROXY_PORT=8194 +SSRF_SANDBOX_HOST=sandbox +SSRF_DEFAULT_TIME_OUT=5 +SSRF_DEFAULT_CONNECT_TIME_OUT=5 +SSRF_DEFAULT_READ_TIME_OUT=5 +SSRF_DEFAULT_WRITE_TIME_OUT=5 +SSRF_POOL_MAX_CONNECTIONS=100 +SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS=20 +SSRF_POOL_KEEPALIVE_EXPIRY=5.0 diff --git a/dify/code/middleware.env.example b/dify/code/envs/middleware.env.example similarity index 100% rename from dify/code/middleware.env.example rename to dify/code/envs/middleware.env.example diff --git a/dify/code/envs/security.env.example b/dify/code/envs/security.env.example new file mode 100644 index 000000000..d7556d91e --- /dev/null +++ b/dify/code/envs/security.env.example @@ -0,0 +1,41 @@ +# ------------------------------ +# Security Configuration +# ------------------------------ + +TIDB_ON_QDRANT_API_KEY=dify +TENCENT_VECTOR_DB_API_KEY=dify +ALIBABACLOUD_MYSQL_PASSWORD=difyai123456 +RELYT_PASSWORD=difyai123456 +LINDORM_PASSWORD=admin +HUAWEI_CLOUD_PASSWORD=admin +UPSTASH_VECTOR_TOKEN=dify +TABLESTORE_ACCESS_KEY_ID=xxx +TABLESTORE_ACCESS_KEY_SECRET=xxx +UNSTRUCTURED_API_KEY= +PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false +NOTION_CLIENT_SECRET= +NOTION_INTERNAL_SECRET= +RESEND_API_KEY=your-resend-api-key +SMTP_PASSWORD= +SENDGRID_API_KEY= +RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5 +EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES=5 +CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5 +OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5 +CODE_EXECUTION_API_KEY=dify-sandbox +ALIYUN_SLS_ACCESS_KEY_ID= +ALIYUN_SLS_ACCESS_KEY_SECRET= +OTLP_API_KEY= +BAIDU_VECTOR_DB_API_KEY=dify +ANALYTICDB_KEY_ID=your-ak +ANALYTICDB_KEY_SECRET=your-sk +ANALYTICDB_PASSWORD=testpassword +ANALYTICDB_NAMESPACE_PASSWORD=difypassword +TIDB_VECTOR_PASSWORD= +TIDB_PUBLIC_KEY=dify +TIDB_PRIVATE_KEY=dify +VIKINGDB_ACCESS_KEY=your-ak +VIKINGDB_SECRET_KEY=your-sk +# Leave empty to auto-generate a persistent key in the storage directory. +SECRET_KEY= +INIT_PASSWORD= diff --git a/dify/code/envs/vectorstores/chroma.env.example b/dify/code/envs/vectorstores/chroma.env.example new file mode 100644 index 000000000..2a15375a3 --- /dev/null +++ b/dify/code/envs/vectorstores/chroma.env.example @@ -0,0 +1,13 @@ +# ------------------------------ +# Chroma Configuration +# ------------------------------ + +CHROMA_DATABASE=default_database +CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthClientProvider +CHROMA_AUTH_CREDENTIALS= +CHROMA_HOST=127.0.0.1 +CHROMA_PORT=8000 +CHROMA_TENANT=default_tenant +CHROMA_SERVER_AUTHN_CREDENTIALS=difyai123456 +CHROMA_SERVER_AUTHN_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider +CHROMA_IS_PERSISTENT=TRUE diff --git a/dify/code/envs/vectorstores/couchbase.env.example b/dify/code/envs/vectorstores/couchbase.env.example new file mode 100644 index 000000000..4329d9c72 --- /dev/null +++ b/dify/code/envs/vectorstores/couchbase.env.example @@ -0,0 +1,9 @@ +# ------------------------------ +# Couchbase Configuration +# ------------------------------ + +COUCHBASE_PASSWORD=password +COUCHBASE_BUCKET_NAME=Embeddings +COUCHBASE_SCOPE_NAME=_default +COUCHBASE_CONNECTION_STRING=couchbase://couchbase-server +COUCHBASE_USER=Administrator diff --git a/dify/code/envs/vectorstores/elasticsearch.env.example b/dify/code/envs/vectorstores/elasticsearch.env.example new file mode 100644 index 000000000..2aaa965cd --- /dev/null +++ b/dify/code/envs/vectorstores/elasticsearch.env.example @@ -0,0 +1,17 @@ +# ------------------------------ +# Elasticsearch Configuration +# ------------------------------ + +ELASTICSEARCH_CLOUD_URL=YOUR-ELASTICSEARCH_CLOUD_URL +ELASTICSEARCH_PASSWORD=elastic +KIBANA_PORT=5601 +ELASTICSEARCH_USE_CLOUD=false +ELASTICSEARCH_API_KEY=YOUR-ELASTICSEARCH_API_KEY +ELASTICSEARCH_VERIFY_CERTS=False +ELASTICSEARCH_CA_CERTS= +ELASTICSEARCH_REQUEST_TIMEOUT=100000 +ELASTICSEARCH_RETRY_ON_TIMEOUT=True +ELASTICSEARCH_MAX_RETRIES=10 +ELASTICSEARCH_HOST=0.0.0.0 +ELASTICSEARCH_PORT=9200 +ELASTICSEARCH_USERNAME=elastic diff --git a/dify/code/envs/vectorstores/iris.env.example b/dify/code/envs/vectorstores/iris.env.example new file mode 100644 index 000000000..b1eb39bff --- /dev/null +++ b/dify/code/envs/vectorstores/iris.env.example @@ -0,0 +1,17 @@ +# ------------------------------ +# Iris Configuration +# ------------------------------ + +IRIS_CONNECTION_URL= +IRIS_MIN_CONNECTION=1 +IRIS_MAX_CONNECTION=3 +IRIS_TEXT_INDEX=true +IRIS_TEXT_INDEX_LANGUAGE=en +IRIS_TIMEZONE=UTC +IRIS_PASSWORD=Dify@1234 +IRIS_DATABASE=USER +IRIS_SCHEMA=dify +IRIS_HOST=iris +IRIS_SUPER_SERVER_PORT=1972 +IRIS_WEB_SERVER_PORT=52773 +IRIS_USER=_SYSTEM diff --git a/dify/code/envs/vectorstores/matrixone.env.example b/dify/code/envs/vectorstores/matrixone.env.example new file mode 100644 index 000000000..931375f8b --- /dev/null +++ b/dify/code/envs/vectorstores/matrixone.env.example @@ -0,0 +1,9 @@ +# ------------------------------ +# Matrixone Configuration +# ------------------------------ + +MATRIXONE_PASSWORD=111 +MATRIXONE_HOST=matrixone +MATRIXONE_PORT=6001 +MATRIXONE_USER=dump +MATRIXONE_DATABASE=dify diff --git a/dify/code/envs/vectorstores/milvus.env.example b/dify/code/envs/vectorstores/milvus.env.example new file mode 100644 index 000000000..d16879ca7 --- /dev/null +++ b/dify/code/envs/vectorstores/milvus.env.example @@ -0,0 +1,13 @@ +# ------------------------------ +# Milvus Configuration +# ------------------------------ + +MINIO_ACCESS_KEY=minioadmin +MINIO_SECRET_KEY=minioadmin +ETCD_ENDPOINTS=etcd:2379 +MINIO_ADDRESS=minio:9000 +ETCD_AUTO_COMPACTION_MODE=revision +ETCD_AUTO_COMPACTION_RETENTION=1000 +ETCD_QUOTA_BACKEND_BYTES=4294967296 +ETCD_SNAPSHOT_COUNT=50000 +MILVUS_AUTHORIZATION_ENABLED=true diff --git a/dify/code/envs/vectorstores/myscale.env.example b/dify/code/envs/vectorstores/myscale.env.example new file mode 100644 index 000000000..eaa9e88cc --- /dev/null +++ b/dify/code/envs/vectorstores/myscale.env.example @@ -0,0 +1,10 @@ +# ------------------------------ +# Myscale Configuration +# ------------------------------ + +MYSCALE_PASSWORD= +MYSCALE_DATABASE=dify +MYSCALE_FTS_PARAMS= +MYSCALE_HOST=myscale +MYSCALE_PORT=8123 +MYSCALE_USER=default diff --git a/dify/code/envs/vectorstores/oceanbase.env.example b/dify/code/envs/vectorstores/oceanbase.env.example new file mode 100644 index 000000000..42bed8df6 --- /dev/null +++ b/dify/code/envs/vectorstores/oceanbase.env.example @@ -0,0 +1,6 @@ +# ------------------------------ +# Oceanbase Configuration +# ------------------------------ + +OCEANBASE_CLUSTER_NAME=difyai +OCEANBASE_MEMORY_LIMIT=6G diff --git a/dify/code/envs/vectorstores/opengauss.env.example b/dify/code/envs/vectorstores/opengauss.env.example new file mode 100644 index 000000000..9f58499b6 --- /dev/null +++ b/dify/code/envs/vectorstores/opengauss.env.example @@ -0,0 +1,12 @@ +# ------------------------------ +# Opengauss Configuration +# ------------------------------ + +OPENGAUSS_PASSWORD=Dify@123 +OPENGAUSS_DATABASE=dify +OPENGAUSS_MIN_CONNECTION=1 +OPENGAUSS_MAX_CONNECTION=5 +OPENGAUSS_ENABLE_PQ=false +OPENGAUSS_HOST=opengauss +OPENGAUSS_PORT=6600 +OPENGAUSS_USER=postgres diff --git a/dify/code/envs/vectorstores/opensearch.env.example b/dify/code/envs/vectorstores/opensearch.env.example new file mode 100644 index 000000000..a6a928337 --- /dev/null +++ b/dify/code/envs/vectorstores/opensearch.env.example @@ -0,0 +1,22 @@ +# ------------------------------ +# Opensearch Configuration +# ------------------------------ + +OPENSEARCH_PASSWORD=admin +OPENSEARCH_AWS_REGION=ap-southeast-1 +OPENSEARCH_AWS_SERVICE=aoss +OPENSEARCH_INITIAL_ADMIN_PASSWORD=Qazwsxedc!@#123 +OPENSEARCH_MEMLOCK_SOFT=-1 +OPENSEARCH_MEMLOCK_HARD=-1 +OPENSEARCH_NOFILE_SOFT=65536 +OPENSEARCH_NOFILE_HARD=65536 +OPENSEARCH_HOST=opensearch +OPENSEARCH_PORT=9200 +OPENSEARCH_SECURE=true +OPENSEARCH_VERIFY_CERTS=true +OPENSEARCH_AUTH_METHOD=basic +OPENSEARCH_USER=admin +OPENSEARCH_DISCOVERY_TYPE=single-node +OPENSEARCH_BOOTSTRAP_MEMORY_LOCK=true +OPENSEARCH_JAVA_OPTS_MIN=512m +OPENSEARCH_JAVA_OPTS_MAX=1024m diff --git a/dify/code/envs/vectorstores/oracle.env.example b/dify/code/envs/vectorstores/oracle.env.example new file mode 100644 index 000000000..c8f24db41 --- /dev/null +++ b/dify/code/envs/vectorstores/oracle.env.example @@ -0,0 +1,13 @@ +# ------------------------------ +# Oracle Configuration +# ------------------------------ + +ORACLE_PASSWORD=dify +ORACLE_DSN=oracle:1521/FREEPDB1 +ORACLE_CONFIG_DIR=/app/api/storage/wallet +ORACLE_WALLET_LOCATION=/app/api/storage/wallet +ORACLE_WALLET_PASSWORD=dify +ORACLE_IS_AUTONOMOUS=false +ORACLE_USER=dify +ORACLE_PWD=Dify123456 +ORACLE_CHARACTERSET=AL32UTF8 diff --git a/dify/code/envs/vectorstores/pgvecto-rs.env.example b/dify/code/envs/vectorstores/pgvecto-rs.env.example new file mode 100644 index 000000000..6428e5dd6 --- /dev/null +++ b/dify/code/envs/vectorstores/pgvecto-rs.env.example @@ -0,0 +1,9 @@ +# ------------------------------ +# Pgvecto Rs Configuration +# ------------------------------ + +PGVECTO_RS_HOST=pgvecto-rs +PGVECTO_RS_PORT=5432 +PGVECTO_RS_USER=postgres +PGVECTO_RS_PASSWORD=difyai123456 +PGVECTO_RS_DATABASE=dify diff --git a/dify/code/envs/vectorstores/pgvector.env.example b/dify/code/envs/vectorstores/pgvector.env.example new file mode 100644 index 000000000..9fd1dbf96 --- /dev/null +++ b/dify/code/envs/vectorstores/pgvector.env.example @@ -0,0 +1,8 @@ +# ------------------------------ +# Pgvector Configuration +# ------------------------------ + +PGVECTOR_PGUSER=postgres +PGVECTOR_POSTGRES_PASSWORD=difyai123456 +PGVECTOR_POSTGRES_DB=dify +PGVECTOR_PGDATA=/var/lib/postgresql/data/pgdata diff --git a/dify/code/envs/vectorstores/qdrant.env.example b/dify/code/envs/vectorstores/qdrant.env.example new file mode 100644 index 000000000..a3555fe54 --- /dev/null +++ b/dify/code/envs/vectorstores/qdrant.env.example @@ -0,0 +1,4 @@ +# ------------------------------ +# Qdrant Configuration +# ------------------------------ + diff --git a/dify/code/envs/vectorstores/seekdb.env.example b/dify/code/envs/vectorstores/seekdb.env.example new file mode 100644 index 000000000..4307fbede --- /dev/null +++ b/dify/code/envs/vectorstores/seekdb.env.example @@ -0,0 +1,4 @@ +# ------------------------------ +# Seekdb Configuration +# ------------------------------ + diff --git a/dify/code/envs/vectorstores/vastbase.env.example b/dify/code/envs/vectorstores/vastbase.env.example new file mode 100644 index 000000000..2c9db50fb --- /dev/null +++ b/dify/code/envs/vectorstores/vastbase.env.example @@ -0,0 +1,11 @@ +# ------------------------------ +# Vastbase Configuration +# ------------------------------ + +VASTBASE_PASSWORD=Difyai123456 +VASTBASE_DATABASE=dify +VASTBASE_MIN_CONNECTION=1 +VASTBASE_MAX_CONNECTION=5 +VASTBASE_HOST=vastbase +VASTBASE_PORT=5432 +VASTBASE_USER=dify diff --git a/dify/code/envs/vectorstores/weaviate.env.example b/dify/code/envs/vectorstores/weaviate.env.example new file mode 100644 index 000000000..82a3ccb17 --- /dev/null +++ b/dify/code/envs/vectorstores/weaviate.env.example @@ -0,0 +1,18 @@ +# ------------------------------ +# Weaviate Configuration +# ------------------------------ + +WEAVIATE_PERSISTENCE_DATA_PATH=/var/lib/weaviate +WEAVIATE_QUERY_DEFAULTS_LIMIT=25 +WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=true +WEAVIATE_DEFAULT_VECTORIZER_MODULE=none +WEAVIATE_CLUSTER_HOSTNAME=node1 +WEAVIATE_AUTHENTICATION_APIKEY_ENABLED=true +WEAVIATE_AUTHENTICATION_APIKEY_ALLOWED_KEYS=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih +WEAVIATE_AUTHENTICATION_APIKEY_USERS=hello@dify.ai +WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED=true +WEAVIATE_AUTHORIZATION_ADMINLIST_USERS=hello@dify.ai +WEAVIATE_DISABLE_TELEMETRY=false +WEAVIATE_ENABLE_TOKENIZER_GSE=false +WEAVIATE_ENABLE_TOKENIZER_KAGOME_JA=false +WEAVIATE_ENABLE_TOKENIZER_KAGOME_KR=false diff --git a/dify/code/generate_docker_compose b/dify/code/generate_docker_compose index bf6c1423c..580091e00 100755 --- a/dify/code/generate_docker_compose +++ b/dify/code/generate_docker_compose @@ -3,6 +3,20 @@ import os import re import sys +# Variables that exist only for Docker Compose orchestration and must NOT be +# injected into containers as environment variables. +SHARED_ENV_EXCLUDE = frozenset( + [ + # Docker Compose profile selection + "COMPOSE_PROFILES", + # Worker health check orchestration flags (consumed by docker-compose, + # not by the application running inside the container) + "COMPOSE_WORKER_HEALTHCHECK_DISABLED", + "COMPOSE_WORKER_HEALTHCHECK_INTERVAL", + "COMPOSE_WORKER_HEALTHCHECK_TIMEOUT", + ] +) + def parse_env_example(file_path): """ @@ -37,7 +51,7 @@ def generate_shared_env_block(env_vars, anchor_name="shared-api-worker-env"): """ lines = [f"x-shared-env: &{anchor_name}"] for key, default in env_vars.items(): - if key == "COMPOSE_PROFILES": + if key in SHARED_ENV_EXCLUDE: continue # If default value is empty, use ${KEY:-} if default == "": @@ -50,35 +64,72 @@ def generate_shared_env_block(env_vars, anchor_name="shared-api-worker-env"): return "\n".join(lines) -def insert_shared_env(template_path, output_path, shared_env_block, header_comments): +def create_env_files_from_example(env_example_path): + """ + Creates actual env files from .env.example by copying the categorized .env.example files. + This allows docker-compose to use env_file references. + Supports per-module structure with subdirectories. """ - Inserts the shared environment variables block and header comments into the template file, - removing any existing x-shared-env anchors, and generates the final docker-compose.yaml file. + base_dir = os.path.dirname(os.path.abspath(env_example_path)) + root_env_file = os.path.join(base_dir, ".env") + if not os.path.exists(root_env_file): + with open(env_example_path, "r", encoding="utf-8") as src, open( + root_env_file, "w", encoding="utf-8", newline="\n" + ) as dst: + dst.write(src.read()) + print(f"Created {root_env_file}") + else: + print(f"{root_env_file} already exists, skipping") + + envs_dir = os.path.join(base_dir, "envs") + if not os.path.isdir(envs_dir): + print(f"No envs directory found at {envs_dir}, skipping split env files") + return [] + + created_files = [] + # Walk through all .env.example files in subdirectories + for root, dirs, files in os.walk(envs_dir): + for file in files: + if file.endswith('.env.example'): + example_file = os.path.join(root, file) + env_file = example_file.replace('.env.example', '.env') + + if os.path.exists(env_file): + print(f"{env_file} already exists, skipping") + continue + + # Copy .example to actual file + with open(example_file, "r", encoding="utf-8") as src, open( + env_file, "w", encoding="utf-8", newline="\n" + ) as dst: + dst.write(src.read()) + created_files.append(env_file) + print(f"Created {env_file}") + + return created_files + + +def insert_shared_env(template_path, output_path, header_comments): + """ + Copies the template file to output path with header comments. + The template now uses env_file references instead of a huge YAML anchor. """ with open(template_path, "r", encoding="utf-8") as f: template_content = f.read() - # Remove existing x-shared-env: &shared-api-worker-env lines - template_content = re.sub( - r"^x-shared-env: &shared-api-worker-env\s*\n?", - "", - template_content, - flags=re.MULTILINE, - ) + # Prepare the final content with header comments + final_content = f"{header_comments}\n{template_content}" - # Prepare the final content with header comments and shared env block - final_content = f"{header_comments}\n{shared_env_block}\n\n{template_content}" - - with open(output_path, "w", encoding="utf-8") as f: + with open(output_path, "w", encoding="utf-8", newline="\n") as f: f.write(final_content) print(f"Generated {output_path}") def main(): - env_example_path = ".env.example" - template_path = "docker-compose-template.yaml" - output_path = "docker-compose.yaml" - anchor_name = "shared-api-worker-env" # Can be modified as needed + base_dir = os.path.dirname(os.path.abspath(__file__)) + env_example_path = os.path.join(base_dir, ".env.example") + template_path = os.path.join(base_dir, "docker-compose-template.yaml") + output_path = os.path.join(base_dir, "docker-compose.yaml") # Define header comments to be added at the top of docker-compose.yaml header_comments = ( @@ -95,17 +146,14 @@ def main(): print(f"Error: File {path} does not exist.") sys.exit(1) - # Parse .env.example file - env_vars = parse_env_example(env_example_path) - - if not env_vars: - print("Warning: No environment variables found in .env.example.") - - # Generate shared environment variables block - shared_env_block = generate_shared_env_block(env_vars, anchor_name) + # Create env files from categorized .env.example files + # These files are used by docker-compose's env_file directive + # This ensures .env files exist even in CI/CD environments + create_env_files_from_example(env_example_path) - # Insert shared environment variables block and header comments into the template - insert_shared_env(template_path, output_path, shared_env_block, header_comments) + # Copy template to output with header comments + # The template now uses env_file references instead of a huge YAML anchor + insert_shared_env(template_path, output_path, header_comments) if __name__ == "__main__": diff --git a/dify/code/nginx/conf.d/default.conf.template b/dify/code/nginx/conf.d/default.conf.template index 1d63c1b97..64c720ca2 100644 --- a/dify/code/nginx/conf.d/default.conf.template +++ b/dify/code/nginx/conf.d/default.conf.template @@ -14,6 +14,16 @@ server { include proxy.conf; } + location /socket.io/ { + resolver 127.0.0.11 valid=30s ipv6=off; + set $socket_io_upstream ${NGINX_SOCKET_IO_UPSTREAM}; + proxy_pass http://$socket_io_upstream; + include proxy.conf; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_cache_bypass $http_upgrade; + } + location /v1 { proxy_pass http://api:5001; include proxy.conf; diff --git a/dify/code/ssrf_proxy/squid.conf.template b/dify/code/ssrf_proxy/squid.conf.template index 256e669c8..fbe9ebc44 100644 --- a/dify/code/ssrf_proxy/squid.conf.template +++ b/dify/code/ssrf_proxy/squid.conf.template @@ -28,6 +28,7 @@ http_access deny manager http_access allow localhost include /etc/squid/conf.d/*.conf http_access deny all +tcp_outgoing_address 0.0.0.0 ################################## Proxy Server ################################ http_port ${HTTP_PORT} diff --git a/dify/code/volumes/sandbox/conf/config.yaml b/dify/code/volumes/sandbox/conf/config.yaml index 8c1a1deb5..3b4a6b843 100644 --- a/dify/code/volumes/sandbox/conf/config.yaml +++ b/dify/code/volumes/sandbox/conf/config.yaml @@ -5,7 +5,8 @@ app: max_workers: 4 max_requests: 50 worker_timeout: 5 -python_path: /usr/local/bin/python3 +python_path: /opt/python/bin/python3 +nodejs_path: /usr/local/bin/node enable_network: True # please make sure there is no network risk in your environment allowed_syscalls: # please leave it empty if you have no idea how seccomp works proxy: diff --git a/dify/code/volumes/sandbox/conf/config.yaml.example b/dify/code/volumes/sandbox/conf/config.yaml.example index f92c19e51..365089cb9 100644 --- a/dify/code/volumes/sandbox/conf/config.yaml.example +++ b/dify/code/volumes/sandbox/conf/config.yaml.example @@ -5,7 +5,7 @@ app: max_workers: 4 max_requests: 50 worker_timeout: 5 -python_path: /usr/local/bin/python3 +python_path: /opt/python/bin/python3 python_lib_path: - /usr/local/lib/python3.10 - /usr/lib/python3.10 diff --git a/plane/code/.env.example b/plane/code/.env.example index 087d2eefd..60f4750a2 100644 --- a/plane/code/.env.example +++ b/plane/code/.env.example @@ -1,5 +1,5 @@ APP_DOMAIN=$(PRIMARY_DOMAIN) -APP_RELEASE=v1.2.3 +APP_RELEASE=v1.3.1 WEB_REPLICAS=1 SPACE_REPLICAS=1 @@ -80,6 +80,15 @@ API_KEY_RATE_LIMIT=60/minute # Live server environment variables # WARNING: You must set a secure value for LIVE_SERVER_SECRET_KEY in production environments. LIVE_SERVER_SECRET_KEY= + +# Webhook IP allowlist — comma-separated IPs or CIDR ranges allowed as webhook targets +# even if they resolve to private networks (e.g. "10.0.0.0/8,192.168.1.0/24,172.16.0.5") +WEBHOOK_ALLOWED_IPS= + +# Webhook hostname allowlist — comma-separated hostnames that bypass the private-IP +# SSRF check. Useful for trusted internal services whose container/service IPs are +# dynamic (e.g. "silo,silo.namespace.svc.cluster.local") +WEBHOOK_ALLOWED_HOSTS= DOCKERHUB_USER=artifacts.plane.so/makeplane PULL_POLICY=if_not_present CUSTOM_BUILD=false diff --git a/plane/code/docker-compose.yml b/plane/code/docker-compose.yml index 59845fbf5..a2d0994c5 100644 --- a/plane/code/docker-compose.yml +++ b/plane/code/docker-compose.yml @@ -60,10 +60,12 @@ x-app-env: &app-env API_KEY_RATE_LIMIT: ${API_KEY_RATE_LIMIT:-60/minute} MINIO_ENDPOINT_SSL: ${MINIO_ENDPOINT_SSL:-0} LIVE_SERVER_SECRET_KEY: ${LIVE_SERVER_SECRET_KEY:-2FiJk1U2aiVPEQtzLehYGlTSnTnrs7LW} + WEBHOOK_ALLOWED_IPS: ${WEBHOOK_ALLOWED_IPS:-} + WEBHOOK_ALLOWED_HOSTS: ${WEBHOOK_ALLOWED_HOSTS:-} services: web: - image: artifacts.plane.so/makeplane/plane-frontend:${APP_RELEASE:-v1.2.3} + image: makeplane/plane-frontend:${APP_RELEASE:-v1.3.1} deploy: replicas: ${WEB_REPLICAS:-1} restart_policy: @@ -73,7 +75,7 @@ services: - worker space: - image: artifacts.plane.so/makeplane/plane-space:${APP_RELEASE:-v1.2.3} + image: makeplane/plane-space:${APP_RELEASE:-v1.3.1} deploy: replicas: ${SPACE_REPLICAS:-1} restart_policy: @@ -84,7 +86,7 @@ services: - web admin: - image: artifacts.plane.so/makeplane/plane-admin:${APP_RELEASE:-v1.2.3} + image: makeplane/plane-admin:${APP_RELEASE:-v1.3.1} deploy: replicas: ${ADMIN_REPLICAS:-1} restart_policy: @@ -94,7 +96,7 @@ services: - web live: - image: artifacts.plane.so/makeplane/plane-live:${APP_RELEASE:-v1.2.3} + image: makeplane/plane-live:${APP_RELEASE:-v1.3.1} environment: <<: [ *live-env, *redis-env ] deploy: @@ -106,7 +108,7 @@ services: - web api: - image: artifacts.plane.so/makeplane/plane-backend:${APP_RELEASE:-v1.2.3} + image: makeplane/plane-backend:${APP_RELEASE:-v1.3.1} command: ./bin/docker-entrypoint-api.sh deploy: replicas: ${API_REPLICAS:-1} @@ -122,7 +124,7 @@ services: - plane-mq worker: - image: artifacts.plane.so/makeplane/plane-backend:${APP_RELEASE:-v1.2.3} + image: makeplane/plane-backend:${APP_RELEASE:-v1.3.1} command: ./bin/docker-entrypoint-worker.sh deploy: replicas: ${WORKER_REPLICAS:-1} @@ -139,7 +141,7 @@ services: - plane-mq beat-worker: - image: artifacts.plane.so/makeplane/plane-backend:${APP_RELEASE:-v1.2.3} + image: makeplane/plane-backend:${APP_RELEASE:-v1.3.1} command: ./bin/docker-entrypoint-beat.sh deploy: replicas: ${BEAT_WORKER_REPLICAS:-1} @@ -156,7 +158,7 @@ services: - plane-mq migrator: - image: artifacts.plane.so/makeplane/plane-backend:${APP_RELEASE:-v1.2.3} + image: makeplane/plane-backend:${APP_RELEASE:-v1.3.1} command: ./bin/docker-entrypoint-migrator.sh deploy: replicas: 1 @@ -218,7 +220,7 @@ services: # Comment this if you already have a reverse proxy running proxy: - image: artifacts.plane.so/makeplane/plane-proxy:${APP_RELEASE:-v1.2.3} + image: makeplane/plane-proxy:${APP_RELEASE:-v1.3.1} deploy: replicas: 1 restart_policy: diff --git a/supabase/code/.env.example b/supabase/code/.env.example index fd52e2578..03a1a67a1 100644 --- a/supabase/code/.env.example +++ b/supabase/code/.env.example @@ -15,11 +15,31 @@ # Postgres POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password -# Symmetric encryption key and JWT API keys +# Legacy symmetric HS256 key JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long +# Legacy API keys (HS256-signed JWTs) ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q +# Asymmetric key pair (ES256) and opaque API keys +# +# Documentation: +# https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys +# +# To generate: +# sh ./utils/add-new-auth-keys.sh +# +# Opaque API key for client-side use (anon role). +SUPABASE_PUBLISHABLE_KEY= +# Opaque API key for server-side use (service_role). Never expose in client code. +SUPABASE_SECRET_KEY= +# JSON array of signing JWKs (EC private + legacy symmetric). +# Used by Auth. +JWT_KEYS= +# JWKS for token verification (EC public + legacy symmetric). +# Used by PostgREST, Realtime, Storage to verify tokens. +JWT_JWKS= + # Access to Dashboard DASHBOARD_USERNAME=supabase DASHBOARD_PASSWORD=this_password_is_insecure_and_should_be_updated @@ -146,7 +166,7 @@ ENABLE_PHONE_AUTOCONFIRM=true ## OAuth / Social login providers # Uncomment and fill in the providers you want to enable. -# You must ALSO uncomment the matching GOTRUE_EXTERNAL_* lines in docker-compose.yml. +# You must ALSO uncomment the matching GOTRUE_EXTERNAL_* lines in docker-compose.yml # Documentation: https://supabase.com/docs/guides/self-hosting/self-hosted-oauth # GOOGLE_ENABLED=false # GOOGLE_CLIENT_ID= @@ -162,7 +182,7 @@ ENABLE_PHONE_AUTOCONFIRM=true # Phone / SMS provider configuration # Uncomment to configure SMS delivery for phone auth and phone MFA. -# You must ALSO uncomment the matching GOTRUE_SMS_* lines in docker-compose.yml. +# You must ALSO uncomment the matching GOTRUE_SMS_* lines in docker-compose.yml # Documentation: https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa # SMS_PROVIDER=twilio # SMS_OTP_EXP=60 @@ -180,7 +200,7 @@ ENABLE_PHONE_AUTOCONFIRM=true # Multi-factor authentication (MFA) # Uncomment to change MFA defaults. -# You must ALSO uncomment the matching GOTRUE_MFA_* lines in docker-compose.yml. +# You must ALSO uncomment the matching GOTRUE_MFA_* lines in docker-compose.yml # App Authenticator (TOTP) - enabled by default # MFA_TOTP_ENROLL_ENABLED=true @@ -190,9 +210,30 @@ ENABLE_PHONE_AUTOCONFIRM=true # MFA_PHONE_ENROLL_ENABLED=false # MFA_PHONE_VERIFY_ENABLED=false -## Maximum MFA factors a user can enroll +# Maximum MFA factors a user can enroll # MFA_MAX_ENROLLED_FACTORS=10 +## SAML SSO + +# You must ALSO uncomment the matching GOTRUE_* lines in docker-compose.yml +# Documentation: https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso + +# SAML_ENABLED=true +# SAML_PRIVATE_KEY= + +# Optional: accept encrypted SAML assertions from IdPs (default: false) +# SAML_ALLOW_ENCRYPTED_ASSERTIONS=false + +# Optional: how long relay state tokens remain valid (default: 2m0s) +# SAML_RELAY_STATE_VALIDITY_PERIOD=2m0s + +# Optional: override the SAML entity ID / ACS base URL +# Defaults to API_EXTERNAL_URL if not set +# SAML_EXTERNAL_URL=https://supabase.example.com:8000 + +# Optional: rate limit on the ACS endpoint (requests per second, default: 15) +# SAML_RATE_LIMIT_ASSERTION=15 + ############ # Storage - Configuration for Storage @@ -237,16 +278,24 @@ FUNCTIONS_VERIFY_JWT=false # Postgres schemas exposed via the REST API PGRST_DB_SCHEMAS=public,storage,graphql_public +# Max number of rows returned by a request +PGRST_DB_MAX_ROWS=1000 + +# Extra schemas added to the search_path of every request +PGRST_DB_EXTRA_SEARCH_PATH=public + ############ # Analytics - Configuration for Logflare ############ -# Check the LOGFLARE_* access token configuration above. -# If Logflare is externally exposed, configure securely! +# Check the LOGFLARE_* access token configuration _above_. +# If Logflare has to be externally exposed - configure securely! -# Docker socket location - this value will differ depending on your OS +# Docker socket location - required for proper Vector operation DOCKER_SOCKET_LOCATION=/var/run/docker.sock +# For Podman use the following: +# DOCKER_SOCKET_LOCATION=/run/podman/podman.sock # Google Cloud Project details GOOGLE_PROJECT_ID=GOOGLE_PROJECT_ID @@ -254,19 +303,26 @@ GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER ############ -# API Proxy - Configuration for the Kong API gateway +# API gateway ############ +# Kong configuration variables KONG_HTTP_PORT=8000 KONG_HTTPS_PORT=8443 +# Used internally by the API gateway - DO NOT use in any client or server code. +# Pre-signed ES256 JWT "API key" for anon role. +ANON_KEY_ASYMMETRIC= +# Pre-signed ES256 JWT "API key" for service_role. +SERVICE_ROLE_KEY_ASYMMETRIC= + ############ # imgproxy ############ # Enable webp support -IMGPROXY_ENABLE_WEBP_DETECTION=true +IMGPROXY_AUTO_WEBP=true ############ diff --git a/supabase/code/.gitattributes b/supabase/code/.gitattributes new file mode 100644 index 000000000..95aa9f911 --- /dev/null +++ b/supabase/code/.gitattributes @@ -0,0 +1,17 @@ +* text=auto + +*.md eol=lf + +*.env eol=lf +.env.example eol=lf + +*.sh eol=lf +*.sql eol=lf +*.yml eol=lf +*.yaml eol=lf +*.ts eol=lf +*.exs eol=lf +*.conf eol=lf +*.tpl eol=lf +Caddyfile eol=lf +Dockerfile* eol=lf diff --git a/supabase/code/CHANGELOG.md b/supabase/code/CHANGELOG.md index e2fc44979..66c40af79 100644 --- a/supabase/code/CHANGELOG.md +++ b/supabase/code/CHANGELOG.md @@ -5,33 +5,142 @@ All notable changes to the Supabase self-hosted Docker configuration. Changes are grouped by service rather than by change type. See [versions.md](./versions.md) for complete image version history and rollback information. -Check updates for each service to learn more. +See per-service updates below for details. -**Note:** Configuration updates marked with "requires [...] update" are already included in the latest version of the repository. Pull the latest changes or refer to the linked PR for manual updates. After updating `docker-compose.yml`, pull latest images and recreate containers - use `docker compose pull && docker compose down && docker compose up -d`. +**Note:** Configuration updates marked with "requires [...] update" are already included in the latest version of the repository. Pull the latest changes or refer to the linked PR for manual updates. After updating `docker-compose.yml`, pull the latest images and recreate containers - use `docker compose pull && docker compose down && docker compose up -d`. --- ## Unreleased +⚠️ **Upcoming default changes:** In a future release, several defaults will change: Postgres 15 → 17, Kong → Envoy, MinIO → RustFS, Analytics/Vector removed from the default stack, and the new API keys and authentication replacing the "legacy" architecture. Most of these are already available as optional configurations. + +--- + +## [2026-04-27] + +### Configuration +- ⚠️ Added `docker-compose.envoy.yml` and `volumes/api/envoy` - PR [#43838](https://github.com/supabase/supabase/pull/43838). See also the API gateway updates below +- ⚠️ Changed Studio healthcheck and some other configuration for better compatibility with Podman (requires `docker-compose.yml` update) - PR [#44754](https://github.com/supabase/supabase/pull/44754) +- ⚠️ Changed Studio configuration to bind to all IPv4 interfaces only (requires `docker-compose.yml` update) - PR [#44772](https://github.com/supabase/supabase/pull/44772) + +### Documentation +- Added a new [how-to](https://supabase.com/docs/guides/self-hosting/remove-superuser-access) describing how to switch from `supabase_admin` to `postgres` role for Studio - PR [#42975](https://github.com/supabase/supabase/pull/42975) (via [@singh-inder](https://github.com/singh-inder/)) +- Added a new [how-to](https://github.com/supabase/supabase/pull/45152) for configuring Envoy as the new API gateway - PR [#45152](https://github.com/supabase/supabase/pull/45152) +- Updated the main [setup guide](https://supabase.com/docs/guides/self-hosting/docker) and the how-tos to reflect the state of the self-hosted Supabase configuration - PR [#45011](https://github.com/supabase/supabase/pull/45011) + +### Utils +- ⚠️ Added `reassign-owner.sh` to update database objects - PR [#42975](https://github.com/supabase/supabase/pull/42975). Read more in the "[Remove superuser access](https://supabase.com/docs/guides/self-hosting/remove-superuser-access)" how-to guide +- ⚠️ Changed `add-new-auth-keys.sh` to also update `docker-compose.yml` - PR [#45056](https://github.com/supabase/supabase/pull/45056) + +### Studio +- Updated to `2026.04.27-sha-5f60601` +- ⚠️ Added 4 new lints to the Security Advisor - PR [#45253](https://github.com/supabase/supabase/pull/45253), PR [#45260](https://github.com/supabase/supabase/pull/45260). Read more about lint rules 0026 - 0029 in the [Performance and Security Advisors](https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0026_pg_graphql_anon_table_exposed) section of the Supabase documentation + +### API gateway +- ⚠️ Added Envoy as the new optional API gateway (requires `docker-compose.envoy.yml`, `volumes/api/envoy`, and `volumes/logs/vector.yml` update) - PR [#43838](https://github.com/supabase/supabase/pull/43838) (via [@luizfelmach](https://github.com/luizfelmach/)) + +--- + +## [2026-04-08] + +### Documentation +- Added new how-to guides for configuring [custom email templates](https://supabase.com/docs/guides/self-hosting/custom-email-templates), setting up [SAML SSO](https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso), and [using Postgres 17](https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17) - PR [#42832](https://github.com/supabase/supabase/pull/42832), PR [#43386](https://github.com/supabase/supabase/pull/43386), PR [#44147](https://github.com/supabase/supabase/pull/44147) + +### Utils +- ⚠️ Added `upgrade-pg17.sh` - PR [#44147](https://github.com/supabase/supabase/pull/44147). Read more in the "[Upgrade to Postgres 17](https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17)" how-to guide + +### Studio +- Updated to `2026.04.08-sha-205cbe7` + +### API gateway +- ⚠️ Added configuration for SAML SSO (requires `.env`, `docker-compose.yml` and `volumes/api/kong.yml` update) - PR [#43385](https://github.com/supabase/supabase/pull/43385) (via [@luizfelmach](https://github.com/luizfelmach/)) + +### PostgREST +- Updated to `v14.8` - [Changelog](https://github.com/PostgREST/postgrest/blob/main/CHANGELOG.md) | [Release](https://github.com/PostgREST/postgrest/releases/tag/v14.8) + +### Storage +- Updated to `v1.48.26` - [Release](https://github.com/supabase/storage/releases/tag/v1.48.26) + +### imgproxy +- Changed `IMGPROXY_ENABLE_WEBP_DETECTION` environment variable to `IMGPROXY_AUTO_WEBP` (requires `.env` and `docker-compose.yml` update) - PR [#43919](https://github.com/supabase/supabase/pull/43919) + +### Postgres Meta +- Updated to `v0.96.3` - [Release](https://github.com/supabase/postgres-meta/releases/tag/v0.96.3) + +### Analytics (Logflare) +- Updated to `v1.36.1` - [Release](https://github.com/Logflare/logflare/releases/tag/v1.36.1) + +### Postgres +- ⚠️ Added `docker-compose.pg17.yml` override - PR [#44147](https://github.com/supabase/supabase/pull/44147) +- ⚠️ Added `upgrade-pg17.sh` - PR [#44147](https://github.com/supabase/supabase/pull/44147) +- ⚠️ Added [documentation](https://supabase.com/docs/guides/self-hosting/postgres-upgrade-17) explaining the upgrade to Postgres 17 + +--- + +## [2026-03-16] + +⚠️ **Note:** This update includes **important changes**. Please check the details below. The following configuration files have been added/updated: `utils/add-new-auth-keys.sh`, `utils/rotate-new-api-keys.sh`, `docker-compose.yml`, `.env.example`, `docker-compose.s3.yml`, `docker-compose.rustfs.yml`, `volumes/api/kong.yml`, `volumes/api/kong-entrypoint.sh`, `docker-compose.caddy.yml`, `docker-compose.nginx.yml`, `volumes/functions/main/index.ts`, and `volumes/proxy`. + +### Configuration +- ⚠️ Added scripts and templates to support the new API key format (`sb_` API keys) and the new asymmetric authentication - PR [#43554](https://github.com/supabase/supabase/pull/43554); see the [how-to guide](https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys) for detailed instructions +- Added optional proxy configuration for Caddy and nginx - PR [#43291](https://github.com/supabase/supabase/pull/43291); read the [how-to guide](https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https) to learn more + +### Documentation +- Added several new how-to guides to the self-hosted Supabase [documentation](https://supabase.com/docs/guides/self-hosting) - PR [#42745](https://github.com/supabase/supabase/pull/42745), PR [#42953](https://github.com/supabase/supabase/pull/42953), PR [#43177](https://github.com/supabase/supabase/pull/43177), PR [#43286](https://github.com/supabase/supabase/pull/43286), PR [#43293](https://github.com/supabase/supabase/pull/43293) + +### Utils and tests +- Added `add-new-auth-keys.sh` and `rotate-new-api-keys.sh` - PR [#43554](https://github.com/supabase/supabase/pull/43554) +- Added `./tests` with 100+ test cases - PR [#43573](https://github.com/supabase/supabase/pull/43573) + +### Studio +- Updated to `2026.03.16-sha-5528817` +- ⚠️ Added the link to the Data API page in Integrations - PR [#43268](https://github.com/supabase/supabase/pull/43268) +- ⚠️ Added `PGRST_DB_SCHEMAS`, `PGRST_DB_EXTRA_SEARCH_PATH`, and `PGRST_DB_MAX_ROWS` to Studio configuration (requires `docker-compose.yml` update) - PR [#43268](https://github.com/supabase/supabase/pull/43268) + +### MCP Server +- Updated to `v0.7.0` - [Release](https://github.com/supabase-community/supabase-mcp/releases/tag/v0.7.0) + +### API gateway +- ⚠️ Updated Kong to `3.9.1` - PR [#43554](https://github.com/supabase/supabase/pull/43554) + +### PostgREST +- Updated to `v14.6` - [Changelog](https://github.com/PostgREST/postgrest/blob/main/CHANGELOG.md) | [Release](https://github.com/PostgREST/postgrest/releases/tag/v14.6) + +### Realtime + +- ⚠️ Added **mandatory** `METRICS_JWT_SECRET` environment variable (requires `docker-compose.s3.yml` update) - PR [realtime#1729](https://github.com/supabase/realtime/pull/1729) + +### Storage +- Updated to `v1.44.2` - [Release](https://github.com/supabase/storage/releases/tag/v1.44.2) +- ⚠️ Added `STORAGE_PUBLIC_URL` environment variable to simplify proxy configuration (requires `docker-compose.s3.yml` update) - PR [storage#900](https://github.com/supabase/storage/pull/900) +- ⚠️ Added RustFS as an optional S3 backend - PR [#42935](https://github.com/supabase/supabase/pull/42935) +- ⚠️ Changed Docker Compose configuration for S3 backends to use named volumes - PR [#43815](https://github.com/supabase/supabase/pull/43815) + +### Edge Runtime +- Updated to `v1.71.2` - [Release](https://github.com/supabase/edge-runtime/releases/tag/v1.71.2) +- ⚠️ Added `SUPABASE_PUBLISHABLE_KEYS`, `SUPABASE_SECRET_KEYS`, and `SUPABASE_PUBLIC_URL` environment variables (requires `docker-compose.yml` update) +- ⚠️ Added an option for a "hybrid" JWT verification following the addition of the new API keys and the new asymmetric authentication (requires `volumes/functions/main/index.ts` update) - PR [#42130](https://github.com/supabase/supabase/pull/42130) +- ⚠️ Added optional rate limiter - PR [edge-runtime#670](https://github.com/supabase/edge-runtime/pull/670) + --- ## [2026-02-18] ### Storage -- Updated `supabase/storage-api` to `v1.37.8` in `docker-compose.s3.yml` -- Changed MinIO image in `docker-compose.s3.yml` to use Chainguard [minio](https://images.chainguard.dev/directory/image/minio/overview) and [minio-client](https://images.chainguard.dev/directory/image/minio-client/overview) (requires `docker-compose.s3.yml` update) - PR [#42942](https://github.com/supabase/supabase/pull/42942) +- Changed MinIO image to use Chainguard [minio](https://images.chainguard.dev/directory/image/minio/overview) and [minio-client](https://images.chainguard.dev/directory/image/minio-client/overview) (requires `docker-compose.s3.yml` update) - PR [#42942](https://github.com/supabase/supabase/pull/42942) +- Updated Storage image version to `v1.37.8` in `docker-compose.s3.yml` - Removed `imgproxy` service from `docker-compose.s3.yml` to minimize redundancy - PR [#42942](https://github.com/supabase/supabase/pull/42942) -- Fixed inconsistent `storage` service entry ordering in `docker-compose.yml` and `docker-compose.s3.yml` to improve diff readability (requires `docker-compose.yml` and `docker-compose.s3.yml` update) - PR [#42942](https://github.com/supabase/supabase/pull/42942) +- Fixed inconsistent `storage` service entry ordering in `docker-compose.yml` and `docker-compose.s3.yml` to improve diff readability - PR [#42942](https://github.com/supabase/supabase/pull/42942) ### Edge Runtime - -- Added a `deno-cache` named volume to to avoid re-downloading dependencies (requires `docker-compose.yml` and `volumes/functions/*` update) - PR [#40822](https://github.com/supabase/supabase/pull/40822) +- Added a `deno-cache` named volume to avoid re-downloading dependencies (requires `docker-compose.yml` and `volumes/functions/*` update) - PR [#40822](https://github.com/supabase/supabase/pull/40822) --- ## [2026-02-16] -⚠️ **Note:** This update includes several breaking changes, including a security fix for Analytics. Please check the details below. The following configuration files have been updated: `docker-compose.yml`, `.env.example`, `docker-compose.s3.yml`, `volumes/api/kong.yml`, and `volumes/logs/vector.yml`. +⚠️ **Note:** This update includes several breaking changes, including a security fix for Analytics. Please check the details below. The following configuration files have been updated: `docker-compose.yml`, `.env.example`, `docker-compose.s3.yml`, `volumes/api/kong.yml`, and `volumes/logs/vector.yml`. ### Studio - Updated to `2026.02.16-sha-26c615c` @@ -41,35 +150,29 @@ Check updates for each service to learn more. - Updated to `v0.6.3` - [Release](https://github.com/supabase-community/supabase-mcp/releases/tag/v0.6.3) ### Auth - - Updated to `v2.186.0` - [Changelog](https://github.com/supabase/auth/blob/master/CHANGELOG.md) | [Release](https://github.com/supabase/auth/releases/tag/v2.186.0) ### PostgREST - - Updated to `v14.5` - [Changelog](https://github.com/PostgREST/postgrest/blob/main/CHANGELOG.md) | [Release](https://github.com/PostgREST/postgrest/releases/tag/v14.5) ### Realtime - - Updated to `v2.76.5` - [Release](https://github.com/supabase/realtime/releases/tag/v2.76.5) ### Storage - - Updated to `v1.37.8` - [Release](https://github.com/supabase/storage/releases/tag/v1.37.8) -- ⚠️ Added configuration to access buckets via `/storage/v1/s3` endpoint (requires `docker-compose.s3.yml` update) - PR [#37185](https://github.com/supabase/supabase/pull/37185) - ⚠️ Changed environment variable configuration for Storage (requires `docker-compose.yml`, `.env.example` and `.env` update) - PR [#37185](https://github.com/supabase/supabase/pull/37185), PR [#42862](https://github.com/supabase/supabase/pull/42862) +- ⚠️ Added **default** configuration to access buckets via `/storage/v1/s3` endpoint (requires `docker-compose.yml` and `.env` update) - PR [#37185](https://github.com/supabase/supabase/pull/37185) +- ⚠️ Changed MinIO configuration for the S3 backend (requires `docker-compose.s3.yml` and `.env` update) - PR [#37185](https://github.com/supabase/supabase/pull/37185) ### Edge Runtime - - Updated to `v1.70.3` - [Release](https://github.com/supabase/edge-runtime/releases/tag/v1.70.3) ### Analytics (Logflare) - - Updated to `v1.31.2` - [Release](https://github.com/Logflare/logflare/releases/tag/v1.31.2) -- ⚠️ Changed default configuration to disable Logflare on `0.0.0.0:4000` to prevent access to `/dashboard` (requires `docker-compose.yml` update). Read more in "Production Recommendations" section of Logflare [documentation](https://supabase.com/docs/reference/self-hosting-analytics/introduction) - PR [#42857](https://github.com/supabase/supabase/pull/42857) +- ⚠️ Changed default configuration to disable Logflare on `0.0.0.0:4000` to prevent access to `/dashboard` (requires `docker-compose.yml` update). Read more in the "Production Recommendations" section of Logflare [documentation](https://supabase.com/docs/reference/self-hosting-analytics/introduction) - PR [#42857](https://github.com/supabase/supabase/pull/42857) - ⚠️ Changed Kong routes to not include `/analytics/v1` by default (requires `/volumes/api/kong.yml` update) - PR [#42857](https://github.com/supabase/supabase/pull/42857) ### Vector - - Updated to `0.53.0-alpine` - [Changelog](https://vector.dev/releases/0.53.0/) | [Release](https://github.com/vectordotdev/vector/releases/tag/v0.53.0) - ⚠️ Major version jump from `0.28.1` (requires `volumes/logs/vector.yml` update) - PR [#42525](https://github.com/supabase/supabase/pull/42525) - ⚠️ Changed Postgres sink configuration to bypass Kong (requires `volumes/logs/vector.yml` update) - PR [#42857](https://github.com/supabase/supabase/pull/42857) @@ -89,13 +192,13 @@ Check updates for each service to learn more. ### Studio - Updated to `2026.01.27-sha-6aa59ff` -- Added SQL snippets (requires `docker-compose.yml` update) - PR [#41112](https://github.com/supabase/supabase/pull/41112), PR [#41557](https://github.com/supabase/supabase/pull/41557) +- Added SQL snippets (requires `docker-compose.yml` update) - PR [#41112](https://github.com/supabase/supabase/pull/41112), PR [#41557](https://github.com/supabase/supabase/pull/41557), discussion [#42031](https://github.com/orgs/supabase/discussions/42031) - Fixed type generator - PR [#40481](https://github.com/supabase/supabase/pull/40481) - Fixed minor UI discrepancies - PR [#40579](https://github.com/supabase/supabase/pull/40579), PR [#41936](https://github.com/supabase/supabase/pull/41936), PR [#41970](https://github.com/supabase/supabase/pull/41970), PR [#41971](https://github.com/supabase/supabase/pull/41971), PR [#41972](https://github.com/supabase/supabase/pull/41972), PR [#42015](https://github.com/supabase/supabase/pull/42015) ### Auth - Updated to `v2.185.0` - [Changelog](https://github.com/supabase/auth/blob/master/CHANGELOG.md) | [Release](https://github.com/supabase/auth/releases/tag/v2.185.0) -- ⚠️ Fixed security related issues +- ⚠️ Fixed security-related issues ### PostgREST - Updated to `v14.3` - [Changelog](https://github.com/PostgREST/postgrest/blob/main/CHANGELOG.md) | [Release](https://github.com/PostgREST/postgrest/releases/tag/v14.3) @@ -159,34 +262,27 @@ Check updates for each service to learn more. ## [2025-12-10] ### Studio - - Updated to `2025.12.09-sha-434634f` - ⚠️ Fixed security issues related to [React2Shell](https://vercel.com/kb/bulletin/react2shell) ### MCP Server - - Updated to `v0.5.9` - [Release](https://github.com/supabase-community/supabase-mcp/releases/tag/v0.5.9) - ⚠️ Changed MCP tool `get_anon_key` to `get_publishable_keys` ### PostgREST - - Updated to `v14.1` - [Changelog](https://github.com/PostgREST/postgrest/blob/main/CHANGELOG.md) | [Release](https://github.com/PostgREST/postgrest/releases/tag/v14.1) - ⚠️ **Major upgrade from v13.x to v14.x** - please report any unexpected behavior ### Realtime - -- Updated to `v2.68.0` - [Releases](https://github.com/supabase/realtime/releases/tag/v2.68.0) +- Updated to `v2.68.0` - [Release](https://github.com/supabase/realtime/releases/tag/v2.68.0) ### Storage - - Updated to `v1.33.0` - [Release](https://github.com/supabase/storage/releases/tag/v1.33.0) ### Edge Runtime - - Updated to `v1.69.28` - [Release](https://github.com/supabase/edge-runtime/releases/tag/v1.69.28) ### Analytics (Logflare) - - Updated to `v1.26.25` - [Release](https://github.com/Logflare/logflare/releases/tag/v1.26.25) --- @@ -231,7 +327,7 @@ Check updates for each service to learn more. ### Realtime - Updated to `v2.65.2` - [Release](https://github.com/supabase/realtime/releases/tag/v2.65.2) -- Fixed handling of boolean configurations options - PR [realtime#1614](https://github.com/supabase/realtime/pull/1614) +- Fixed handling of boolean configuration options - PR [realtime#1614](https://github.com/supabase/realtime/pull/1614) ### Storage - Updated to `v1.32.0` - [Release](https://github.com/supabase/storage/releases/tag/v1.32.0) diff --git a/supabase/code/README.md b/supabase/code/README.md index c774254fe..605ee3f3b 100644 --- a/supabase/code/README.md +++ b/supabase/code/README.md @@ -1,3 +1,10 @@ +
+ +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/supabase/supabase/3-self-hosted-deployment) + +
+ # Self-Hosted Supabase with Docker This is the official Docker Compose setup for self-hosted Supabase. It provides a complete stack with all Supabase services running locally or on your infrastructure. @@ -33,9 +40,10 @@ This Docker Compose configuration includes the following services: ## Documentation -- **[Documentation](https://supabase.com/docs/guides/self-hosting/docker)** - Setup and configuration guides +- **[Self-Hosting with Docker](https://supabase.com/docs/guides/self-hosting/docker)** - Setup and configuration guides - **[CHANGELOG.md](./CHANGELOG.md)** - Track recent updates and changes to services - **[versions.md](./versions.md)** - Complete history of Docker image versions for rollback reference +- **[Ask DeepWiki / Supabase](https://deepwiki.com/supabase/supabase/3-self-hosted-deployment)** - DeepWiki-generated description of self-hosted configuration ## Updates diff --git a/supabase/code/docker-compose.caddy.yml b/supabase/code/docker-compose.caddy.yml index b31ee8db5..35510081e 100644 --- a/supabase/code/docker-compose.caddy.yml +++ b/supabase/code/docker-compose.caddy.yml @@ -1,7 +1,22 @@ services: + # By default, Kong is used as the API gateway and its ports/env are reset + # below so Caddy can terminate TLS in front of it. + # + # When running Envoy instead, e.g.: + # docker compose -f docker-compose.yml -f docker-compose.envoy.yml \ + # -f docker-compose.caddy.yml up -d + # comment out the `kong:` block below and uncomment the `api-gw:` block + # (and the matching `depends_on` entry further down) so Caddy sits in front + # of Envoy rather than Kong. + + #api-gw: + # ports: !reset [] + kong: ports: !reset [] + environment: + KONG_PORT_MAPS: "443:8000,443:8443" caddy: container_name: supabase-caddy @@ -12,8 +27,12 @@ services: - "443:443" - "443:443/udp" depends_on: + #api-gw: + # condition: service_healthy kong: condition: service_healthy + studio: + condition: service_healthy environment: PROXY_DOMAIN: ${PROXY_DOMAIN} PROXY_AUTH_USERNAME: ${DASHBOARD_USERNAME} diff --git a/supabase/code/docker-compose.envoy.yml b/supabase/code/docker-compose.envoy.yml new file mode 100644 index 000000000..315535240 --- /dev/null +++ b/supabase/code/docker-compose.envoy.yml @@ -0,0 +1,51 @@ +# Envoy override for Kong +# Usage: docker compose -f docker-compose.yml -f docker-compose.envoy.yml up + +services: + # Disable the original Kong service + kong: + profiles: + - disabled + + # Rewire dependencies that require Kong to Envoy + functions: + depends_on: !override + api-gw: + condition: service_healthy + + # Envoy API gateway + api-gw: + container_name: supabase-envoy + image: envoyproxy/envoy:v1.37.2 + restart: unless-stopped + ports: + - ${KONG_HTTP_PORT}:8000/tcp + volumes: + - ./volumes/api/envoy/envoy.yaml:/etc/envoy/envoy.yaml:ro + - ./volumes/api/envoy/cds.yaml:/etc/envoy/cds.yaml:ro + - ./volumes/api/envoy/lds.template.yaml:/etc/envoy/lds.template.yaml:ro + - ./volumes/api/envoy/docker-entrypoint.sh:/docker-entrypoint.sh:ro + depends_on: + studio: + condition: service_healthy + environment: + ANON_KEY: ${ANON_KEY} + SERVICE_ROLE_KEY: ${SERVICE_ROLE_KEY} + SUPABASE_PUBLISHABLE_KEY: ${SUPABASE_PUBLISHABLE_KEY:-} + SUPABASE_SECRET_KEY: ${SUPABASE_SECRET_KEY:-} + ANON_KEY_ASYMMETRIC: ${ANON_KEY_ASYMMETRIC:-} + SERVICE_ROLE_KEY_ASYMMETRIC: ${SERVICE_ROLE_KEY_ASYMMETRIC:-} + DASHBOARD_USERNAME: ${DASHBOARD_USERNAME} + DASHBOARD_PASSWORD: ${DASHBOARD_PASSWORD} + entrypoint: ["/bin/sh", "/docker-entrypoint.sh"] + healthcheck: + # Using a TCP port check because this image does not include curl or wget. + test: ["CMD-SHELL", "timeout 1 bash -c ' + /bin/sh -ec " + rc alias set supa-rustfs http://rustfs:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} && + rc mb --ignore-existing supa-rustfs/${GLOBAL_S3_BUCKET} + " + + storage: + depends_on: + rustfs-createbucket: + condition: service_completed_successfully + rustfs: + condition: service_healthy + environment: + STORAGE_BACKEND: s3 + GLOBAL_S3_ENDPOINT: http://rustfs:9000 + GLOBAL_S3_PROTOCOL: http + GLOBAL_S3_FORCE_PATH_STYLE: true + AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} + AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} + +volumes: + rustfs-data: diff --git a/supabase/code/docker-compose.s3.yml b/supabase/code/docker-compose.s3.yml index cb3e25c3e..165463f9d 100644 --- a/supabase/code/docker-compose.s3.yml +++ b/supabase/code/docker-compose.s3.yml @@ -15,7 +15,7 @@ services: timeout: 10s retries: 5 volumes: - - ./volumes/storage:/data:z + - minio-data:/data minio-createbucket: image: cgr.dev/chainguard/minio-client:latest-dev @@ -29,59 +29,18 @@ services: " storage: - container_name: supabase-storage - image: supabase/storage-api:v1.37.8 - restart: unless-stopped depends_on: - db: - # Disable this if you are using an external Postgres database - condition: service_healthy - rest: - condition: service_started - imgproxy: - condition: service_started minio-createbucket: condition: service_completed_successfully minio: condition: service_healthy - healthcheck: - test: - [ - "CMD", - "wget", - "--no-verbose", - "--tries=1", - "--spider", - "http://storage:5000/status" - ] - timeout: 5s - interval: 5s - retries: 3 environment: - ANON_KEY: ${ANON_KEY} - SERVICE_KEY: ${SERVICE_ROLE_KEY} - POSTGREST_URL: http://rest:3000 - PGRST_JWT_SECRET: ${JWT_SECRET} - DATABASE_URL: postgres://supabase_storage_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - REQUEST_ALLOW_X_FORWARDED_PATH: "true" - FILE_SIZE_LIMIT: 52428800 STORAGE_BACKEND: s3 - # S3 bucket when using S3 backend, directory name when using 'file' - GLOBAL_S3_BUCKET: ${GLOBAL_S3_BUCKET} - # S3 Backend configuration GLOBAL_S3_ENDPOINT: http://minio:9000 GLOBAL_S3_PROTOCOL: http GLOBAL_S3_FORCE_PATH_STYLE: true AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} - #FILE_STORAGE_BACKEND_PATH: /var/lib/storage - TENANT_ID: ${STORAGE_TENANT_ID} - # TODO: https://github.com/supabase/storage-api/issues/55 - REGION: ${REGION} - ENABLE_IMAGE_TRANSFORMATION: "true" - IMGPROXY_URL: http://imgproxy:5001 - # S3 protocol endpoint configuration - S3_PROTOCOL_ACCESS_KEY_ID: ${S3_PROTOCOL_ACCESS_KEY_ID} - S3_PROTOCOL_ACCESS_KEY_SECRET: ${S3_PROTOCOL_ACCESS_KEY_SECRET} - #volumes: - # - ./volumes/storage:/var/lib/storage:z + +volumes: + minio-data: diff --git a/supabase/code/docker-compose.yml b/supabase/code/docker-compose.yml index e5fbe1ae0..a5ffd2f5e 100644 --- a/supabase/code/docker-compose.yml +++ b/supabase/code/docker-compose.yml @@ -1,25 +1,27 @@ # Usage -# Start: docker compose up -# With helpers: docker compose -f docker-compose.yml -f ./dev/docker-compose.dev.yml up +# Start: docker compose up -d # Stop: docker compose down -# Destroy: docker compose -f docker-compose.yml -f ./dev/docker-compose.dev.yml down -v --remove-orphans +# Dev mode: docker compose -f docker-compose.yml -f ./dev/docker-compose.dev.yml up -d # Reset everything: ./reset.sh +# +# TODO: +# - Podman does not support nested variable interpolation (${A:-${B}}) +# name: supabase services: studio: - image: supabase/studio:2026.02.16-sha-26c615c + image: supabase/studio:2026.04.27-sha-5f60601 restart: unless-stopped healthcheck: test: [ - "CMD", - "node", - "-e", - "fetch('http://studio:3000/api/platform/profile').then((r) => {if - (r.status !== 200) throw new Error(r.status)})" + "CMD-SHELL", + "node -e + \"fetch('http://localhost:3000/api/platform/profile').then((r) => + {if (r.status !== 200) throw new Error(r.status)})\"" ] timeout: 10s interval: 5s @@ -28,15 +30,22 @@ services: analytics: condition: service_healthy environment: - # Binds nestjs listener to both IPv4 and IPv6 network interfaces - HOSTNAME: "::" + # Listen on all IPv4 interfaces + HOSTNAME: "0.0.0.0" STUDIO_PG_META_URL: http://meta:8080 POSTGRES_PORT: ${POSTGRES_PORT} POSTGRES_HOST: ${POSTGRES_HOST} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + + # See: https://supabase.com/docs/guides/self-hosting/remove-superuser-access + #POSTGRES_USER_READ_WRITE: postgres + PG_META_CRYPTO_KEY: ${PG_META_CRYPTO_KEY} + PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} + PGRST_DB_MAX_ROWS: ${PGRST_DB_MAX_ROWS:-1000} + PGRST_DB_EXTRA_SEARCH_PATH: ${PGRST_DB_EXTRA_SEARCH_PATH:-public} DEFAULT_ORGANIZATION_NAME: ${STUDIO_DEFAULT_ORGANIZATION} DEFAULT_PROJECT_NAME: ${STUDIO_DEFAULT_PROJECT} @@ -54,7 +63,7 @@ services: LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} LOGFLARE_URL: http://analytics:4000 - NEXT_PUBLIC_ENABLE_LOGS: true + NEXT_PUBLIC_ENABLE_LOGS: "true" # Comment to use Big Query backend for analytics NEXT_ANALYTICS_BACKEND_PROVIDER: postgres # Uncomment to use Big Query backend for analytics @@ -66,33 +75,47 @@ services: - ./volumes/functions:/app/edge-functions:Z kong: - image: kong:2.8.1 + image: kong/kong:3.9.1 restart: unless-stopped + networks: + default: + aliases: + - api-gw + healthcheck: + test: [ "CMD", "kong", "health" ] + interval: 5s + timeout: 5s + retries: 5 + depends_on: + studio: + condition: service_healthy volumes: # https://github.com/supabase/supabase/issues/12661 - ./volumes/api/kong.yml:/home/kong/temp.yml:ro,z + - ./volumes/api/kong-entrypoint.sh:/home/kong/kong-entrypoint.sh:ro,z #- ./volumes/api/server.crt:/home/kong/server.crt:ro #- ./volumes/api/server.key:/home/kong/server.key:ro - depends_on: - analytics: - condition: service_healthy environment: KONG_DATABASE: "off" - KONG_DECLARATIVE_CONFIG: /home/kong/kong.yml + KONG_DECLARATIVE_CONFIG: /usr/local/kong/kong.yml # https://github.com/supabase/cli/issues/14 KONG_DNS_ORDER: LAST,A,CNAME - KONG_PLUGINS: request-transformer,cors,key-auth,acl,basic-auth,request-termination,ip-restriction + KONG_DNS_NOT_FOUND_TTL: 1 + KONG_PLUGINS: request-transformer,cors,key-auth,acl,basic-auth,request-termination,ip-restriction,post-function KONG_NGINX_PROXY_PROXY_BUFFER_SIZE: 160k KONG_NGINX_PROXY_PROXY_BUFFERS: 64 160k + KONG_PROXY_ACCESS_LOG: /dev/stdout combined #KONG_SSL_CERT: /home/kong/server.crt #KONG_SSL_CERT_KEY: /home/kong/server.key SUPABASE_ANON_KEY: ${ANON_KEY} SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} + SUPABASE_PUBLISHABLE_KEY: ${SUPABASE_PUBLISHABLE_KEY:-} + SUPABASE_SECRET_KEY: ${SUPABASE_SECRET_KEY:-} + ANON_KEY_ASYMMETRIC: ${ANON_KEY_ASYMMETRIC:-} + SERVICE_ROLE_KEY_ASYMMETRIC: ${SERVICE_ROLE_KEY_ASYMMETRIC:-} DASHBOARD_USERNAME: ${DASHBOARD_USERNAME} DASHBOARD_PASSWORD: ${DASHBOARD_PASSWORD} - # https://unix.stackexchange.com/a/294837 - entrypoint: bash -c 'eval "echo \"$$(cat ~/temp.yml)\"" > ~/kong.yml && - /docker-entrypoint.sh kong docker-start' + entrypoint: /home/kong/kong-entrypoint.sh auth: image: supabase/gotrue:v2.186.0 @@ -114,8 +137,6 @@ services: db: # Disable this if you are using an external Postgres database condition: service_healthy - analytics: - condition: service_healthy environment: GOTRUE_API_HOST: 0.0.0.0 GOTRUE_API_PORT: 9999 @@ -132,16 +153,22 @@ services: GOTRUE_JWT_AUD: authenticated GOTRUE_JWT_DEFAULT_GROUP_NAME: authenticated GOTRUE_JWT_EXP: ${JWT_EXPIRY} + + # Legacy symmetric HS256 key GOTRUE_JWT_SECRET: ${JWT_SECRET} + # JSON array of signing JWKs (EC private + legacy symmetric) + # For Podman, use: GOTRUE_JWT_KEYS: ${JWT_KEYS} + #GOTRUE_JWT_KEYS: ${JWT_KEYS:-[]} + GOTRUE_EXTERNAL_EMAIL_ENABLED: ${ENABLE_EMAIL_SIGNUP} GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: ${ENABLE_ANONYMOUS_USERS} GOTRUE_MAILER_AUTOCONFIRM: ${ENABLE_EMAIL_AUTOCONFIRM} # Uncomment to bypass nonce check in ID Token flow. Commonly set to true when using Google Sign In on mobile. - # GOTRUE_EXTERNAL_SKIP_NONCE_CHECK: true + # GOTRUE_EXTERNAL_SKIP_NONCE_CHECK: "true" - # GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED: true + # GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED: "true" # GOTRUE_SMTP_MAX_FREQUENCY: 1s GOTRUE_SMTP_ADMIN_EMAIL: ${SMTP_ADMIN_EMAIL} GOTRUE_SMTP_HOST: ${SMTP_HOST} @@ -194,6 +221,14 @@ services: # GOTRUE_MFA_PHONE_VERIFY_ENABLED: ${MFA_PHONE_VERIFY_ENABLED} # GOTRUE_MFA_MAX_ENROLLED_FACTORS: ${MFA_MAX_ENROLLED_FACTORS} + # SAML SSO + # GOTRUE_SAML_ENABLED: ${SAML_ENABLED} + # GOTRUE_SAML_PRIVATE_KEY: ${SAML_PRIVATE_KEY} + # GOTRUE_SAML_ALLOW_ENCRYPTED_ASSERTIONS: ${SAML_ALLOW_ENCRYPTED_ASSERTIONS} + # GOTRUE_SAML_RELAY_STATE_VALIDITY_PERIOD: ${SAML_RELAY_STATE_VALIDITY_PERIOD} + # GOTRUE_SAML_EXTERNAL_URL: ${SAML_EXTERNAL_URL} + # GOTRUE_SAML_RATE_LIMIT_ASSERTION: ${SAML_RATE_LIMIT_ASSERTION} + # Uncomment to enable custom access token hook. # See: https://supabase.com/docs/guides/auth/auth-hooks for # full list of hooks and additional details about custom_access_token_hook @@ -217,19 +252,24 @@ services: # GOTRUE_HOOK_SEND_EMAIL_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" rest: - image: postgrest/postgrest:v14.5 + image: postgrest/postgrest:v14.8 restart: unless-stopped depends_on: db: # Disable this if you are using an external Postgres database condition: service_healthy - analytics: - condition: service_healthy environment: PGRST_DB_URI: postgres://authenticator:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} + PGRST_DB_MAX_ROWS: ${PGRST_DB_MAX_ROWS:-1000} + PGRST_DB_EXTRA_SEARCH_PATH: ${PGRST_DB_EXTRA_SEARCH_PATH:-public} PGRST_DB_ANON_ROLE: anon - PGRST_JWT_SECRET: ${JWT_SECRET} + + # PostgREST accepts a plain-text symmetric secret, a single JWK, or a JWKS. + # For Podman, use either PGRST_JWT_SECRET: ${JWT_SECRET} or + # PGRST_JWT_SECRET: ${JWT_JWKS} + PGRST_JWT_SECRET: ${JWT_JWKS:-${JWT_SECRET}} + PGRST_DB_USE_LEGACY_GUCS: "false" PGRST_APP_SETTINGS_JWT_SECRET: ${JWT_SECRET} PGRST_APP_SETTINGS_JWT_EXP: ${JWT_EXPIRY} @@ -243,8 +283,6 @@ services: db: # Disable this if you are using an external Postgres database condition: service_healthy - analytics: - condition: service_healthy healthcheck: test: [ @@ -265,8 +303,16 @@ services: DB_NAME: ${POSTGRES_DB} DB_AFTER_CONNECT_QUERY: 'SET search_path TO _realtime' DB_ENC_KEY: supabaserealtime + + # Legacy symmetric HS256 key API_JWT_SECRET: ${JWT_SECRET} + + # JWKS for token verification (EC public + legacy symmetric). + # For Podman, use: API_JWT_JWKS: ${JWT_JWKS} + #API_JWT_JWKS: ${JWT_JWKS:-{"keys":[]}} + SECRET_KEY_BASE: ${SECRET_KEY_BASE} + METRICS_JWT_SECRET: ${JWT_SECRET} ERL_AFLAGS: -proto_dist inet_tcp DNS_NODES: "''" RLIMIT_NOFILE: "10000" @@ -277,7 +323,7 @@ services: # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up storage: - image: supabase/storage-api:v1.37.8 + image: supabase/storage-api:v1.48.26 restart: unless-stopped depends_on: db: @@ -300,12 +346,21 @@ services: timeout: 5s interval: 5s retries: 3 + start_period: 10s environment: ANON_KEY: ${ANON_KEY} SERVICE_KEY: ${SERVICE_ROLE_KEY} POSTGREST_URL: http://rest:3000 - PGRST_JWT_SECRET: ${JWT_SECRET} + + # Legacy symmetric HS256 key + AUTH_JWT_SECRET: ${JWT_SECRET} + + # JWKS for token verification (EC public + legacy symmetric). + # For Podman, use: JWT_JWKS: ${JWT_JWKS} + #JWT_JWKS: ${JWT_JWKS:-{"keys":[]}} + DATABASE_URL: postgres://supabase_storage_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + STORAGE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} REQUEST_ALLOW_X_FORWARDED_PATH: "true" FILE_SIZE_LIMIT: 52428800 STORAGE_BACKEND: file @@ -314,7 +369,7 @@ services: # S3 Backend configuration #GLOBAL_S3_ENDPOINT: https://your-s3-endpoint #GLOBAL_S3_PROTOCOL: https - #GLOBAL_S3_FORCE_PATH_STYLE: true + #GLOBAL_S3_FORCE_PATH_STYLE: "true" #AWS_ACCESS_KEY_ID: your-access-key-id #AWS_SECRET_ACCESS_KEY: your-secret-access-key FILE_STORAGE_BACKEND_PATH: /var/lib/storage @@ -343,18 +398,16 @@ services: IMGPROXY_BIND: ":5001" IMGPROXY_LOCAL_FILESYSTEM_ROOT: / IMGPROXY_USE_ETAG: "true" - IMGPROXY_ENABLE_WEBP_DETECTION: ${IMGPROXY_ENABLE_WEBP_DETECTION} + IMGPROXY_AUTO_WEBP: ${IMGPROXY_AUTO_WEBP} IMGPROXY_MAX_SRC_RESOLUTION: 16.8 meta: - image: supabase/postgres-meta:v0.95.2 + image: supabase/postgres-meta:v0.96.3 restart: unless-stopped depends_on: db: # Disable this if you are using an external Postgres database condition: service_healthy - analytics: - condition: service_healthy environment: PG_META_PORT: 8080 PG_META_DB_HOST: ${POSTGRES_HOST} @@ -365,27 +418,32 @@ services: CRYPTO_KEY: ${PG_META_CRYPTO_KEY} functions: - image: supabase/edge-runtime:v1.70.3 + image: supabase/edge-runtime:v1.71.2 restart: unless-stopped volumes: - ./volumes/functions:/home/deno/functions:Z - deno-cache:/root/.cache/deno depends_on: - analytics: + kong: condition: service_healthy environment: + # Legacy symmetric HS256 key JWT_SECRET: ${JWT_SECRET} SUPABASE_URL: http://kong:8000 SUPABASE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} + # Legacy API keys (HS256-signed JWTs) SUPABASE_ANON_KEY: ${ANON_KEY} SUPABASE_SERVICE_ROLE_KEY: ${SERVICE_ROLE_KEY} + # New opaque API keys + SUPABASE_PUBLISHABLE_KEYS: "{\"default\":\"${SUPABASE_PUBLISHABLE_KEY:-}\"}" + SUPABASE_SECRET_KEYS: "{\"default\":\"${SUPABASE_SECRET_KEY:-}\"}" SUPABASE_DB_URL: postgresql://postgres:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - # TODO: Allow configuring VERIFY_JWT per function. This PR might help: https://github.com/supabase/cli/pull/786 + # TODO: Allow configuring VERIFY_JWT per function. VERIFY_JWT: "${FUNCTIONS_VERIFY_JWT}" command: [ "start", "--main-service", "/home/deno/functions/main" ] analytics: - image: supabase/logflare:1.31.2 + image: supabase/logflare:1.36.1 restart: unless-stopped # ports: # - 4000:4000 @@ -396,7 +454,7 @@ services: # target: /opt/app/rel/logflare/bin/gcloud.json # read_only: true healthcheck: - test: [ "CMD", "curl", "http://localhost:4000/health" ] + test: [ "CMD-SHELL", "curl -sSfL -o /dev/null http://localhost:4000/health" ] timeout: 5s interval: 5s retries: 10 @@ -414,8 +472,8 @@ services: DB_SCHEMA: _analytics LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} - LOGFLARE_SINGLE_TENANT: true - LOGFLARE_SUPABASE_MODE: true + LOGFLARE_SINGLE_TENANT: "true" + LOGFLARE_SUPABASE_MODE: "true" # Comment variables to use Big Query backend for analytics POSTGRES_BACKEND_URL: postgresql://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase @@ -452,9 +510,6 @@ services: interval: 5s timeout: 5s retries: 10 - depends_on: - vector: - condition: service_healthy environment: POSTGRES_HOST: /var/run/postgresql PGPORT: ${POSTGRES_PORT} @@ -518,19 +573,19 @@ services: ] interval: 10s timeout: 5s - retries: 5 + retries: 10 + start_period: 30s depends_on: db: condition: service_healthy - analytics: - condition: service_healthy environment: PORT: 4000 POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_HOST: ${POSTGRES_HOST} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} DATABASE_URL: ecto://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase - CLUSTER_POSTGRES: true + CLUSTER_POSTGRES: "true" SECRET_KEY_BASE: ${SECRET_KEY_BASE} VAULT_ENC_KEY: ${VAULT_ENC_KEY} API_JWT_SECRET: ${JWT_SECRET} diff --git a/supabase/code/tests/docker-compose.rustfs.test.yml b/supabase/code/tests/docker-compose.rustfs.test.yml new file mode 100644 index 000000000..e7a7641c8 --- /dev/null +++ b/supabase/code/tests/docker-compose.rustfs.test.yml @@ -0,0 +1,11 @@ +# Test override: exposes the S3 backend port for direct testing. +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.rustfs.yml \ +# -f ./tests/docker-compose.rustfs.test.yml up -d +# + +services: + rustfs: + ports: + - "${S3_BACKEND_TEST_PORT:-9100}:9000" diff --git a/supabase/code/tests/docker-compose.s3.test.yml b/supabase/code/tests/docker-compose.s3.test.yml new file mode 100644 index 000000000..c2e34f4d1 --- /dev/null +++ b/supabase/code/tests/docker-compose.s3.test.yml @@ -0,0 +1,13 @@ +# Test override: exposes the S3 backend port for direct testing. +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.s3.yml \ +# -f ./tests/docker-compose.s3.test.yml up -d +# +# When swapping to a different S3 backend (e.g. RustFS), update the +# service name and internal port to match the new backend. + +services: + minio: + ports: + - "${S3_BACKEND_TEST_PORT:-9100}:9000" diff --git a/supabase/code/tests/test-auth-keys.sh b/supabase/code/tests/test-auth-keys.sh new file mode 100644 index 000000000..03a90f979 --- /dev/null +++ b/supabase/code/tests/test-auth-keys.sh @@ -0,0 +1,369 @@ +#!/bin/sh +# +# Test API key types and asymmetric auth against a running self-hosted instance. +# +# Usage: +# sh test-auth-keys.sh # Uses http://localhost:8000 +# sh test-auth-keys.sh # Custom URL +# +# Prerequisites: +# - Running self-hosted Supabase instance +# - .env file with all keys configured +# - jq (for JSON parsing) +# - node >= 16 (for HS256 token minting test only) +# + +set -e + +BASE_URL="${1:-http://localhost:8000}" + +if [ ! -f .env ]; then + echo "Error: .env file not found. Run from the project directory." + exit 1 +fi + +for cmd in jq node; do + if ! command -v $cmd >/dev/null 2>&1; then + echo "Error: $cmd not found." + exit 1 + fi +done + +# Read keys from .env +JWT_SECRET=$(grep '^JWT_SECRET=' .env | cut -d= -f2-) +ANON_KEY=$(grep '^ANON_KEY=' .env | cut -d= -f2-) +SERVICE_ROLE_KEY=$(grep '^SERVICE_ROLE_KEY=' .env | cut -d= -f2-) +SUPABASE_PUBLISHABLE_KEY=$(grep '^SUPABASE_PUBLISHABLE_KEY=' .env | cut -d= -f2-) +SUPABASE_SECRET_KEY=$(grep '^SUPABASE_SECRET_KEY=' .env | cut -d= -f2-) + +pass=0 +fail=0 + +check() { + test_name="$1" + expected="$2" + actual="$3" + + if [ "$actual" = "$expected" ]; then + echo " PASS: $test_name (HTTP $actual)" + pass=$((pass + 1)) + else + echo " FAIL: $test_name (expected $expected, got $actual)" + fail=$((fail + 1)) + fi +} + +http_status() { + url="$1" + shift + curl -s -o /dev/null -w "%{http_code}" "$@" "$url" +} + +echo "" +echo "=== Testing against $BASE_URL ===" +echo "" + +# --------------------------------------------- +# 1. Route tests with API key types +# --------------------------------------------- + +echo "--- REST API (/rest/v1/) ---" +check "Legacy ANON_KEY" "200" \ + "$(http_status "$BASE_URL/rest/v1/" -H "apikey: $ANON_KEY")" +check "Legacy SERVICE_ROLE_KEY" "200" \ + "$(http_status "$BASE_URL/rest/v1/" -H "apikey: $SERVICE_ROLE_KEY")" + +if [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + check "New PUBLISHABLE_KEY" "200" \ + "$(http_status "$BASE_URL/rest/v1/" -H "apikey: $SUPABASE_PUBLISHABLE_KEY")" + check "New SECRET_KEY" "200" \ + "$(http_status "$BASE_URL/rest/v1/" -H "apikey: $SUPABASE_SECRET_KEY")" +else + echo " SKIP: Opaque keys not configured" +fi + +check "No key -> 401" "401" \ + "$(http_status "$BASE_URL/rest/v1/")" +check "Invalid key -> 401" "401" \ + "$(http_status "$BASE_URL/rest/v1/" -H "apikey: invalid-key")" + +echo "" +echo "--- Auth (/auth/v1/settings) ---" +check "Legacy ANON_KEY" "200" \ + "$(http_status "$BASE_URL/auth/v1/settings" -H "apikey: $ANON_KEY")" + +if [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + check "New PUBLISHABLE_KEY" "200" \ + "$(http_status "$BASE_URL/auth/v1/settings" -H "apikey: $SUPABASE_PUBLISHABLE_KEY")" +fi + +check "No key -> 401" "401" \ + "$(http_status "$BASE_URL/auth/v1/settings")" + +echo "" +echo "--- Storage (/storage/v1/bucket) ---" +# Storage has no key-auth - passes through, Storage returns its own errors +check "No key -> not 401 (Storage handles auth)" "true" \ + "$([ "$(http_status "$BASE_URL/storage/v1/bucket")" != "401" ] && echo true || echo false)" +check "Legacy ANON_KEY" "200" \ + "$(http_status "$BASE_URL/storage/v1/bucket" -H "apikey: $ANON_KEY" -H "Authorization: Bearer $ANON_KEY")" + +if [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + # With opaque key, Kong translates to asymmetric JWT in Authorization + check "New PUBLISHABLE_KEY" "200" \ + "$(http_status "$BASE_URL/storage/v1/bucket" -H "apikey: $SUPABASE_PUBLISHABLE_KEY")" +fi + +echo "" +echo "--- Storage S3 (/storage/v1/s3/) ---" +# S3 uses AWS SigV4 auth (not apikey) - the request-transformer Lua expression +# passes the Authorization header through unchanged for non-sb_ values +check "S3 route accessible" "true" \ + "$([ "$(http_status "$BASE_URL/storage/v1/s3/")" != "502" ] && echo true || echo false)" + +echo "" +echo "--- GraphQL (/graphql/v1) ---" +check "Legacy ANON_KEY" "200" \ + "$(http_status "$BASE_URL/graphql/v1" \ + -H "apikey: $ANON_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query":"{ __typename }"}')" + +if [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + check "New PUBLISHABLE_KEY" "200" \ + "$(http_status "$BASE_URL/graphql/v1" \ + -H "apikey: $SUPABASE_PUBLISHABLE_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query":"{ __typename }"}')" +fi + +check "No key -> 401" "401" \ + "$(http_status "$BASE_URL/graphql/v1" \ + -H "Content-Type: application/json" \ + -d '{"query":"{ __typename }"}')" + +echo "" +echo "--- Realtime REST (/realtime/v1/api/) ---" +# Realtime REST API - expect 200 or other non-401 response with valid key +check "Legacy ANON_KEY -> not 401" "true" \ + "$([ "$(http_status "$BASE_URL/realtime/v1/api/tenants" -H "apikey: $ANON_KEY")" != "401" ] && echo true || echo false)" + +if [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + check "New PUBLISHABLE_KEY -> not 401" "true" \ + "$([ "$(http_status "$BASE_URL/realtime/v1/api/tenants" -H "apikey: $SUPABASE_PUBLISHABLE_KEY")" != "401" ] && echo true || echo false)" +fi + +check "No key -> 401" "401" \ + "$(http_status "$BASE_URL/realtime/v1/api/tenants")" + +echo "" +echo "--- supabase-js style requests (apikey + Authorization) ---" +# supabase-js sends both apikey header AND Authorization: Bearer +if [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + check "apikey + Authorization: Bearer sb_ (replace path)" "200" \ + "$(http_status "$BASE_URL/rest/v1/" \ + -H "apikey: $SUPABASE_PUBLISHABLE_KEY" \ + -H "Authorization: Bearer $SUPABASE_PUBLISHABLE_KEY")" +fi + +check "Legacy apikey + Authorization: Bearer " "200" \ + "$(http_status "$BASE_URL/rest/v1/" \ + -H "apikey: $ANON_KEY" \ + -H "Authorization: Bearer $ANON_KEY")" + +echo "" +echo "--- Edge cases ---" +# Opaque key in Authorization only (no apikey header) - should be rejected by key-auth +if [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + check "sb_ in Authorization only (no apikey) -> 401" "401" \ + "$(http_status "$BASE_URL/rest/v1/" \ + -H "Authorization: Bearer $SUPABASE_PUBLISHABLE_KEY")" +fi + +echo "" +echo "--- JWKS endpoint ---" +check "JWKS public endpoint (no auth)" "200" \ + "$(http_status "$BASE_URL/auth/v1/.well-known/jwks.json")" + +# Verify JWKS content: should have EC key, should NOT have symmetric key +jwks_content=$(curl -s "$BASE_URL/auth/v1/.well-known/jwks.json") +jwks_has_ec=$(echo "$jwks_content" | jq -r '[.keys[] | .kty] | if any(. == "EC") then "true" else "false" end' 2>/dev/null) +jwks_has_oct=$(echo "$jwks_content" | jq -r '[.keys[] | .kty] | if any(. == "oct") then "true" else "false" end' 2>/dev/null) +check "JWKS contains EC public key" "true" "$jwks_has_ec" +check "JWKS does NOT contain symmetric key" "false" "$jwks_has_oct" + +#echo "" +#echo "--- OAuth metadata endpoint ---" +#check "well-known oauth (no auth)" "200" \ +# "$(http_status "$BASE_URL/.well-known/oauth-authorization-server")" + +echo "" +echo "--- Realtime WebSocket upgrade ---" +# Test that WebSocket upgrade request gets through (expect 101 or non-401) +# curl --max-time to prevent hanging on successful upgrade (101 keeps connection open) +ws_status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 \ + "$BASE_URL/realtime/v1/websocket?apikey=$ANON_KEY&vsn=1.0.0" \ + -H "Upgrade: websocket" \ + -H "Connection: Upgrade" \ + -H "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==" \ + -H "Sec-WebSocket-Version: 13" 2>/dev/null || echo "000") +# 101 = upgrade success, 000 = timeout (connection stayed open = success) +check "WebSocket upgrade with legacy key -> not 401" "true" \ + "$([ "$ws_status" != "401" ] && echo true || echo false)" + +if [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + ws_status_new=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 \ + "$BASE_URL/realtime/v1/websocket?apikey=$SUPABASE_PUBLISHABLE_KEY&vsn=1.0.0" \ + -H "Upgrade: websocket" \ + -H "Connection: Upgrade" \ + -H "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==" \ + -H "Sec-WebSocket-Version: 13" 2>/dev/null || echo "000") + check "WebSocket upgrade with opaque key -> not 401" "true" \ + "$([ "$ws_status_new" != "401" ] && echo true || echo false)" +fi + +# --------------------------------------------- +# 2. User session JWT tests +# --------------------------------------------- + +echo "" +echo "--- User session JWT ---" + +# Create user via admin API (works regardless of email autoconfirm setting) +test_email="test-keys-$$@example.com" +test_password="test-password-123456" + +create_resp=$(curl -s "$BASE_URL/auth/v1/admin/users" \ + -X POST \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"$test_email\",\"password\":\"$test_password\",\"email_confirm\":true}") + +test_user_id=$(echo "$create_resp" | jq -r '.id // empty' 2>/dev/null) + +# Sign in to get session JWT +auth_response=$(curl -s "$BASE_URL/auth/v1/token?grant_type=password" \ + -H "apikey: $ANON_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"$test_email\",\"password\":\"$test_password\"}") + +access_token=$(echo "$auth_response" | jq -r '.access_token // empty' 2>/dev/null) + +if [ -n "$access_token" ]; then + # Check the algorithm in the JWT header + jwt_alg=$(echo "$access_token" | cut -d. -f1 | \ + jq -Rr '@base64d | fromjson | .alg // empty' 2>/dev/null) + + if [ -n "$jwt_alg" ]; then + echo " INFO: User session JWT signed with: $jwt_alg" + if [ "$jwt_alg" = "ES256" ]; then + check "JWT uses ES256 (asymmetric)" "ES256" "$jwt_alg" + else + check "JWT uses HS256 (legacy)" "HS256" "$jwt_alg" + fi + fi + + # Use the session JWT with PostgREST + check "Session JWT with PostgREST" "200" \ + "$(http_status "$BASE_URL/rest/v1/" \ + -H "apikey: $ANON_KEY" \ + -H "Authorization: Bearer $access_token")" + + # Use the session JWT with Storage + check "Session JWT with Storage" "200" \ + "$(http_status "$BASE_URL/storage/v1/bucket" \ + -H "apikey: $ANON_KEY" \ + -H "Authorization: Bearer $access_token")" + + # CRITICAL: Authenticated user + opaque key (most common supabase-js flow) + # supabase-js sends apikey: sb_publishable_xxx AND Authorization: Bearer + # The expression MUST keep the user JWT and NOT replace it with the anon asymmetric JWT + if [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + echo "" + echo "--- Authenticated user + opaque key (critical path) ---" + check "Opaque apikey + user JWT -> PostgREST uses user JWT" "200" \ + "$(http_status "$BASE_URL/rest/v1/" \ + -H "apikey: $SUPABASE_PUBLISHABLE_KEY" \ + -H "Authorization: Bearer $access_token")" + check "Opaque apikey + user JWT -> Storage uses user JWT" "200" \ + "$(http_status "$BASE_URL/storage/v1/bucket" \ + -H "apikey: $SUPABASE_PUBLISHABLE_KEY" \ + -H "Authorization: Bearer $access_token")" + check "Opaque apikey + user JWT -> Auth uses user JWT" "200" \ + "$(http_status "$BASE_URL/auth/v1/user" \ + -H "apikey: $SUPABASE_PUBLISHABLE_KEY" \ + -H "Authorization: Bearer $access_token")" + fi +else + check "Sign in test user" "true" "false" +fi + +# Clean up test user +if [ -n "$test_user_id" ]; then + curl -s -o /dev/null "$BASE_URL/auth/v1/admin/users/$test_user_id" \ + -X DELETE \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY" +fi + +# --------------------------------------------- +# 3. HS256 backward compatibility +# --------------------------------------------- + +echo "" +echo "--- HS256 backward compatibility ---" + +# Mint a legacy HS256 JWT with role=anon (simulating a pre-migration token) +hs256_token=$(JWT_SECRET="$JWT_SECRET" node -e " +const crypto = require('crypto'); +const header = Buffer.from(JSON.stringify({alg:'HS256',typ:'JWT'})).toString('base64url'); +const payload = Buffer.from(JSON.stringify({ + role:'anon',iss:'supabase', + iat:Math.floor(Date.now()/1000), + exp:Math.floor(Date.now()/1000)+3600 +})).toString('base64url'); +const sig = crypto.createHmac('sha256',process.env.JWT_SECRET) + .update(header+'.'+payload).digest('base64url'); +console.log(header+'.'+payload+'.'+sig); +" 2>/dev/null) + +if [ -n "$hs256_token" ]; then + check "HS256 token with PostgREST (backward compat)" "200" \ + "$(http_status "$BASE_URL/rest/v1/" \ + -H "apikey: $ANON_KEY" \ + -H "Authorization: Bearer $hs256_token")" +else + echo " SKIP: Could not mint HS256 token (node required)" +fi + +# --------------------------------------------- +# 4. JWT_KEYS format validation +# --------------------------------------------- + +echo "" +echo "--- JWT_KEYS format ---" + +JWT_KEYS_VAL=$(grep '^JWT_KEYS=' .env | cut -d= -f2-) +if [ -n "$JWT_KEYS_VAL" ]; then + # Auth expects a JSON array, not a JWKS object + jwt_keys_is_array=$(echo "$JWT_KEYS_VAL" | jq -r 'if type == "array" then "true" else "false" end' 2>/dev/null) + check "JWT_KEYS is JSON array (not JWKS object)" "true" "$jwt_keys_is_array" + + jwt_keys_has_sign=$(echo "$JWT_KEYS_VAL" | jq -r 'if any(.[]; .key_ops and (.key_ops | index("sign"))) then "true" else "false" end' 2>/dev/null) + check "JWT_KEYS has a signing key (key_ops: sign)" "true" "$jwt_keys_has_sign" +else + echo " SKIP: JWT_KEYS not configured" +fi + +# --------------------------------------------- +# Summary +# --------------------------------------------- + +echo "" +echo "=== Results: $pass passed, $fail failed ===" +echo "" + +if [ "$fail" -gt 0 ]; then + exit 1 +fi diff --git a/supabase/code/tests/test-container-logs.sh b/supabase/code/tests/test-container-logs.sh new file mode 100644 index 000000000..7a32f34b6 --- /dev/null +++ b/supabase/code/tests/test-container-logs.sh @@ -0,0 +1,105 @@ +#!/bin/sh +# +# Verify all self-hosted Supabase services started correctly by checking log output. +# +# Usage: +# sh test-container-logs.sh +# +# Prerequisites: +# - Running self-hosted Supabase instance (docker compose up) +# + +set -e + +pass=0 +fail=0 + +fail_msg() { + fail=$((fail + 1)) + echo " FAIL: $1" +} + +pass_msg() { + pass=$((pass + 1)) + echo " PASS: $1" +} + +# Check that a service's logs contain all expected patterns +check_logs() { + service="$1" + shift + + logs=$(docker compose logs "$service" 2>/dev/null) + if [ -z "$logs" ]; then + fail_msg "$service (no logs found)" + return + fi + + for pattern in "$@"; do + if ! echo "$logs" | grep -q -i -E "$pattern"; then + fail_msg "$service (missing: $pattern)" + return + fi + done + + pass_msg "$service" +} + +echo "" +echo "=== Checking service startup logs ===" +echo "" + +check_logs db \ + 'PostgreSQL init process complete; ready for start up.|Skipping initialization' + +check_logs auth \ + 'db worker started' + +check_logs kong \ + 'init.lua.*declarative config loaded' + +check_logs rest \ + 'Schema cache loaded in.*milliseconds' + +check_logs realtime \ + 'Starting Realtime' \ + 'Connected to Postgres database' \ + 'Janitor started' \ + 'Starting MetricsCleaner' + +check_logs storage \ + 'Started Successfully' + +check_logs studio \ + 'ready in.*s$' + +check_logs meta \ + 'Server listening at http' + +check_logs functions \ + 'main function started' + +check_logs analytics \ + 'Access LogflareWeb.Endpoint at http://localhost:4000' \ + 'Executing startup tasks' \ + 'Ensuring single tenant user is seeded' + +check_logs supavisor \ + 'Connected to Postgres database' \ + 'HEAD /api/health$' + +check_logs vector \ + 'Vector has started' + +check_logs imgproxy \ + 'Starting server at :5001' + +echo "" +echo "=== Results: $pass passed, $fail failed ===" +echo "" + +if [ "$fail" -gt 0 ]; then + echo "Inspect logs: docker compose logs " + echo "" + exit 1 +fi diff --git a/supabase/code/tests/test-pg17-upgrade.sh b/supabase/code/tests/test-pg17-upgrade.sh new file mode 100644 index 000000000..0015b868f --- /dev/null +++ b/supabase/code/tests/test-pg17-upgrade.sh @@ -0,0 +1,239 @@ +#!/bin/sh +# +# Test Postgres 15 -> 17 upgrade for self-hosted Supabase. +# +# Seeds test data on a running Postgres 15 stack, runs the upgrade script, +# and verifies data integrity + service connectivity using pgTAP. +# +# Usage: +# cd docker/ +# sudo bash tests/test-pg17-upgrade.sh +# +# Prerequisites: +# - Running self-hosted Supabase with a clean, tests-only Postgres 15: +# docker compose up -d +# - .env file with POSTGRES_PASSWORD, ANON_KEY +# + +set -eu + +DB_CONTAINER="supabase-db" + +if [ ! -f .env ]; then + echo "Error: .env file not found. Run from the docker/ directory." + exit 1 +fi + +pg_password=$(grep '^POSTGRES_PASSWORD=' .env | cut -d '=' -f 2-) +anon_key=$(grep '^ANON_KEY=' .env | cut -d '=' -f 2- || true) + +if [ -z "$pg_password" ]; then + echo "Error: POSTGRES_PASSWORD not set in .env" + exit 1 +fi + +run_sql() { + docker exec -i \ + -e PGPASSWORD="$pg_password" \ + "$DB_CONTAINER" \ + psql -h localhost -U supabase_admin -d postgres -v ON_ERROR_STOP=1 "$@" +} + +echo "" +echo "=== Postgres 15 -> 17 Upgrade Test ===" +echo "" + +# --- Verify we're starting from Postgres 15 -------------------------------- + +current_version=$(run_sql -A -t -c "SHOW server_version;" | head -1) +case "$current_version" in + 15.*) echo "Starting version: PostgreSQL $current_version" ;; + 17.*) echo "Error: Already on Postgres 17. Start with a PG15 stack."; exit 1 ;; + *) echo "Error: Unexpected version: $current_version"; exit 1 ;; +esac + +# --- Seed test data -------------------------------------------------------- +# Note: this script is designed to run against a fresh docker-compose stack, +# not an existing database with user data. + +echo "" +echo "Seeding test data on Postgres 15..." + +run_sql <<'EOSQL' +-- Test table with various column types +CREATE TABLE IF NOT EXISTS public._upgrade_test ( + id serial PRIMARY KEY, + name text NOT NULL, + value numeric(10,2), + created_at timestamptz DEFAULT now(), + metadata jsonb +); + +TRUNCATE public._upgrade_test; +INSERT INTO public._upgrade_test (name, value, metadata) VALUES + ('alpha', 1.50, '{"tag": "a"}'), + ('bravo', 2.75, '{"tag": "b"}'), + ('charlie', 3.00, '{"tag": "c"}'), + ('delta', 4.25, '{"tag": "d"}'), + ('echo', 5.99, '{"tag": "e"}'); + +-- Index +CREATE INDEX IF NOT EXISTS _upgrade_test_name_idx ON public._upgrade_test (name); + +-- Function +CREATE OR REPLACE FUNCTION public._upgrade_test_fn(n int) +RETURNS int LANGUAGE sql IMMUTABLE AS $$ + SELECT n * 2; +$$; + +-- Grant access so PostgREST can read it +GRANT SELECT ON public._upgrade_test TO anon, authenticated; +EOSQL + +pre_count=$(run_sql -A -t -c "SELECT count(*) FROM public._upgrade_test;" | tr -d '[:space:]') +pre_checksum=$(run_sql -A -t -c "SELECT md5(string_agg(name || value::text, ',' ORDER BY id)) FROM public._upgrade_test;" | tr -d '[:space:]') + +echo " Rows: $pre_count" +echo " Checksum: $pre_checksum" + +# --- Run upgrade ----------------------------------------------------------- + +echo "" +echo "Running upgrade script..." +echo "" + +bash utils/upgrade-pg17.sh --yes + +echo "" + +# --- Verify with pgTAP ---------------------------------------------------- + +echo "Running pgTAP verification..." +echo "" + +# Use a non-quoted heredoc so $pre_count and $pre_checksum are interpolated +run_sql </dev/null) || rest_status="000" + check "PostgREST connectivity" "200" "$rest_status" +fi + +# Auth health (needs apikey header through Kong) +if [ -n "$anon_key" ]; then + auth_status=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "apikey: $anon_key" \ + "http://localhost:8000/auth/v1/health" 2>/dev/null) || auth_status="000" + check "Auth service health" "200" "$auth_status" +fi + +echo "" +echo " Services: $pass passed, $fail failed" + +# --- Clean up test artifacts ---------------------------------------------- + +echo "" +echo "Cleaning up test artifacts..." + +run_sql <<'EOSQL' || true +DROP FUNCTION IF EXISTS public._upgrade_test_fn(int); +DROP TABLE IF EXISTS public._upgrade_test; +DROP EXTENSION IF EXISTS pgtap; +EOSQL + +# --- Summary -------------------------------------------------------------- + +echo "" +if [ "$fail" -gt 0 ]; then + echo "=== SOME TESTS FAILED ===" + exit 1 +fi + +echo "=== Upgrade test passed ===" +echo "" +echo "To reclaim disk space:" +echo " rm -rf ./volumes/db/data.bak.pg15 ./volumes/db/pg17_upgrade_bin_*.tar.gz" +echo "" diff --git a/supabase/code/tests/test-s3-backend.sh b/supabase/code/tests/test-s3-backend.sh new file mode 100644 index 000000000..799994acd --- /dev/null +++ b/supabase/code/tests/test-s3-backend.sh @@ -0,0 +1,358 @@ +#!/bin/sh +# +# Test S3 backend directly, bypassing the Storage service. +# +# Validates that the S3-compatible backend (MinIO, RustFS, etc.) handles +# all S3 operations that Storage relies on. Uses the aws cli so the test +# is backend-agnostic — no vendor-specific tools required. +# +# Usage: +# sh test-s3-backend.sh # Uses localhost:9100 +# sh test-s3-backend.sh # Custom URL +# +# Prerequisites: +# - Running self-hosted Supabase instance with S3 backend + test override: +# docker compose -f docker-compose.yml -f docker-compose.s3.yml \ +# -f ./tests/docker-compose.s3.test.yml up -d +# - .env file with MINIO_ROOT_USER, MINIO_ROOT_PASSWORD, GLOBAL_S3_BUCKET +# - aws cli v2 +# - jq (for JSON parsing) +# + +set -e + +cleanup_files="" +trap 'rm -f $cleanup_files' EXIT + +BACKEND_URL="${1:-http://localhost:${S3_BACKEND_TEST_PORT:-9100}}" + +if [ ! -f .env ]; then + echo "Error: .env file not found. Run from the project directory." + exit 1 +fi + +for cmd in aws jq; do + if ! command -v $cmd >/dev/null 2>&1; then + echo "Error: $cmd not found." + exit 1 + fi +done + +# Read backend credentials from .env +BACKEND_ACCESS_KEY=$(grep '^MINIO_ROOT_USER=' .env | cut -d= -f2-) +BACKEND_SECRET_KEY=$(grep '^MINIO_ROOT_PASSWORD=' .env | cut -d= -f2-) +GLOBAL_S3_BUCKET=$(grep '^GLOBAL_S3_BUCKET=' .env | cut -d= -f2-) +REGION=$(grep '^REGION=' .env | cut -d= -f2-) +REGION="${REGION:-us-east-1}" + +if [ -z "$BACKEND_ACCESS_KEY" ] || [ -z "$BACKEND_SECRET_KEY" ]; then + echo "Error: MINIO_ROOT_USER or MINIO_ROOT_PASSWORD not set in .env" + exit 1 +fi + +pass=0 +fail=0 + +check() { + test_name="$1" + expected="$2" + actual="$3" + + if [ "$actual" = "$expected" ]; then + echo " PASS: $test_name" + pass=$((pass + 1)) + else + echo " FAIL: $test_name (expected $expected, got $actual)" + fail=$((fail + 1)) + fi +} + +# Wrapper for aws commands against the backend directly +s3() { + AWS_ACCESS_KEY_ID="$BACKEND_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$BACKEND_SECRET_KEY" \ + aws "$@" --endpoint-url "$BACKEND_URL" --region "$REGION" 2>&1 +} + +bucket_name="backend-test-$$" + +echo "" +echo "=== S3 backend test against $BACKEND_URL ===" +echo "" + +# --------------------------------------------- +# 1. ListBuckets (backend reachable) +# --------------------------------------------- + +echo "--- Connectivity ---" +list_output=$(s3 s3api list-buckets --output json) +list_ok=$(echo "$list_output" | jq -r 'if .Buckets then "true" else "false" end' 2>/dev/null) +check "Backend reachable (ListBuckets)" "true" "$list_ok" + +if [ "$list_ok" != "true" ]; then + echo " Cannot reach backend. Is the test override running?" + echo " Response: $list_output" + echo "" + echo "=== Results: $pass passed, $fail failed ===" + exit 1 +fi + +# --------------------------------------------- +# 2. Verify GLOBAL_S3_BUCKET exists +# --------------------------------------------- + +echo "" +echo "--- Storage bucket ---" +if [ -n "$GLOBAL_S3_BUCKET" ]; then + storage_bucket_exists=$(s3 s3api list-buckets --output json | \ + jq -r --arg name "$GLOBAL_S3_BUCKET" '[.Buckets[] | .Name] | if any(. == $name) then "true" else "false" end' 2>/dev/null) + check "GLOBAL_S3_BUCKET ($GLOBAL_S3_BUCKET) exists" "true" "$storage_bucket_exists" +else + echo " SKIP: GLOBAL_S3_BUCKET not set" +fi + +# --------------------------------------------- +# 3. CreateBucket +# --------------------------------------------- + +echo "" +echo "--- CreateBucket ---" +s3 s3api create-bucket --bucket "$bucket_name" --output json >/dev/null 2>&1 + +# Verify create succeeded +create_found=$(s3 s3api list-buckets --output json | \ + jq -r --arg name "$bucket_name" '[.Buckets[] | .Name] | if any(. == $name) then "true" else "false" end' 2>/dev/null) +check "CreateBucket" "true" "$create_found" + +if [ "$create_found" != "true" ]; then + echo " Cannot continue without a bucket. Aborting." + echo "" + echo "=== Results: $pass passed, $fail failed ===" + exit 1 +fi + +# Verify in ListBuckets (separate call) +bucket_found=$(s3 s3api list-buckets --output json | \ + jq -r --arg name "$bucket_name" '[.Buckets[] | .Name] | if any(. == $name) then "true" else "false" end' 2>/dev/null) +check "Bucket visible in ListBuckets" "true" "$bucket_found" + +# --------------------------------------------- +# 4. PutObject +# --------------------------------------------- + +echo "" +echo "--- PutObject ---" +tmpfile=$(mktemp); cleanup_files="$cleanup_files $tmpfile" +echo "hello from backend test" > "$tmpfile" +put_output=$(s3 s3 cp "$tmpfile" "s3://$bucket_name/test-file.txt") +put_ok=$(echo "$put_output" | grep -q "upload:" && echo "true" || echo "false") +check "PutObject" "true" "$put_ok" + +# --------------------------------------------- +# 5. ListObjectsV2 +# --------------------------------------------- + +echo "" +echo "--- ListObjectsV2 ---" +list_objects=$(s3 s3api list-objects-v2 --bucket "$bucket_name" --output json) +object_found=$(echo "$list_objects" | \ + jq -r '[.Contents[]? | .Key] | if any(. == "test-file.txt") then "true" else "false" end' 2>/dev/null) +check "Object found in ListObjectsV2" "true" "$object_found" + +# --------------------------------------------- +# 6. HeadObject +# --------------------------------------------- + +echo "" +echo "--- HeadObject ---" +head_output=$(s3 s3api head-object --bucket "$bucket_name" --key "test-file.txt" --output json) +head_size=$(echo "$head_output" | jq -r '.ContentLength // 0' 2>/dev/null) +original_size=$(wc -c < "$tmpfile" | tr -d ' ') +check "HeadObject returns correct size" "$original_size" "$head_size" +rm -f "$tmpfile" + +# --------------------------------------------- +# 7. GetObject + content verify +# --------------------------------------------- + +echo "" +echo "--- GetObject ---" +download_file=$(mktemp); cleanup_files="$cleanup_files $download_file" +s3 s3 cp "s3://$bucket_name/test-file.txt" "$download_file" >/dev/null +downloaded_content=$(cat "$download_file") +check "GetObject content matches" "hello from backend test" "$downloaded_content" +rm -f "$download_file" + +# --------------------------------------------- +# 8. CopyObject +# --------------------------------------------- + +echo "" +echo "--- CopyObject ---" +copy_output=$(s3 s3 cp "s3://$bucket_name/test-file.txt" "s3://$bucket_name/test-copy.txt") +copy_ok=$(echo "$copy_output" | grep -q "copy:" && echo "true" || echo "false") +check "CopyObject" "true" "$copy_ok" + +copy_download=$(mktemp); cleanup_files="$cleanup_files $copy_download" +s3 s3 cp "s3://$bucket_name/test-copy.txt" "$copy_download" >/dev/null +check "Copied object content matches" "hello from backend test" "$(cat "$copy_download")" +rm -f "$copy_download" + +# --------------------------------------------- +# 9. DeleteObject +# --------------------------------------------- + +echo "" +echo "--- DeleteObject ---" +s3 s3 rm "s3://$bucket_name/test-copy.txt" >/dev/null +list_after_delete=$(s3 s3api list-objects-v2 --bucket "$bucket_name" --output json) +copy_gone=$(echo "$list_after_delete" | \ + jq -r '[.Contents[]? | .Key] | if any(. == "test-copy.txt") then "false" else "true" end' 2>/dev/null) +check "Deleted object no longer listed" "true" "$copy_gone" + +# --------------------------------------------- +# 10. Multipart upload (7MB) +# --------------------------------------------- + +echo "" +echo "--- Multipart upload (7MB) ---" +large_file=$(mktemp); cleanup_files="$cleanup_files $large_file" +dd if=/dev/urandom of="$large_file" bs=1048576 count=7 2>/dev/null +large_size=$(wc -c < "$large_file" | tr -d ' ') +large_put=$(s3 s3 cp "$large_file" "s3://$bucket_name/large-file.bin") +large_ok=$(echo "$large_put" | grep -q "upload:" && echo "true" || echo "false") +check "Multipart upload (7MB)" "true" "$large_ok" + +large_head=$(s3 s3api head-object --bucket "$bucket_name" --key "large-file.bin" --output json) +remote_size=$(echo "$large_head" | jq -r '.ContentLength // 0' 2>/dev/null) +check "Multipart size matches ($large_size bytes)" "$large_size" "$remote_size" + +large_download=$(mktemp); cleanup_files="$cleanup_files $large_download" +s3 s3 cp "s3://$bucket_name/large-file.bin" "$large_download" >/dev/null +download_size=$(wc -c < "$large_download" | tr -d ' ') +check "Multipart download size matches" "$large_size" "$download_size" +rm -f "$large_file" "$large_download" + +# --------------------------------------------- +# 11. DeleteObjects (batch delete) +# --------------------------------------------- + +echo "" +echo "--- DeleteObjects (batch) ---" +batch_file=$(mktemp); cleanup_files="$cleanup_files $batch_file" +echo "batch-a" > "$batch_file" +s3 s3 cp "$batch_file" "s3://$bucket_name/batch-a.txt" >/dev/null +echo "batch-b" > "$batch_file" +s3 s3 cp "$batch_file" "s3://$bucket_name/batch-b.txt" >/dev/null +echo "batch-c" > "$batch_file" +s3 s3 cp "$batch_file" "s3://$bucket_name/batch-c.txt" >/dev/null +rm -f "$batch_file" +delete_objects_output=$(s3 s3api delete-objects --bucket "$bucket_name" \ + --delete '{"Objects":[{"Key":"batch-a.txt"},{"Key":"batch-b.txt"},{"Key":"batch-c.txt"}]}' \ + --output json) +deleted_count=$(echo "$delete_objects_output" | jq -r '.Deleted | length' 2>/dev/null) +check "DeleteObjects removed 3 objects" "3" "$deleted_count" + +# Verify all gone +batch_list=$(s3 s3api list-objects-v2 --bucket "$bucket_name" --prefix "batch-" --output json) +batch_remaining=$(echo "$batch_list" | jq -r '[.Contents[]?] | length' 2>/dev/null) +check "Batch-deleted objects gone" "0" "$batch_remaining" + +# --------------------------------------------- +# 12. Presigned URLs +# --------------------------------------------- + +echo "" +echo "--- Presigned URLs ---" +presign_file=$(mktemp); cleanup_files="$cleanup_files $presign_file" +echo "presigned content test" > "$presign_file" +s3 s3 cp "$presign_file" "s3://$bucket_name/presign-test.txt" >/dev/null +rm -f "$presign_file" + +presigned_url=$(s3 s3 presign "s3://$bucket_name/presign-test.txt") +presign_body=$(curl -s "$presigned_url") +check "Presigned URL returns correct content" "presigned content test" "$presign_body" + +# --------------------------------------------- +# 13. Conditional request (IfNoneMatch) +# --------------------------------------------- + +echo "" +echo "--- Conditional request (IfNoneMatch) ---" +# --if-none-match requires aws cli v2.22+ ; skip if not supported +if aws s3api put-object help 2>&1 | grep -q 'if-none-match'; then + cond_file=$(mktemp); cleanup_files="$cleanup_files $cond_file" + echo "conditional test" > "$cond_file" + + # First put should succeed (key doesn't exist) + first_put_err=$(s3 s3api put-object --bucket "$bucket_name" --key "cond-test.txt" \ + --body "$cond_file" --if-none-match '*' --output json 2>&1 || true) + first_put_ok=$(echo "$first_put_err" | grep -qi "error\|denied\|PreconditionFailed" && echo "false" || echo "true") + check "IfNoneMatch put (new key) succeeds" "true" "$first_put_ok" + + # Second put should fail with PreconditionFailed (key exists) + cond_err=$(s3 s3api put-object --bucket "$bucket_name" --key "cond-test.txt" \ + --body "$cond_file" --if-none-match '*' --output json 2>&1 || true) + cond_rejected=$(echo "$cond_err" | grep -qi "PreconditionFailed" && echo "true" || echo "false") + check "IfNoneMatch put (existing key) rejected" "true" "$cond_rejected" + rm -f "$cond_file" +else + echo " SKIP: aws cli does not support --if-none-match (requires v2.22+)" +fi + +# --------------------------------------------- +# 14. Range request (partial GetObject) +# --------------------------------------------- + +echo "" +echo "--- Range request ---" +range_file=$(mktemp); cleanup_files="$cleanup_files $range_file" +echo "hello range test content" > "$range_file" +s3 s3 cp "$range_file" "s3://$bucket_name/range-test.txt" >/dev/null +rm -f "$range_file" + +range_download=$(mktemp); cleanup_files="$cleanup_files $range_download" +s3 s3api get-object --bucket "$bucket_name" --key "range-test.txt" \ + --range "bytes=0-4" "$range_download" --output json >/dev/null +range_content=$(cat "$range_download") +check "Range request returns partial content" "hello" "$range_content" +rm -f "$range_download" + +# --------------------------------------------- +# 15. Authentication +# --------------------------------------------- + +echo "" +echo "--- Authentication ---" +bad_output=$(AWS_ACCESS_KEY_ID="invalid-key" \ + AWS_SECRET_ACCESS_KEY="invalid-secret" \ + aws s3api list-buckets \ + --endpoint-url "$BACKEND_URL" \ + --region "$REGION" \ + --output json 2>&1 || true) +bad_ok=$(echo "$bad_output" | grep -qi "denied\|invalid\|error\|403\|401" && echo "true" || echo "false") +check "Invalid credentials rejected" "true" "$bad_ok" + +# --------------------------------------------- +# 16. Cleanup +# --------------------------------------------- + +echo "" +echo "--- Cleanup ---" +s3 s3 rm "s3://$bucket_name/" --recursive >/dev/null +s3 s3api delete-bucket --bucket "$bucket_name" >/dev/null +bucket_gone=$(s3 s3api list-buckets --output json | \ + jq -r --arg name "$bucket_name" '[.Buckets[] | .Name] | if any(. == $name) then "false" else "true" end' 2>/dev/null) +check "Test bucket deleted" "true" "$bucket_gone" + +# --------------------------------------------- +# Summary +# --------------------------------------------- + +echo "" +echo "=== Results: $pass passed, $fail failed ===" +echo "" + +if [ "$fail" -gt 0 ]; then + exit 1 +fi diff --git a/supabase/code/tests/test-s3.sh b/supabase/code/tests/test-s3.sh new file mode 100644 index 000000000..939547dd1 --- /dev/null +++ b/supabase/code/tests/test-s3.sh @@ -0,0 +1,292 @@ +#!/bin/sh +# +# Test S3 protocol endpoint for self-hosted Supabase Storage. +# +# Verifies that the S3-compatible endpoint at /storage/v1/s3 works with +# standard S3 clients — the same way end users interact with it via +# aws cli, rclone, or other S3-compatible tools. +# +# Usage: +# sh test-s3.sh # Uses http://localhost:8000 +# sh test-s3.sh # Custom URL +# +# Prerequisites: +# - Running self-hosted Supabase instance with S3 enabled: +# docker compose -f docker-compose.yml -f docker-compose.s3.yml up -d +# - .env file with S3_PROTOCOL_ACCESS_KEY_ID, S3_PROTOCOL_ACCESS_KEY_SECRET, REGION +# - aws cli v2 (for S3 operations) +# - jq (for JSON parsing) +# + +set -e + +cleanup_files="" +trap 'rm -f $cleanup_files' EXIT + +BASE_URL="${1:-http://localhost:8000}" +S3_ENDPOINT="$BASE_URL/storage/v1/s3" + +if [ ! -f .env ]; then + echo "Error: .env file not found. Run from the project directory." + exit 1 +fi + +for cmd in aws jq; do + if ! command -v $cmd >/dev/null 2>&1; then + echo "Error: $cmd not found." + exit 1 + fi +done + +# Read keys from .env +S3_ACCESS_KEY=$(grep '^S3_PROTOCOL_ACCESS_KEY_ID=' .env | cut -d= -f2-) +S3_SECRET_KEY=$(grep '^S3_PROTOCOL_ACCESS_KEY_SECRET=' .env | cut -d= -f2-) +REGION=$(grep '^REGION=' .env | cut -d= -f2-) + +if [ -z "$S3_ACCESS_KEY" ] || [ -z "$S3_SECRET_KEY" ]; then + echo "Error: S3_PROTOCOL_ACCESS_KEY_ID or S3_PROTOCOL_ACCESS_KEY_SECRET not set in .env" + exit 1 +fi + +pass=0 +fail=0 + +check() { + test_name="$1" + expected="$2" + actual="$3" + + if [ "$actual" = "$expected" ]; then + echo " PASS: $test_name" + pass=$((pass + 1)) + else + echo " FAIL: $test_name (expected $expected, got $actual)" + fail=$((fail + 1)) + fi +} + +# Wrapper for aws s3/s3api commands with correct endpoint and credentials +s3() { + AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ + AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ + aws "$@" --endpoint-url "$S3_ENDPOINT" --region "$REGION" 2>&1 +} + +bucket_name="s3-test-$$" + +echo "" +echo "=== S3 protocol test against $BASE_URL ===" +echo "" + +# --------------------------------------------- +# 1. S3 ListBuckets +# --------------------------------------------- + +echo "--- S3 ListBuckets ---" +list_output=$(s3 s3api list-buckets --output json) +list_ok=$(echo "$list_output" | jq -r 'if .Buckets then "true" else "false" end' 2>/dev/null) +check "ListBuckets returns valid response" "true" "$list_ok" + +# --------------------------------------------- +# 2. S3 CreateBucket +# --------------------------------------------- + +echo "" +echo "--- S3 CreateBucket ---" +s3 s3api create-bucket --bucket "$bucket_name" --output json >/dev/null 2>&1 + +# Verify create succeeded +create_found=$(s3 s3api list-buckets --output json | \ + jq -r --arg name "$bucket_name" '[.Buckets[] | .Name] | if any(. == $name) then "true" else "false" end' 2>/dev/null) +check "CreateBucket" "true" "$create_found" + +if [ "$create_found" != "true" ]; then + echo " Cannot continue without a bucket. Aborting." + echo "" + echo "=== Results: $pass passed, $fail failed ===" + exit 1 +fi + +# Verify bucket appears in ListBuckets (separate call) +s3_bucket_found=$(s3 s3api list-buckets --output json | \ + jq -r --arg name "$bucket_name" '[.Buckets[] | .Name] | if any(. == $name) then "true" else "false" end' 2>/dev/null) +check "Bucket visible in ListBuckets" "true" "$s3_bucket_found" + +# --------------------------------------------- +# 3. S3 PutObject +# --------------------------------------------- + +echo "" +echo "--- S3 PutObject ---" +tmpfile=$(mktemp); cleanup_files="$cleanup_files $tmpfile" +echo "hello from s3 upload test" > "$tmpfile" +put_output=$(s3 s3 cp "$tmpfile" "s3://$bucket_name/s3-uploaded.txt") +put_ok=$(echo "$put_output" | grep -q "upload:" && echo "true" || echo "false") +check "PutObject upload" "true" "$put_ok" + +# --------------------------------------------- +# 4. S3 ListObjectsV2 +# --------------------------------------------- + +echo "" +echo "--- S3 ListObjectsV2 ---" +list_objects=$(s3 s3api list-objects-v2 --bucket "$bucket_name" --output json) +object_found=$(echo "$list_objects" | \ + jq -r '[.Contents[]? | .Key] | if any(. == "s3-uploaded.txt") then "true" else "false" end' 2>/dev/null) +check "Object found in ListObjectsV2" "true" "$object_found" + +# --------------------------------------------- +# 5. S3 HeadObject +# --------------------------------------------- + +echo "" +echo "--- S3 HeadObject ---" +head_output=$(s3 s3api head-object --bucket "$bucket_name" --key "s3-uploaded.txt" --output json) +head_size=$(echo "$head_output" | jq -r '.ContentLength // 0' 2>/dev/null) +original_size=$(wc -c < "$tmpfile" | tr -d ' ') +check "HeadObject returns correct size" "$original_size" "$head_size" +rm -f "$tmpfile" + +# --------------------------------------------- +# 6. S3 GetObject (download) + content verify +# --------------------------------------------- + +echo "" +echo "--- S3 GetObject ---" +download_file=$(mktemp); cleanup_files="$cleanup_files $download_file" +s3 s3 cp "s3://$bucket_name/s3-uploaded.txt" "$download_file" >/dev/null +downloaded_content=$(cat "$download_file") +check "GetObject content matches" "hello from s3 upload test" "$downloaded_content" +rm -f "$download_file" + +# --------------------------------------------- +# 7. S3 CopyObject (server-side copy) +# --------------------------------------------- + +echo "" +echo "--- S3 CopyObject ---" +copy_output=$(s3 s3 cp "s3://$bucket_name/s3-uploaded.txt" "s3://$bucket_name/s3-copied.txt") +copy_ok=$(echo "$copy_output" | grep -q "copy:" && echo "true" || echo "false") +check "CopyObject" "true" "$copy_ok" + +# Verify copied content +copy_download=$(mktemp); cleanup_files="$cleanup_files $copy_download" +s3 s3 cp "s3://$bucket_name/s3-copied.txt" "$copy_download" >/dev/null +check "Copied object content matches" "hello from s3 upload test" "$(cat "$copy_download")" +rm -f "$copy_download" + +# --------------------------------------------- +# 8. S3 DeleteObject +# --------------------------------------------- + +echo "" +echo "--- S3 DeleteObject ---" +s3 s3 rm "s3://$bucket_name/s3-copied.txt" >/dev/null +# Verify object is gone +list_after_delete=$(s3 s3api list-objects-v2 --bucket "$bucket_name" --output json) +copied_gone=$(echo "$list_after_delete" | \ + jq -r '[.Contents[]? | .Key] | if any(. == "s3-copied.txt") then "false" else "true" end' 2>/dev/null) +check "Deleted object no longer listed" "true" "$copied_gone" + +# --------------------------------------------- +# 9. Multipart upload (>5MB triggers multipart) +# --------------------------------------------- + +echo "" +echo "--- S3 multipart upload (7MB) ---" +large_file=$(mktemp); cleanup_files="$cleanup_files $large_file" +dd if=/dev/urandom of="$large_file" bs=1048576 count=7 2>/dev/null +large_size=$(wc -c < "$large_file" | tr -d ' ') +large_put=$(s3 s3 cp "$large_file" "s3://$bucket_name/large-file.bin") +large_ok=$(echo "$large_put" | grep -q "upload:" && echo "true" || echo "false") +check "Multipart upload (7MB)" "true" "$large_ok" + +# Verify size via HeadObject +large_head=$(s3 s3api head-object --bucket "$bucket_name" --key "large-file.bin" --output json) +remote_size=$(echo "$large_head" | jq -r '.ContentLength // 0' 2>/dev/null) +check "Multipart upload size matches ($large_size bytes)" "$large_size" "$remote_size" + +# Download and verify size +large_download=$(mktemp); cleanup_files="$cleanup_files $large_download" +s3 s3 cp "s3://$bucket_name/large-file.bin" "$large_download" >/dev/null +download_size=$(wc -c < "$large_download" | tr -d ' ') +check "Multipart download size matches" "$large_size" "$download_size" +rm -f "$large_file" "$large_download" + +# --------------------------------------------- +# 10. Range request (partial GetObject) +# --------------------------------------------- + +echo "" +echo "--- Range request ---" +range_file=$(mktemp); cleanup_files="$cleanup_files $range_file" +echo "hello range test content" > "$range_file" +s3 s3 cp "$range_file" "s3://$bucket_name/range-test.txt" >/dev/null +rm -f "$range_file" + +range_download=$(mktemp); cleanup_files="$cleanup_files $range_download" +s3 s3api get-object --bucket "$bucket_name" --key "range-test.txt" \ + --range "bytes=0-4" "$range_download" --output json >/dev/null +range_content=$(cat "$range_download") +check "Range request returns partial content" "hello" "$range_content" +rm -f "$range_download" + +# --------------------------------------------- +# 11. Presigned URLs +# --------------------------------------------- +# Storage supports S3 presigned URLs (query-parameter auth), but Kong's +# request-transformer adds an empty Authorization header when the Lua +# expression evaluates to nil. Storage sees typeof "" === "string" and +# enters parseAuthorizationHeader instead of parseQuerySignature. +# This test will pass once the Kong config is fixed. + +echo "" +echo "--- Presigned URLs ---" +presign_file=$(mktemp); cleanup_files="$cleanup_files $presign_file" +echo "presigned content test" > "$presign_file" +s3 s3 cp "$presign_file" "s3://$bucket_name/presign-test.txt" >/dev/null +rm -f "$presign_file" + +presigned_url=$(s3 s3 presign "s3://$bucket_name/presign-test.txt") +presign_body=$(curl -s "$presigned_url") +check "Presigned URL returns correct content" "presigned content test" "$presign_body" + +# --------------------------------------------- +# 12. Authentication +# --------------------------------------------- + +echo "" +echo "--- Authentication ---" +bad_output=$(AWS_ACCESS_KEY_ID="invalid-key" \ + AWS_SECRET_ACCESS_KEY="invalid-secret" \ + aws s3api list-buckets \ + --endpoint-url "$S3_ENDPOINT" \ + --region "$REGION" \ + --output json 2>&1 || true) +bad_ok=$(echo "$bad_output" | grep -qi "denied\|invalid\|error\|403\|401" && echo "true" || echo "false") +check "Invalid credentials rejected" "true" "$bad_ok" + +# --------------------------------------------- +# 13. Cleanup +# --------------------------------------------- + +echo "" +echo "--- Cleanup ---" +s3 s3 rm "s3://$bucket_name/" --recursive >/dev/null +s3 s3api delete-bucket --bucket "$bucket_name" >/dev/null +# Verify bucket is gone +bucket_gone=$(s3 s3api list-buckets --output json | \ + jq -r --arg name "$bucket_name" '[.Buckets[] | .Name] | if any(. == $name) then "false" else "true" end' 2>/dev/null) +check "Bucket deleted via S3" "true" "$bucket_gone" + +# --------------------------------------------- +# Summary +# --------------------------------------------- + +echo "" +echo "=== Results: $pass passed, $fail failed ===" +echo "" + +if [ "$fail" -gt 0 ]; then + exit 1 +fi diff --git a/supabase/code/tests/test-self-hosted.sh b/supabase/code/tests/test-self-hosted.sh new file mode 100644 index 000000000..87e3c5b64 --- /dev/null +++ b/supabase/code/tests/test-self-hosted.sh @@ -0,0 +1,351 @@ +#!/bin/sh +# +# Smoke test for self-hosted Supabase - verifies core functionality end-to-end. +# +# Usage: +# sh test-self-hosted.sh # Uses http://localhost:8000 +# sh test-self-hosted.sh # Custom URL +# +# Prerequisites: +# - Running self-hosted Supabase instance +# - .env file with keys configured +# - jq (for JSON parsing) +# + +set -e + +cleanup_files="" +trap 'rm -f $cleanup_files' EXIT + +BASE_URL="${1:-http://localhost:8000}" + +if [ ! -f .env ]; then + echo "Error: .env file not found. Run from the project directory." + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq not found. Install it: https://jqlang.github.io/jq/download/" + exit 1 +fi + +# Read keys from .env +ANON_KEY=$(grep '^ANON_KEY=' .env | cut -d= -f2-) +SERVICE_ROLE_KEY=$(grep '^SERVICE_ROLE_KEY=' .env | cut -d= -f2-) +DASHBOARD_USERNAME=$(grep '^DASHBOARD_USERNAME=' .env | cut -d= -f2-) +DASHBOARD_PASSWORD=$(grep '^DASHBOARD_PASSWORD=' .env | cut -d= -f2-) + +pass=0 +fail=0 + +check() { + test_name="$1" + expected="$2" + actual="$3" + + if [ "$actual" = "$expected" ]; then + echo " PASS: $test_name" + pass=$((pass + 1)) + else + echo " FAIL: $test_name (expected $expected, got $actual)" + fail=$((fail + 1)) + fi +} + +http_status() { + url="$1" + shift + curl -s -o /dev/null -w "%{http_code}" "$@" "$url" +} + +http_body() { + url="$1" + shift + curl -s "$@" "$url" +} + +echo "" +echo "=== Self-hosted smoke test against $BASE_URL ===" +echo "" + +# --------------------------------------------- +# 1. Container health (via docker compose) +# --------------------------------------------- + +echo "--- Container health ---" +if command -v docker >/dev/null 2>&1; then + container_status=$(docker compose ps --format json 2>/dev/null | jq -rs ' + [.[] | select(.State != "running" or (.Health != "" and .Health != "healthy"))] + | (length | tostring) + "|" + ([.[] | .Service + ": State=" + .State + " Health=" + (.Health // "none")] | join(", ")) + ' 2>/dev/null || echo "?|") + unhealthy="${container_status%%|*}" + container_issues="${container_status#*|}" + if [ "$unhealthy" = "0" ]; then + check "All containers healthy" "0" "$unhealthy" + elif [ "$unhealthy" = "?" ]; then + echo " SKIP: Could not check container health" + else + check "All containers healthy ($container_issues)" "0" "$unhealthy" + fi +else + echo " SKIP: docker not available" +fi + +# --------------------------------------------- +# 2. Studio dashboard +# --------------------------------------------- + +echo "" +echo "--- Studio dashboard ---" +# Studio may redirect (307/302) after auth - follow redirects +check "Studio accessible with basic auth" "200" \ + "$(http_status "$BASE_URL/" -L -u "$DASHBOARD_USERNAME:$DASHBOARD_PASSWORD")" +check "Studio rejects without auth" "401" \ + "$(http_status "$BASE_URL/")" + +# --------------------------------------------- +# 3. Auth: create user, sign in, get user, public signup, delete +# --------------------------------------------- + +echo "" +echo "--- Auth: user lifecycle ---" + +test_email="smoke-test-$$@example.com" +test_password="smoke-test-password-123456" + +# Create user via admin API (works regardless of email autoconfirm setting) +create_resp=$(http_body "$BASE_URL/auth/v1/admin/users" \ + -X POST \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"$test_email\",\"password\":\"$test_password\",\"email_confirm\":true}") + +user_id=$(echo "$create_resp" | jq -r '.id // empty' 2>/dev/null) + +if [ -n "$user_id" ]; then + check "Create user (admin)" "true" "true" + + # Sign in via public endpoint + signin_resp=$(http_body "$BASE_URL/auth/v1/token?grant_type=password" \ + -H "apikey: $ANON_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"$test_email\",\"password\":\"$test_password\"}") + + access_token=$(echo "$signin_resp" | jq -r '.access_token // empty' 2>/dev/null) + + if [ -n "$access_token" ]; then + check "Sign in user" "true" "true" + + # Get user profile with session JWT + check "Get user profile" "200" \ + "$(http_status "$BASE_URL/auth/v1/user" \ + -H "apikey: $ANON_KEY" \ + -H "Authorization: Bearer $access_token")" + else + check "Sign in user" "true" "false" + fi + + # Delete user + delete_status=$(http_status "$BASE_URL/auth/v1/admin/users/$user_id" \ + -X DELETE \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY") + check "Delete user (admin)" "200" "$delete_status" +else + check "Create user (admin)" "true" "false" +fi + +# Public signup (optional — depends on email autoconfirm setting) +signup_email="smoke-signup-$$@example.com" +signup_resp=$(http_body "$BASE_URL/auth/v1/signup" \ + -H "apikey: $ANON_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"email\":\"$signup_email\",\"password\":\"$test_password\"}") + +signup_token=$(echo "$signup_resp" | jq -r '.access_token // empty' 2>/dev/null) +signup_user_id=$(echo "$signup_resp" | jq -r '.id // .user.id // empty' 2>/dev/null) + +if [ -n "$signup_token" ]; then + check "Public signup (autoconfirm on)" "true" "true" +else + echo " SKIP: Public signup (autoconfirm is off)" +fi + +# Clean up signup user if created +if [ -n "$signup_user_id" ]; then + http_status "$BASE_URL/auth/v1/admin/users/$signup_user_id" \ + -X DELETE \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY" >/dev/null 2>&1 +fi + +# --------------------------------------------- +# 4. PostgREST: query +# --------------------------------------------- + +echo "" +echo "--- PostgREST ---" +check "REST API query" "200" \ + "$(http_status "$BASE_URL/rest/v1/" \ + -H "apikey: $ANON_KEY")" + +# --------------------------------------------- +# 5. GraphQL +# --------------------------------------------- + +echo "" +echo "--- GraphQL ---" +gql_resp=$(http_body "$BASE_URL/graphql/v1" \ + -H "apikey: $ANON_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query":"{ __typename }"}') +gql_has_data=$(echo "$gql_resp" | jq -r 'if .data then "true" else "false" end' 2>/dev/null) +check "GraphQL introspection" "true" "$gql_has_data" + +# --------------------------------------------- +# 6. Storage: create bucket, upload >6MB file, download, cleanup +# --------------------------------------------- + +echo "" +echo "--- Storage: bucket + file lifecycle ---" + +bucket_name="smoke-test-$$" + +# Create bucket +create_bucket_status=$(http_status "$BASE_URL/storage/v1/bucket" \ + -X POST \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"id\":\"$bucket_name\",\"name\":\"$bucket_name\",\"public\":true}") +check "Create bucket" "200" "$create_bucket_status" + +if [ "$create_bucket_status" = "200" ]; then + # Generate a ~7MB file + tmpfile=$(mktemp); cleanup_files="$cleanup_files $tmpfile" + dd if=/dev/urandom of="$tmpfile" bs=1048576 count=7 2>/dev/null + + # Upload file + upload_status=$(http_status "$BASE_URL/storage/v1/object/$bucket_name/test-large-file.bin" \ + -X POST \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY" \ + -H "Content-Type: application/octet-stream" \ + --data-binary "@$tmpfile") + check "Upload 7MB file" "200" "$upload_status" + + # Download file and verify size + download_size=$(curl -s \ + "$BASE_URL/storage/v1/object/public/$bucket_name/test-large-file.bin" | wc -c | tr -d ' ') + original_size=$(wc -c < "$tmpfile" | tr -d ' ') + check "Download file (size matches)" "true" \ + "$([ "$download_size" = "$original_size" ] && echo true || echo false)" + + rm -f "$tmpfile" + + # Signed URL: upload a small file, create signed URL, fetch without auth + sign_upload_status=$(http_status "$BASE_URL/storage/v1/object/$bucket_name/sign-test.txt" \ + -X POST \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY" \ + -H "Content-Type: text/plain" \ + --data-binary "signed url test content") + check "Upload file for signing" "200" "$sign_upload_status" + + if [ "$sign_upload_status" = "200" ]; then + sign_resp=$(http_body "$BASE_URL/storage/v1/object/sign/$bucket_name/sign-test.txt" \ + -X POST \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY" \ + -H "Content-Type: application/json" \ + -d '{"expiresIn": 600}') + signed_path=$(echo "$sign_resp" | jq -r '.signedURL // empty' 2>/dev/null) + + if [ -n "$signed_path" ]; then + check "Create signed URL" "true" "true" + # Fetch signed URL without any auth headers (goes through Kong) + signed_content=$(curl -s "$BASE_URL/storage/v1$signed_path") + check "Fetch signed URL (no auth)" "signed url test content" "$signed_content" + else + check "Create signed URL" "true" "false" + fi + fi + + # Delete file + delete_file_status=$(http_status "$BASE_URL/storage/v1/object/$bucket_name/test-large-file.bin" \ + -X DELETE \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY") + check "Delete file" "200" "$delete_file_status" + + # Delete signed test file + http_status "$BASE_URL/storage/v1/object/$bucket_name/sign-test.txt" \ + -X DELETE \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY" >/dev/null 2>&1 + + # Delete bucket + delete_bucket_status=$(http_status "$BASE_URL/storage/v1/bucket/$bucket_name" \ + -X DELETE \ + -H "apikey: $SERVICE_ROLE_KEY" \ + -H "Authorization: Bearer $SERVICE_ROLE_KEY") + check "Delete bucket" "200" "$delete_bucket_status" +fi + +# --------------------------------------------- +# 7. Edge Functions +# --------------------------------------------- + +echo "" +echo "--- Edge Functions ---" +fn_resp=$(http_body "$BASE_URL/functions/v1/hello" \ + -X POST \ + -H "Authorization: Bearer $ANON_KEY" \ + -H "Content-Type: application/json" \ + -d '{}') +check "Call hello function" '"Hello from Edge Functions!"' "$fn_resp" + +# --------------------------------------------- +# 8. pg-meta (Studio backend) +# --------------------------------------------- + +echo "" +echo "--- pg-meta ---" +check "pg-meta with service_role key" "200" \ + "$(http_status "$BASE_URL/pg/schemas" \ + -H "apikey: $SERVICE_ROLE_KEY")" +check "pg-meta rejects anon key" "403" \ + "$(http_status "$BASE_URL/pg/schemas" \ + -H "apikey: $ANON_KEY")" +check "pg-meta rejects no key" "401" \ + "$(http_status "$BASE_URL/pg/schemas")" + +echo "" +echo "--- MCP (blocked by default) ---" +check "/api/mcp blocked" "403" \ + "$(http_status "$BASE_URL/api/mcp")" +check "/mcp blocked" "403" \ + "$(http_status "$BASE_URL/mcp")" + +# --------------------------------------------- +# 9. Realtime +# --------------------------------------------- + +echo "" +echo "--- Realtime ---" +check "Realtime health" "true" \ + "$([ "$(http_status "$BASE_URL/realtime/v1/api/tenants" \ + -H "apikey: $ANON_KEY")" != "401" ] && echo true || echo false)" + +# --------------------------------------------- +# Summary +# --------------------------------------------- + +echo "" +echo "=== Results: $pass passed, $fail failed ===" +echo "" + +if [ "$fail" -gt 0 ]; then + exit 1 +fi diff --git a/supabase/code/utils/add-new-auth-keys.sh b/supabase/code/utils/add-new-auth-keys.sh new file mode 100644 index 000000000..67a284864 --- /dev/null +++ b/supabase/code/utils/add-new-auth-keys.sh @@ -0,0 +1,223 @@ +#!/bin/sh +# +# Add asymmetric key pair and opaque API keys to a self-hosted Supabase installation. +# +# Reads JWT_SECRET from .env and generates: +# - EC P-256 key pair (JWT_KEYS, JWT_JWKS) +# - Opaque API keys (SUPABASE_PUBLISHABLE_KEY, SUPABASE_SECRET_KEY) +# - Internal: ES256 JWT API keys (ANON_KEY_ASYMMETRIC, SERVICE_ROLE_KEY_ASYMMETRIC) +# +# Usage: +# sh add-new-auth-keys.sh # Interactive: prints keys, prompts to update .env +# sh add-new-auth-keys.sh --update-env # Prints keys and writes them to .env +# sh add-new-auth-keys.sh | tee keys # Non-interactive: prints keys only +# +# Prerequisites: +# - .env file with JWT_SECRET set (run generate-keys.sh first) +# - node (>= 16) or docker +# + +set -e + +node_ok() { + command -v node >/dev/null 2>&1 || return 1 + major=$(node -v 2>/dev/null | sed 's/^v//' | cut -d. -f1) + [ -n "$major" ] && [ "$major" -ge 16 ] 2>/dev/null +} + +# Resolve how to run node: local install (>= 16) preferred, docker fallback. +if node_ok; then + node_runner="node" +else + if command -v node >/dev/null 2>&1; then + echo "Local node $(node -v) is too old (need >= 16), falling back to docker." + fi + + if ! command -v docker >/dev/null 2>&1; then + echo "Error: requires either node (>= 16) or docker." + exit 1 + fi + + if ! docker info >/dev/null 2>&1; then + echo "Error: docker is installed but the daemon is not running." + exit 1 + fi + + if ! docker image inspect node:22-alpine >/dev/null 2>&1; then + echo "Pulling node:22-alpine (first-run only)..." + docker pull node:22-alpine + fi + + node_runner="docker run --rm node:22-alpine node" +fi + +# Read JWT_SECRET from .env +if [ ! -f .env ]; then + echo "Error: .env file not found. Run generate-keys.sh first." + exit 1 +fi + +jwt_secret=$(grep '^JWT_SECRET=' .env | cut -d= -f2- | tr -d '\r') +if [ -z "$jwt_secret" ]; then + echo "Error: JWT_SECRET not found in .env. Run generate-keys.sh first." + exit 1 +fi + +tmpdir=$(mktemp -d) +trap 'rm -rf "$tmpdir"' EXIT + +# Node.js does the crypto-heavy work: +# - EC P-256 keypair generation +# - JWKS construction (with symmetric key included) +# - ES256 JWT signing +# - Opaque API key generation with checksum +$node_runner -e ' +const crypto = require("crypto"); + +const jwtSecret = process.argv[1]; + +// Generate EC P-256 keypair and export as JWK +const { privateKey } = crypto.generateKeyPairSync("ec", { namedCurve: "P-256" }); +const jwkPrivate = privateKey.export({ format: "jwk" }); + +const kid = crypto.randomUUID(); + +// Symmetric key as JWK (base64url-encoded) +const octKey = { + kty: "oct", + k: Buffer.from(jwtSecret).toString("base64url"), + alg: "HS256" +}; + +// JWKS with private key (for Auth to sign tokens) +const jwksKeypair = { keys: [ + { kty: "EC", kid, use: "sig", key_ops: ["sign", "verify"], alg: "ES256", ext: true, + crv: jwkPrivate.crv, x: jwkPrivate.x, y: jwkPrivate.y, d: jwkPrivate.d }, + octKey +]}; + +// JWKS with public key only (for PostgREST, Realtime, Storage to verify) +const jwksPublic = { keys: [ + { kty: "EC", kid, use: "sig", key_ops: ["verify"], alg: "ES256", ext: true, + crv: jwkPrivate.crv, x: jwkPrivate.x, y: jwkPrivate.y }, + octKey +]}; + +// Sign ES256 JWT +function signES256(payload) { + const header = { alg: "ES256", typ: "JWT", kid }; + const b64Header = Buffer.from(JSON.stringify(header)).toString("base64url"); + const b64Payload = Buffer.from(JSON.stringify(payload)).toString("base64url"); + const data = b64Header + "." + b64Payload; + const sig = crypto.sign("SHA256", Buffer.from(data), { + key: privateKey, + dsaEncoding: "ieee-p1363" + }).toString("base64url"); + return data + "." + sig; +} + +const iat = Math.floor(Date.now() / 1000); +const exp = iat + 5 * 365 * 24 * 3600; // 5 years + +const anonJwt = signES256({ role: "anon", iss: "supabase", iat, exp }); +const serviceJwt = signES256({ role: "service_role", iss: "supabase", iat, exp }); + +// Generate opaque API keys with checksum +const PROJECT_REF = "supabase-self-hosted"; + +function generateOpaqueKey(prefix) { + const random = crypto.randomBytes(17).toString("base64url").slice(0, 22); + const intermediate = prefix + random; + const checksum = crypto.createHash("sha256") + .update(PROJECT_REF + "|" + intermediate) + .digest("base64url") + .slice(0, 8); + return intermediate + "_" + checksum; +} + +const publishableKey = generateOpaqueKey("sb_publishable_"); +const secretKey = generateOpaqueKey("sb_secret_"); + +// Output as KEY=value lines for shell to parse +console.log("SUPABASE_PUBLISHABLE_KEY=" + publishableKey); +console.log("SUPABASE_SECRET_KEY=" + secretKey); +console.log("ANON_KEY_ASYMMETRIC=" + anonJwt); +console.log("SERVICE_ROLE_KEY_ASYMMETRIC=" + serviceJwt); +console.log("JWT_KEYS=" + JSON.stringify(jwksKeypair.keys)); +console.log("JWT_JWKS=" + JSON.stringify(jwksPublic)); +' "$jwt_secret" > "$tmpdir/output" + +# Read generated values +SUPABASE_PUBLISHABLE_KEY=$(grep '^SUPABASE_PUBLISHABLE_KEY=' "$tmpdir/output" | cut -d= -f2-) +SUPABASE_SECRET_KEY=$(grep '^SUPABASE_SECRET_KEY=' "$tmpdir/output" | cut -d= -f2-) +ANON_KEY_ASYMMETRIC=$(grep '^ANON_KEY_ASYMMETRIC=' "$tmpdir/output" | cut -d= -f2-) +SERVICE_ROLE_KEY_ASYMMETRIC=$(grep '^SERVICE_ROLE_KEY_ASYMMETRIC=' "$tmpdir/output" | cut -d= -f2-) +JWT_KEYS=$(grep '^JWT_KEYS=' "$tmpdir/output" | cut -d= -f2-) +JWT_JWKS=$(grep '^JWT_JWKS=' "$tmpdir/output" | cut -d= -f2-) + +echo "" +echo "SUPABASE_PUBLISHABLE_KEY=${SUPABASE_PUBLISHABLE_KEY}" +echo "SUPABASE_SECRET_KEY=${SUPABASE_SECRET_KEY}" +echo "" +echo "JWT_KEYS=${JWT_KEYS}" +echo "" +echo "JWT_JWKS=${JWT_JWKS}" +echo "" +echo "To enable asymmetric key pair, the following should be enabled in docker-compose.yml:" +echo "" +echo " Auth: GOTRUE_JWT_KEYS: \${JWT_KEYS:-[]}" +echo " Realtime: API_JWT_JWKS: \${JWT_JWKS:-{\"keys\":[]}}" +echo " Storage: JWT_JWKS: \${JWT_JWKS:-{\"keys\":[]}}" +echo "" + +if [ "$1" = "--update-env" ]; then + update_env=true +elif test -t 0; then + printf "Update .env file? (y/N) " + read -r REPLY + case "$REPLY" in + [Yy]) update_env=true ;; + *) update_env=false ;; + esac +else + echo "Running non-interactively. Pass --update-env to write to .env." + update_env=false +fi + +if [ "$update_env" != "true" ]; then + exit 0 +fi + +echo "Updating .env..." + +# Append new variables if they don't exist, or update them if they do +for var in SUPABASE_PUBLISHABLE_KEY SUPABASE_SECRET_KEY ANON_KEY_ASYMMETRIC SERVICE_ROLE_KEY_ASYMMETRIC JWT_KEYS JWT_JWKS; do + eval "val=\$$var" + if grep -q "^${var}=" .env; then + sed -i.old -e "s|^${var}=.*$|${var}=${val}|" .env + else + echo "${var}=${val}" >> .env + fi +done + +# Uncomment new auth configuration in docker-compose.yml +echo "Updating docker-compose.yml..." +if [ ! -f docker-compose.yml ]; then + echo "Error: docker-compose.yml not found in $(pwd)" + exit 1 +fi + +# Always fall through to the grep check +sed -i.old \ + -e '/^[ ]*#GOTRUE_JWT_KEYS:/ s/#//' \ + -e '/^[ ]*#API_JWT_JWKS:/ s/#//' \ + -e '/^[ ]*#JWT_JWKS:/ s/#//' \ + docker-compose.yml || true + +if grep -q '^[ ]*GOTRUE_JWT_KEYS:' docker-compose.yml && \ + grep -q '^[ ]*API_JWT_JWKS:' docker-compose.yml && \ + grep -q '^[ ]*JWT_JWKS:' docker-compose.yml; then + echo "Done." +else + echo "Warning: could not edit docker-compose.yml. Uncomment auth configuration manually." +fi diff --git a/supabase/code/utils/generate-keys.sh b/supabase/code/utils/generate-keys.sh index d9a16395b..23b0b1a4d 100644 --- a/supabase/code/utils/generate-keys.sh +++ b/supabase/code/utils/generate-keys.sh @@ -1,5 +1,15 @@ #!/bin/sh # +# Generate secrets and legacy symmetric JWT API keys for self-hosted Supabase. +# +# Generates: JWT_SECRET, ANON_KEY, SERVICE_ROLE_KEY, and other secrets +# needed for a fresh installation. +# +# Usage: +# sh generate-keys.sh # Interactive: prints keys, prompts to update .env +# sh generate-keys.sh --update-env # Prints keys and writes them to .env +# sh generate-keys.sh | tee keys # Non-interactive: prints keys only +# # Portions of this code are derived from Inder Singh's setup.sh shell script. # Copyright 2025 Inder Singh. Licensed under Apache License 2.0. # Original source: https://github.com/singh-inder/supabase-automated-self-host/blob/main/setup.sh @@ -35,7 +45,7 @@ fi jwt_secret="$(gen_base64 30)" -# Used in get_token() +# Used in gen_token() header='{"alg":"HS256","typ":"JWT"}' iat=$(date +%s) exp=$((iat + 5 * 3600 * 24 * 365)) # 5 years @@ -87,21 +97,23 @@ echo "POSTGRES_PASSWORD=${postgres_password}" echo "DASHBOARD_PASSWORD=${dashboard_password}" echo "" -if ! test -t 0; then - echo "Running non-interactively. Skipping .env update." - exit 0 +if [ "$1" = "--update-env" ]; then + update_env=true +elif test -t 0; then + printf "Update .env file? (y/N) " + read -r REPLY + case "$REPLY" in + [Yy]) update_env=true ;; + *) update_env=false ;; + esac +else + echo "Running non-interactively. Pass --update-env to write to .env." + update_env=false fi -printf "Update .env file? (y/N) " -read -r REPLY -case "$REPLY" in - [Yy]) - ;; - *) - echo "Not updating .env" - exit 0 - ;; -esac +if [ "$update_env" != "true" ]; then + exit 0 +fi echo "Updating .env..." diff --git a/supabase/code/utils/reassign-owner.sh b/supabase/code/utils/reassign-owner.sh new file mode 100644 index 000000000..d13b17073 --- /dev/null +++ b/supabase/code/utils/reassign-owner.sh @@ -0,0 +1,153 @@ +#!/bin/sh +# +# Reassign ownership of public schema objects from supabase_admin to postgres. +# +# Context and documentation: +# https://supabase.com/docs/guides/self-hosting/remove-superuser-access +# +# Credits: +# Original version by Inder Singh. +# +# Usage: +# sh utils/reassign-owner.sh +# + +set -e + +if ! docker compose version >/dev/null 2>&1; then + echo "Docker Compose not found." + exit 1 +fi + +# Check Postgres service +db_image_prefix="supabase.postgres:" + +compose_output=$(docker compose ps \ + --format '{{.Image}}\t{{.Service}}\t{{.Status}}' 2>/dev/null | + grep -m1 "^$db_image_prefix" || true) + +if [ -z "$compose_output" ]; then + echo "Postgres container not found. Exiting." + exit 1 +fi + +db_srv_name=$(echo "$compose_output" | cut -f2) +db_srv_status=$(echo "$compose_output" | cut -f3) + +case "$db_srv_status" in + Up*) + ;; + *) + echo "Postgres container status: $db_srv_status" + echo "Exiting." + exit 1 + ;; +esac + +if ! test -t 0; then + echo "" + echo "Running non-interactively. Not reassigning ownership." + exit 0 +fi + +printf "Reassign public schema objects to postgres user? (y/N) " +read -r REPLY +case "$REPLY" in + [Yy]) + ;; + *) + echo "Canceled. Not reassigning ownership." + exit 0 + ;; +esac + +docker compose exec -T "$db_srv_name" psql -v ON_ERROR_STOP=1 -U supabase_admin -d postgres <<'EOF' +\echo 'Current supabase_admin-owned objects in public schema:' +SELECT c.relname, c.relkind, c.relowner::regrole +FROM pg_class c +WHERE c.relnamespace = 'public'::regnamespace +AND c.relowner = 'supabase_admin'::regrole; + +-- Reassign user objects in public schema from supabase_admin to postgres. +-- (Only affects public schema; Supabase-managed schemas stay as-is. +-- Extension-owned objects are skipped.) +DO $$ +DECLARE + rec record; + rel_count int := 0; + fn_count int := 0; + type_count int := 0; +BEGIN + -- Tables, views, sequences, materialized views, partitioned tables + FOR rec IN + SELECT c.relname, c.relkind + FROM pg_class c + WHERE c.relnamespace = 'public'::regnamespace + AND c.relowner = 'supabase_admin'::regrole + AND c.relkind IN ('r', 'v', 'S', 'm', 'p') + AND NOT EXISTS ( + SELECT 1 FROM pg_depend d + WHERE d.classid = 'pg_class'::regclass + AND d.objid = c.oid + AND d.deptype = 'e' + ) + ORDER BY CASE c.relkind + WHEN 'p' THEN 0 -- partitioned parents first; cascades ownership to partitions + WHEN 'm' THEN 1 + WHEN 'r' THEN 2 + WHEN 'v' THEN 3 + WHEN 'S' THEN 4 + END + LOOP + EXECUTE format('ALTER TABLE public.%I OWNER TO postgres', rec.relname); + rel_count := rel_count + 1; + END LOOP; + + -- Functions and procedures + FOR rec IN + SELECT p.oid, p.proname, pg_get_function_identity_arguments(p.oid) AS args + FROM pg_proc p + WHERE p.pronamespace = 'public'::regnamespace + AND p.proowner = 'supabase_admin'::regrole + AND NOT EXISTS ( + SELECT 1 FROM pg_depend d + WHERE d.classid = 'pg_proc'::regclass + AND d.objid = p.oid + AND d.deptype = 'e' + ) + LOOP + EXECUTE format('ALTER ROUTINE public.%I(%s) OWNER TO postgres', rec.proname, rec.args); + fn_count := fn_count + 1; + END LOOP; + + -- Types (excluding array types and table-bound composites) + FOR rec IN + SELECT t.typname + FROM pg_type t + WHERE t.typnamespace = 'public'::regnamespace + AND t.typowner = 'supabase_admin'::regrole + AND t.typrelid = 0 + AND NOT EXISTS ( + SELECT 1 FROM pg_type el + WHERE el.oid = t.typelem + AND el.typarray = t.oid + ) + AND NOT EXISTS ( + SELECT 1 FROM pg_depend d + WHERE d.classid = 'pg_type'::regclass + AND d.objid = t.oid + AND d.deptype = 'e' + ) + LOOP + EXECUTE format('ALTER TYPE public.%I OWNER TO postgres', rec.typname); + type_count := type_count + 1; + END LOOP; + + RAISE NOTICE 'Reassigned % relation(s), % routine(s), % type(s) from supabase_admin to postgres.', + rel_count, fn_count, type_count; +END +$$; +EOF + +echo "" +echo "Done." diff --git a/supabase/code/utils/rotate-new-api-keys.sh b/supabase/code/utils/rotate-new-api-keys.sh new file mode 100644 index 000000000..48fd9f02d --- /dev/null +++ b/supabase/code/utils/rotate-new-api-keys.sh @@ -0,0 +1,117 @@ +#!/bin/sh +# +# Rotate opaque API keys for a self-hosted Supabase installation. +# +# Regenerates SUPABASE_PUBLISHABLE_KEY and SUPABASE_SECRET_KEY +# without touching the asymmetric key pair (JWKS) or JWT tokens. +# +# Usage: +# sh rotate-new-api-keys.sh # Interactive: prints keys, prompts to update .env +# sh rotate-new-api-keys.sh --update-env # Prints keys and writes them to .env +# sh rotate-new-api-keys.sh | tee keys # Non-interactive: prints keys only +# +# Prerequisites: +# - .env file (run generate-keys.sh and add-new-auth-keys.sh first) +# - node (>= 16) or docker +# + +set -e + +node_ok() { + command -v node >/dev/null 2>&1 || return 1 + major=$(node -v 2>/dev/null | sed 's/^v//' | cut -d. -f1) + [ -n "$major" ] && [ "$major" -ge 16 ] 2>/dev/null +} + +# Resolve how to run node: local install (>= 16) preferred, docker fallback. +if node_ok; then + node_runner="node" +else + if command -v node >/dev/null 2>&1; then + echo "Local node $(node -v) is too old (need >= 16), falling back to docker." + fi + + if ! command -v docker >/dev/null 2>&1; then + echo "Error: requires either node (>= 16) or docker." + exit 1 + fi + + if ! docker info >/dev/null 2>&1; then + echo "Error: docker is installed but the daemon is not running." + exit 1 + fi + + if ! docker image inspect node:22-alpine >/dev/null 2>&1; then + echo "Pulling node:22-alpine (first-run only)..." + docker pull node:22-alpine + fi + + node_runner="docker run --rm node:22-alpine node" +fi + +if [ ! -f .env ]; then + echo "Error: .env file not found. Run generate-keys.sh first." + exit 1 +fi + +tmpdir=$(mktemp -d) +trap 'rm -rf "$tmpdir"' EXIT + +$node_runner -e ' +const crypto = require("crypto"); + +const PROJECT_REF = "supabase-self-hosted"; + +function generateOpaqueKey(prefix) { + const random = crypto.randomBytes(17).toString("base64url").slice(0, 22); + const intermediate = prefix + random; + const checksum = crypto.createHash("sha256") + .update(PROJECT_REF + "|" + intermediate) + .digest("base64url") + .slice(0, 8); + return intermediate + "_" + checksum; +} + +const publishableKey = generateOpaqueKey("sb_publishable_"); +const secretKey = generateOpaqueKey("sb_secret_"); + +console.log("SUPABASE_PUBLISHABLE_KEY=" + publishableKey); +console.log("SUPABASE_SECRET_KEY=" + secretKey); +' > "$tmpdir/output" + +SUPABASE_PUBLISHABLE_KEY=$(grep '^SUPABASE_PUBLISHABLE_KEY=' "$tmpdir/output" | cut -d= -f2-) +SUPABASE_SECRET_KEY=$(grep '^SUPABASE_SECRET_KEY=' "$tmpdir/output" | cut -d= -f2-) + +echo "" +echo "SUPABASE_PUBLISHABLE_KEY=${SUPABASE_PUBLISHABLE_KEY}" +echo "SUPABASE_SECRET_KEY=${SUPABASE_SECRET_KEY}" +echo "" + +if [ "$1" = "--update-env" ]; then + update_env=true +elif test -t 0; then + printf "Update .env file? (y/N) " + read -r REPLY + case "$REPLY" in + [Yy]) update_env=true ;; + *) update_env=false ;; + esac +else + echo "Running non-interactively. Pass --update-env to write to .env." + update_env=false +fi + +if [ "$update_env" != "true" ]; then + exit 0 +fi + +echo "Updating .env..." + +for var in SUPABASE_PUBLISHABLE_KEY SUPABASE_SECRET_KEY; do + eval "val=\$$var" + if grep -q "^${var}=" .env; then + sed -i.old -e "s|^${var}=.*$|${var}=${val}|" .env + else + echo "${var}=${val}" >> .env + fi +done diff --git a/supabase/code/utils/upgrade-pg17.sh b/supabase/code/utils/upgrade-pg17.sh new file mode 100644 index 000000000..6bdc97bd3 --- /dev/null +++ b/supabase/code/utils/upgrade-pg17.sh @@ -0,0 +1,754 @@ +#!/usr/bin/env bash +# +# Requires bash (not sh) for pipefail, which ensures failures in piped +# commands are caught during the upgrade. +# +# Upgrade self-hosted Supabase Postgres from 15 to 17. +# +# Uses Supabase's pg_upgrade scripts (initiate.sh + complete.sh) inside a +# temporary PG15 container, then swaps data directories and starts Postgres 17. +# +# Usage (must be run as root or with sudo): +# cd docker/ +# sudo bash utils/upgrade-pg17.sh # Interactive (prompts for confirmation) +# sudo bash utils/upgrade-pg17.sh --yes # Non-interactive (skip all prompts) +# +# Requirements: +# - Docker with Docker Compose (docker compose, not docker-compose) +# - Running Supabase self-hosted setup with Postgres 15 +# - At least 2x current database size + 5 GB free disk space +# +# Backup: +# The original Postgres 15 data directory is preserved as +# ./volumes/db/data.bak.pg15 during the upgrade. +# DO NOT DELETE it until you have verified the upgrade was successful. +# +# Rollback (if the upgrade fails or you want to revert): +# 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down +# 2. rm -rf ./volumes/db/data +# 3. mv ./volumes/db/data.bak.pg15 ./volumes/db/data +# 4. docker compose run --rm db chown -R postgres:postgres /etc/postgresql-custom/ +# 5. docker compose up -d +# + +# Ensure we're running under bash (not sh/zsh/dash). +# Check that $BASH ends with /bash (not /sh, /zsh, etc.). +case "${BASH:-}" in + */bash) ;; + *) echo "Error: This script requires bash. Run it with: sudo bash $0" >&2; exit 1 ;; +esac + +set -euo pipefail + +AUTO_CONFIRM=false +for arg in "$@"; do + case "$arg" in + --yes|-y) AUTO_CONFIRM=true ;; + esac +done + +# --- Configuration ---------------------------------------------------------- + +# Image used for the upgrade tarball + complete.sh container. +# Must share glibc with PG15 (the extracted ELF binaries run inside PG15). +PG17_UPGRADE_IMAGE="supabase/postgres:17.6.1.063" +# Tag in supabase/postgres repo matching the upgrade image (for downloading scripts) +PG17_SCRIPTS_REF="17.6.1.063" +DB_CONTAINER="supabase-db" +UPGRADE_CONTAINER="supabase-pg-upgrade" +COMPLETE_CONTAINER="supabase-pg-complete" + +DATA_DIR="./volumes/db/data" +BACKUP_DIR="./volumes/db/data.bak.pg15" +# Include image tag in cache filename so changing PG17_UPGRADE_IMAGE invalidates it +PG17_TAG="${PG17_UPGRADE_IMAGE##*:}" +TARBALL_CACHE="./volumes/db/pg17_upgrade_bin_${PG17_TAG}.tar.gz" +# initiate.sh writes pg_upgrade output here: pgdata/, conf/, sql/ +MIGRATION_DIR="./volumes/db/data_migration" + +# --- Helpers ---------------------------------------------------------------- + +die() { printf 'Error: %s\n' "$*" >&2; exit 1; } +info() { printf '\n==> %s\n' "$*"; } +warn() { printf 'Warning: %s\n' "$*" >&2; } + +# Temp dir on host for tarball + scripts (mounted into containers) +staging_dir="" +pg_password="" +current_image="" +drop_extensions="" +db_config_vol="" + +# Remove leftover containers and staging dir on exit. +# Uses an alpine container for rm because the tarball build runs as root +# inside Docker - the resulting files are root-owned and can't be deleted +# by the host user on macOS. +cleanup() { + docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true + docker rm -f "$COMPLETE_CONTAINER" >/dev/null 2>&1 || true + if [ -n "$staging_dir" ] && [ -d "$staging_dir" ]; then + docker run --rm -v "$staging_dir:/cleanup" alpine rm -rf /cleanup 2>/dev/null || true + rm -rf "$staging_dir" 2>/dev/null || true + fi +} +trap cleanup EXIT + +on_interrupt() { + echo "" + warn "Interrupted. Cleaning up..." + # If db-config was chowned to PG17, restore for PG15 rollback + if [ -n "$db_config_vol" ] && [ -n "$current_image" ]; then + docker run --rm -v "${db_config_vol}:/vol" "$current_image" \ + chown -R postgres:postgres /vol/ 2>/dev/null || true + fi + die "Interrupted." +} +trap on_interrupt INT + +confirm() { + if [ "$AUTO_CONFIRM" = true ]; then return 0; fi + if ! test -t 0; then + die "This script must be run interactively, or use --yes to skip prompts." + fi + printf '%s (y/N) ' "$1" + read -r reply + case "$reply" in + [Yy]*) return 0 ;; + *) echo "Aborted."; exit 0 ;; + esac +} + +run_sql_on() { + local container=$1; shift + docker exec -i \ + -e PGPASSWORD="$pg_password" \ + "$container" \ + psql -h localhost -U supabase_admin -d postgres -v ON_ERROR_STOP=1 "$@" +} + +wait_for_healthy() { + local container=$1 retries=30 + while [ $retries -gt 0 ]; do + if docker exec "$container" pg_isready -U postgres -h localhost >/dev/null 2>&1; then + return 0 + fi + retries=$((retries - 1)) + sleep 1 + done + die "Postgres in '$container' did not become ready in 30 seconds." +} + +# --- Pre-flight checks ----------------------------------------------------- + +preflight() { + info "Running pre-flight checks" + + if [ "$(id -u)" -ne 0 ]; then + die "This script must be run as root (e.g. sudo bash $0)." + fi + + docker compose version >/dev/null 2>&1 || die "Docker Compose not found." + command -v curl >/dev/null 2>&1 || die "curl is required (for downloading upgrade scripts)." + [ -f docker-compose.yml ] || die "Run this script from the docker/ directory." + [ -f docker-compose.pg17.yml ] || die "Missing docker-compose.pg17.yml." + [ -f .env ] || die "Missing .env file." + + # Resolve db-config volume (exact match on _db-config suffix or bare db-config) + db_config_vol=$(docker volume ls --filter "name=db-config" --format '{{.Name}}' \ + | grep -E '^db-config$|_db-config$' | head -n 1) + [ -n "$db_config_vol" ] || die "Could not find db-config volume. Is Supabase running?" + + # Read the target PG17 image from the compose override (what the user will run) + PG17_TARGET_IMAGE=$(grep 'image:.*postgres' docker-compose.pg17.yml | awk '{print $2}' | head -n 1) + [ -n "$PG17_TARGET_IMAGE" ] || die "Could not read image from docker-compose.pg17.yml." + + pg_password=$(grep '^POSTGRES_PASSWORD=' .env | cut -d '=' -f 2- | sed "s/^['\"]//;s/['\"]$//" | head -n 1) + [ -n "$pg_password" ] || die "POSTGRES_PASSWORD not set in .env." + + docker inspect "$DB_CONTAINER" >/dev/null 2>&1 \ + || die "Container '$DB_CONTAINER' not found. Is Supabase running?" + + current_image=$(docker inspect "$DB_CONTAINER" --format '{{.Config.Image}}') + case "$current_image" in + supabase/postgres:15.*|supabase.postgres:15.*) ;; + supabase/postgres:17.*|supabase.postgres:17.*) die "Already running Postgres 17 ($current_image)." ;; + *) die "Unexpected database image: $current_image" ;; + esac + + local status + status=$(docker inspect "$DB_CONTAINER" --format '{{.State.Status}}') + [ "$status" = "running" ] || die "'$DB_CONTAINER' is not running (status: $status)." + [ -d "$DATA_DIR" ] || die "Data directory not found: $DATA_DIR" + + if [ -d "$BACKUP_DIR" ]; then + warn "Backup directory already exists: $BACKUP_DIR" + warn "This is likely from a previous upgrade attempt." + warn "If you haven't verified that previous upgrade, roll back first:" + warn " 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down" + warn " 2. rm -rf $DATA_DIR" + warn " 3. mv $BACKUP_DIR $DATA_DIR" + warn " 4. docker compose run --rm db chown -R postgres:postgres /etc/postgresql-custom/" + warn " 5. docker compose up -d" + echo "" + warn "Continuing will DELETE the existing backup permanently." + confirm "Delete $BACKUP_DIR and start a fresh upgrade?" + rm -rf "$BACKUP_DIR" + fi + if [ -d "$MIGRATION_DIR" ]; then + rm -rf "$MIGRATION_DIR" + fi + + # Disk space + local data_size_kb data_size_mb avail_kb avail_mb needed_mb + data_size_kb=$(du -sk "$DATA_DIR" 2>/dev/null | cut -f1) + [ -n "$data_size_kb" ] || die "Could not calculate data size for $DATA_DIR" + data_size_mb=$((data_size_kb / 1024)) + avail_kb=$(df -k "$(dirname "$DATA_DIR")" | awk 'NR==2 { print $4 }') + [ -n "$avail_kb" ] || die "Could not calculate available disk space for $(dirname "$DATA_DIR")" + avail_mb=$((avail_kb / 1024)) + needed_mb=$((data_size_mb * 2 + 5000)) + echo " Data size: ${data_size_mb} MB" + echo " Available space: ${avail_mb} MB" + echo " Estimated need: ${needed_mb} MB" + if [ "$avail_mb" -lt "$needed_mb" ]; then + warn "Disk space may be insufficient." + warn "pg_upgrade copies data; need ~2x data size + ~5 GB for the upgrade tarball." + confirm "Continue anyway?" + fi + + # Incompatible extensions + info "Checking for incompatible extensions" + local incompatible + incompatible=$(run_sql_on "$DB_CONTAINER" -A -t -c " + SELECT string_agg(extname, ', ') + FROM pg_extension + WHERE extname IN ('timescaledb', 'plv8', 'plcoffee', 'plls'); + " 2>/dev/null | tr -d '[:space:]') || true + + if [ -n "$incompatible" ]; then + warn "Incompatible extensions found: $incompatible" + warn "These do not exist in Postgres 17 and must be dropped before upgrading." + warn "If you proceed, they will be dropped automatically." + warn "The original data is preserved as a backup so you can roll back." + confirm "Drop these extensions and continue with the upgrade?" + drop_extensions="$incompatible" + fi + + echo "" + echo "This script will:" + echo " 1. Pull the Postgres 17 image" + echo " 2. Build an upgrade tarball from the image (~1.2 GB compressed, temporary)" + echo " 3. Stop all Supabase services" + echo " 4. Run pg_upgrade (Postgres 15 -> 17)" + echo " 5. Apply post-upgrade patches" + echo " 6. Start Supabase with Postgres 17" + echo " 7. Apply additional migrations" + echo "" + echo " Current image: $current_image" + echo " Target image: $PG17_TARGET_IMAGE" + echo " Upgrade image: $PG17_UPGRADE_IMAGE" + echo " Data directory: $DATA_DIR" + echo " Backup location: $BACKUP_DIR" + echo "" + confirm "Proceed with the upgrade?" +} + +# --- Step 1: Pull Postgres 17 image ---------------------------------------- + +pull_image() { + info "Pulling Postgres 17 images" + docker pull "$PG17_UPGRADE_IMAGE" + if [ "$PG17_TARGET_IMAGE" != "$PG17_UPGRADE_IMAGE" ]; then + docker pull "$PG17_TARGET_IMAGE" + fi +} + +# --- Step 2: Build upgrade tarball ----------------------------------------- +# +# Extracts PG17 binaries, libraries, share data, and upgrade scripts from +# the PG17 Docker image into a tarball that initiate.sh can consume. +# +# The tarball uses the "non-nix" layout (17/bin, 17/lib, 17/share - no +# nix_flake_version file), so initiate.sh sets LD_LIBRARY_PATH to find +# the bundled libraries. + +build_tarball() { + local tmpbase="${TMPDIR:-/tmp}" + staging_dir=$(mktemp -d "${tmpbase%/}/supabase-pg17-upgrade.XXXXXX") + # World-writable so Docker containers can write to bind mounts on macOS, + # where the VM's root user has no special access to host directories. + chmod 777 "$staging_dir" + echo " Staging directory: $staging_dir" + + # Download upgrade scripts from the supabase/postgres repo (pinned to PG17_SCRIPTS_REF). + # These are no longer bundled in the latest PG17 Docker images. + info "Downloading upgrade scripts (ref: $PG17_SCRIPTS_REF)" + local scripts_base="https://raw.githubusercontent.com/supabase/postgres/${PG17_SCRIPTS_REF}/ansible/files/admin_api_scripts/pg_upgrade_scripts" + mkdir -p "$staging_dir/scripts" + for script in initiate.sh complete.sh common.sh pgsodium_getkey.sh check.sh prepare.sh; do + curl -fsSL "$scripts_base/$script" -o "$staging_dir/scripts/$script" \ + || die "Failed to download $script from GitHub" + done + + if [ -f "$TARBALL_CACHE" ]; then + info "Using cached upgrade tarball: $TARBALL_CACHE" + cp "$TARBALL_CACHE" "$staging_dir/pg_upgrade_bin.tar.gz" + return + fi + + info "Building upgrade tarball from Postgres 17 image (first run)" + docker run --rm --user root --entrypoint bash \ + -v "$staging_dir:/export" \ + "$PG17_UPGRADE_IMAGE" \ + -c ' + set -euo pipefail + mkdir -p /export/17/bin /export/17/lib /export/17/share + + echo " Copying binaries..." + # Binaries in the nix profile are either ELF binaries or shell + # wrappers that exec a .xxx-wrapped ELF from the nix store. + # Extract the actual ELF binaries so they work outside nix. + BIN_DIR=$(dirname $(readlink -f /usr/lib/postgresql/bin/postgres)) + for f in "$BIN_DIR"/*; do + name=$(basename "$f") + + # Skip nix wrapper-internal files + case "$name" in .*-wrapped) continue ;; esac + + # Check for ELF + if [ -x "$f" ] && file -b "$f" | grep -q "ELF .* executable"; then + cp "$f" /export/17/bin/"$name" + else + # Shell wrapper - extract the real .xxx-wrapped ELF path + wrapped=$(grep -o "/nix/store/[^ \"]*-wrapped" "$f" 2>/dev/null | head -n 1 || true) + if [ -n "$wrapped" ] && [ -f "$wrapped" ]; then + cp "$wrapped" /export/17/bin/"$name" + else + cp "$f" /export/17/bin/"$name" + fi + fi + done + + echo " Copying libraries..." + PKGLIBDIR=$(pg_config --pkglibdir) + LIBDIR=$(pg_config --libdir) + + # These paths may overlap (PKGLIBDIR and LIBDIR often point to the + # same nix store path). Use cp -Lf to handle overwrites from + # read-only nix store source files. + cp -Lf "$PKGLIBDIR"/*.so /export/17/lib/ || echo " Warning: cp from $PKGLIBDIR failed" >&2 + cp -Lf "$LIBDIR"/*.so* /export/17/lib/ || echo " Warning: cp from $LIBDIR failed" >&2 + cp -Lf /nix/var/nix/profiles/default/lib/*.so* /export/17/lib/ || echo " Warning: cp from nix profile lib failed" >&2 + + echo " Copying share data..." + + # Nix-built binaries resolve share dir relative to their location: + # /../share/postgresql/ + # so we need share/postgresql/ not just share/ + mkdir -p /export/17/share/postgresql + + # Remove cyclic symlink (timezonesets/timezonesets -> timezonesets). + rm -f /usr/share/postgresql/timezonesets/timezonesets 2>/dev/null || true + + # Pre-create subdirectories so cp -rL does not need to mkdir them. + # On macOS with Docker Desktop bind mount rejects mkdir with the nix store + # read-only (dr-xr-xr-x) permissions; pre-creating with default + # writable permissions fixed this + mkdir -p /export/17/share/postgresql/{extension,timezonesets,tsearch_data} + mkdir -p /export/17/share/postgresql/extension/{functions,procedures,tables,types} + + cp -rL /usr/share/postgresql/* /export/17/share/postgresql/ || echo " Warning: cp share data had errors" >&2 + + # initiate.sh copies .control/.sql from PGLIBNEW to PGSHARENEW/extension/ + echo " Copying extension definitions to lib..." + SHAREDIR=$(pg_config --sharedir) + cp "$SHAREDIR"/extension/*.control /export/17/lib/ || echo " Warning: cp .control from $SHAREDIR/extension failed" >&2 + cp "$SHAREDIR"/extension/*.sql /export/17/lib/ || echo " Warning: cp .sql from $SHAREDIR/extension failed" >&2 + + # Verify critical files before creating tarball + echo " Checking for key files..." + [ -f /export/17/bin/postgres ] || { echo "Error: bin/postgres missing"; exit 1; } + [ -f /export/17/share/postgresql/timezonesets/Default ] || { echo "Error: timezonesets/Default missing"; exit 1; } + ls /export/17/share/postgresql/extension/*.control >/dev/null 2>&1 || { echo "Error: no .control files in extension/"; exit 1; } + ls /export/17/lib/*.so >/dev/null 2>&1 || { echo "Error: no .so files in lib/"; exit 1; } + + echo " Creating tarball (this may take several minutes)..." + cd /export && tar czf pg_upgrade_bin.tar.gz 17/ + + echo " Tarball: $(du -sh /export/pg_upgrade_bin.tar.gz | cut -f1)" + ' + + # Cache for next run + cp "$staging_dir/pg_upgrade_bin.tar.gz" "$TARBALL_CACHE" + info "Tarball cached at $TARBALL_CACHE" +} + +# --- Step 3: Drop incompatible extensions ---------------------------------- + +drop_incompatible_extensions() { + if [ -z "$drop_extensions" ]; then + return + fi + info "Dropping incompatible extensions" + + local ext + echo "$drop_extensions" | tr ',' '\n' | while read -r ext; do + ext=$(echo "$ext" | tr -d '[:space:]') + [ -z "$ext" ] && continue + echo " DROP EXTENSION $ext CASCADE" + run_sql_on "$DB_CONTAINER" -c "DROP EXTENSION IF EXISTS \"$ext\" CASCADE;" + done +} + +# --- Step 4: Stop services and back up ------------------------------------- + +stop_and_backup() { + info "Backing up pgsodium root key" + local key_backup="./volumes/db/pgsodium_root.key.bak.pg15" + docker run --rm -v "${db_config_vol}:/src:ro" -v "$(pwd)/volumes/db:/dst" \ + alpine cp /src/pgsodium_root.key /dst/pgsodium_root.key.bak.pg15 \ + || die "Failed to back up pgsodium root key from db-config volume." + echo " Saved to: $key_backup" + + info "Stopping all Supabase services" + docker compose down + + echo " Original data will be preserved as: $BACKUP_DIR" +} + +# --- Step 5: Run pg_upgrade via initiate.sh + complete.sh ------------------ +# +# Host directories are mounted at non-standard paths (/mnt/host-*) with +# symlinks at the paths the upgrade scripts expect. This lets complete.sh's +# CI wrapper (which does rm/mv/ln on /var/lib/postgresql/data and +# /data_migration) operate on symlinks rather than bind mounts. + +run_upgrade() { + local abs_data_dir abs_migration_dir + + mkdir -p "$MIGRATION_DIR" + # World-writable for macOS Docker bind mount compatibility (see build_tarball) + chmod 777 "$MIGRATION_DIR" + abs_data_dir=$(cd "$DATA_DIR" && pwd) + abs_migration_dir=$(cd "$MIGRATION_DIR" && pwd) + + info "Starting upgrade container" + docker run -d --name "$UPGRADE_CONTAINER" \ + --entrypoint sleep \ + -v "${abs_data_dir}:/mnt/host-pgdata" \ + -v "${abs_migration_dir}:/mnt/host-migration" \ + -v "${db_config_vol}:/etc/postgresql-custom" \ + -v "${staging_dir}:/tmp/staging:ro" \ + -e PGPASSWORD="$pg_password" \ + "$current_image" \ + infinity + + info "Preparing upgrade environment" + docker exec "$UPGRADE_CONTAINER" bash -c ' + # Symlink bind mounts to the paths the upgrade scripts expect + rm -rf /var/lib/postgresql/data + ln -s /mnt/host-pgdata /var/lib/postgresql/data + ln -s /mnt/host-migration /data_migration + + mkdir -p /tmp/persistent /tmp/upgrade /tmp/pg_upgrade + cp /tmp/staging/pg_upgrade_bin.tar.gz /tmp/persistent/ + cp /tmp/staging/scripts/*.sh /tmp/upgrade/ + chmod +x /tmp/upgrade/*.sh + + # Patch CI_start_postgres to use "restart" instead of "start" so it + # is idempotent (initiate.sh starts postgres for top-level queries, + # then handle_extensions calls CI_start_postgres again) + sed -i "s/pg_ctl start -o/pg_ctl restart -o/g" /tmp/upgrade/common.sh + + # Patch PGSHARENEW to match nix binary expectations (share/postgresql/) + sed -i "s|PGSHARENEW=\"\$PG_UPGRADE_BIN_DIR/share\"|PGSHARENEW=\"\$PG_UPGRADE_BIN_DIR/share/postgresql\"|" /tmp/upgrade/initiate.sh + ' + + info "Starting Postgres 15 in upgrade container" + docker exec "$UPGRADE_CONTAINER" bash -c ' + su postgres -c "pg_ctl start -o \"-c config_file=/etc/postgresql/postgresql.conf\" -l /tmp/postgres.log" + ' + wait_for_healthy "$UPGRADE_CONTAINER" + + # initiate.sh expects the PG17 binaries tarball at /tmp/persistent/pg_upgrade_bin.tar.gz + # (hardcoded path - copied there during container setup above). + # + # Env vars for the unwrapped nix ELF binaries in the tarball: + # LD_LIBRARY_PATH - find libpq, libssl, etc. (RUNPATH points to absent nix store paths) + # NIX_PGLIBDIR - postgres uses this to find extension .so files + + info "Running initiate.sh (pg_upgrade: Postgres 15 -> 17)" + echo " This may take several minutes depending on database size..." + echo "" + if ! docker exec \ + -e IS_CI=true \ + -e PG_MAJOR_VERSION=17 \ + -e PGPASSWORD="$pg_password" \ + -e LD_LIBRARY_PATH=/tmp/pg_upgrade_bin/17/lib \ + -e NIX_PGLIBDIR=/tmp/pg_upgrade_bin/17/lib \ + "$UPGRADE_CONTAINER" \ + /tmp/upgrade/initiate.sh 17; then + echo "" + warn "initiate.sh failed. Its cleanup may have restored the original state" + warn "(re-enabled extensions, revoked superuser). Your data directory is" + warn "unchanged - no data was moved or deleted." + warn "" + warn "Check the output above for the root cause, fix it, and re-run." + docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true + die "initiate.sh failed" + fi + + info "initiate.sh completed successfully" + docker rm -f "$UPGRADE_CONTAINER" >/dev/null 2>&1 || true +} + +# --- Step 6: Run complete.sh in a native PG17 container ------------------- +# +# complete.sh applies post-upgrade patches (pg_net grants, vault re-encryption, +# pg_cron, predefined roles, vacuumdb, etc.). We run it in a PG17 container +# where the binaries are native - no nix extraction or LD_LIBRARY_PATH needed. + +run_complete() { + local abs_migration_dir + + abs_migration_dir=$(cd "$MIGRATION_DIR" && pwd) + + info "Starting PG17 container for complete.sh" + docker run -d --name "$COMPLETE_CONTAINER" \ + --entrypoint sleep \ + -v "${abs_migration_dir}:/mnt/host-migration" \ + -v "${db_config_vol}:/etc/postgresql-custom" \ + -v "${staging_dir}:/tmp/staging:ro" \ + -e PGPASSWORD="$pg_password" \ + "$PG17_UPGRADE_IMAGE" \ + infinity + + info "Preparing complete.sh environment" + # Save original db-config ownership so we can restore it if complete.sh fails. + # complete.sh needs PG17 ownership to start postgres, but if it fails the + # user needs to fall back to PG15 which uses a different uid. + docker exec "$COMPLETE_CONTAINER" bash -c ' + stat -c "%u:%g" /etc/postgresql-custom/pgsodium_root.key 2>/dev/null > /tmp/dbconfig_owner || true + ' + + docker exec "$COMPLETE_CONTAINER" bash -c ' + # Symlink bind mount so complete.sh CI wrapper can mv/rm/ln + ln -s /mnt/host-migration /data_migration + + # Remove the image default data dir (complete.sh creates a symlink here) + rm -rf /var/lib/postgresql/data + + # Fix ownership on db-config volume (PG15 uid differs from PG17) + chown -R postgres:postgres /etc/postgresql-custom/ + + # PG17 config includes this directory; may not exist from PG15 + mkdir -p /etc/postgresql-custom/conf.d + + mkdir -p /tmp/upgrade + + # Copy upgrade scripts + cp /tmp/staging/scripts/*.sh /tmp/upgrade/ + chmod +x /tmp/upgrade/*.sh + + # Patch --new-bin to use native bindir (we are in a PG17 container, + # no need for /tmp/pg_upgrade_bin/ paths) + sed -i "s|BINDIR=\"/tmp/pg_upgrade_bin/\$PG_MAJOR_VERSION/bin\"|BINDIR=\$(pg_config --bindir)|g" /tmp/upgrade/common.sh + ' + + info "Running complete.sh (post-upgrade patches, vacuum analyze)" + docker exec \ + -e IS_CI=true \ + -e PG_MAJOR_VERSION=17 \ + -e PGPASSWORD="$pg_password" \ + "$COMPLETE_CONTAINER" \ + /tmp/upgrade/complete.sh || true + + # complete.sh's ERR trap exits with 0 in some cases; check status file + local status + status=$(docker exec "$COMPLETE_CONTAINER" cat /tmp/pg-upgrade-status 2>/dev/null || echo "unknown") + if [ "$status" != "complete" ]; then + warn "complete.sh failed. Postgres log:" + docker exec "$COMPLETE_CONTAINER" cat /tmp/postgres.log 2>/dev/null || true + echo "" + # Restore db-config ownership so PG15 can start for rollback + warn "Restoring db-config ownership for PG15..." + local orig_owner + orig_owner=$(docker exec "$COMPLETE_CONTAINER" cat /tmp/dbconfig_owner 2>/dev/null || true) + if [ -n "$orig_owner" ]; then + docker exec "$COMPLETE_CONTAINER" chown -R "$orig_owner" /etc/postgresql-custom/ 2>/dev/null || true + fi + docker rm -f "$COMPLETE_CONTAINER" >/dev/null 2>&1 || true + echo "" + echo " Your Postgres 15 data is unchanged (data swap has not happened yet)." + echo " To restart Postgres 15:" + echo " rm -rf $MIGRATION_DIR" + echo " docker compose up -d" + echo "" + die "complete.sh failed (status: $status)" + fi + + info "complete.sh finished successfully" + docker rm -f "$COMPLETE_CONTAINER" >/dev/null 2>&1 || true +} + +# --- Step 7: Swap data directories ----------------------------------------- + +swap_data() { + info "Swapping data directories" + + echo " $DATA_DIR -> $BACKUP_DIR" + mv "$DATA_DIR" "$BACKUP_DIR" + + echo " $MIGRATION_DIR/pgdata -> $DATA_DIR" + mv "$MIGRATION_DIR/pgdata" "$DATA_DIR" + rm -rf "$MIGRATION_DIR" +} + +# --- Step 8: Start Postgres 17 --------------------------------------------- + +start_pg17() { + info "Starting Supabase with Postgres 17" + + # Ensure db-config volume has correct ownership and structure for PG17. + # complete.sh does this too, but just in case of partial + # failures from previous runs. + docker run --rm -v "${db_config_vol}:/vol" "$PG17_TARGET_IMAGE" sh -c ' + mkdir -p /vol/conf.d + chown -R postgres:postgres /vol/ + ' + + docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d + + echo " Waiting for Postgres 17 to be ready..." + local retries=60 + while [ $retries -gt 0 ]; do + if docker exec "$DB_CONTAINER" pg_isready -U postgres -h localhost >/dev/null 2>&1; then + break + fi + retries=$((retries - 1)) + sleep 2 + done + [ $retries -gt 0 ] || die "Postgres 17 did not start within 120 seconds." + + local new_version + new_version=$(run_sql_on "$DB_CONTAINER" -A -t -c "SHOW server_version;" 2>/dev/null | head -n 1) + echo " Postgres version: $new_version" + case "$new_version" in + 17.*) ;; + *) die "Expected Postgres 17.x, got: $new_version" ;; + esac +} + +# --- Step 9: Apply migrations not covered by complete.sh ------------------- +# +# These PG17 migrations run on fresh installs via initdb but not after +# pg_upgrade (init scripts don't rerun when PG_VERSION already exists). +# complete.sh doesn't cover them either. +# +# Source: postgres/migrations/db/migrations/ +# - 20250710151649_supabase_read_only_user_default_transaction_read_only.sql +# - 20251001204436_predefined_role_grants.sql (supabase_etl_admin + pg_monitor) +# - 20251105172723_grant_pg_reload_conf_to_postgres.sql +# - 20251121132723_correct_search_path_pgbouncer.sql + +apply_role_migrations() { + info "Applying Postgres 17 migrations" + + # Fix collation version mismatch first (upgrade used glibc 2.39, target + # image may use glibc 2.40). Do this before any other SQL to suppress + # the noisy warnings on every subsequent command. + for db in postgres template1 _supabase; do + docker exec -i -e PGPASSWORD="$pg_password" "$DB_CONTAINER" \ + psql -h localhost -U supabase_admin -d "$db" \ + -c "ALTER DATABASE \"$db\" REFRESH COLLATION VERSION;" || true + done + + # Create supabase_etl_admin role (doesn't exist in PG15 images). + # Must be created before running predefined_role_grants.sql which + # assumes it exists. + run_sql_on "$DB_CONTAINER" -c " + DO \$\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'supabase_etl_admin') THEN + CREATE USER supabase_etl_admin WITH LOGIN REPLICATION; + GRANT pg_read_all_data TO supabase_etl_admin; + GRANT CREATE ON DATABASE postgres TO supabase_etl_admin; + END IF; + END + \$\$;" || true + + # Run the migration files directly from the PG17 container image. + # They're idempotent (IF EXISTS / IF NOT EXISTS guards). + local migration_dir="/docker-entrypoint-initdb.d/migrations" + local migrations=" + 20250710151649_supabase_read_only_user_default_transaction_read_only.sql + 20251001204436_predefined_role_grants.sql + 20251105172723_grant_pg_reload_conf_to_postgres.sql + 20251121132723_correct_search_path_pgbouncer.sql + " + + for m in $migrations; do + echo " Running: $m" + docker exec -i \ + -e PGPASSWORD="$pg_password" \ + "$DB_CONTAINER" \ + psql -h localhost -U supabase_admin -d postgres -v ON_ERROR_STOP=1 \ + -f "${migration_dir}/${m}" || warn " $m failed (non-fatal)" + done +} + +# --- Step 10: Verify ------------------------------------------------------ + +verify() { + info "Verification" + + local version + version=$(run_sql_on "$DB_CONTAINER" -A -t -c "SELECT version();" 2>/dev/null | head -n 1) + echo " $version" + + echo "" + echo " Extensions:" + run_sql_on "$DB_CONTAINER" -c \ + "SELECT extname, extversion FROM pg_extension ORDER BY extname;" + + echo "" + info "Upgrade complete!" + echo "" + echo " To use Postgres 17 going forward, always include the override:" + echo " docker compose -f docker-compose.yml -f docker-compose.pg17.yml up -d" + echo "" + echo " Postgres 15 backup: $BACKUP_DIR" + echo " pgsodium key backup: ./volumes/db/pgsodium_root.key.bak.pg15" + echo " Once satisfied, you can reclaim space:" + echo " rm -rf $BACKUP_DIR ./volumes/db/pg17_upgrade_bin_*.tar.gz" + echo "" + echo " Rollback (if needed):" + echo " 1. docker compose -f docker-compose.yml -f docker-compose.pg17.yml down" + echo " 2. rm -rf $DATA_DIR" + echo " 3. mv $BACKUP_DIR $DATA_DIR" + echo " 4. docker compose run --rm db chown -R postgres:postgres /etc/postgresql-custom/" + echo " 5. docker compose up -d" + echo "" +} + +# --- Main ------------------------------------------------------------------- + +main() { + echo "" + echo "Supabase Self-Hosted: Postgres 15 -> 17 Upgrade" + echo "================================================" + + preflight + pull_image + build_tarball + drop_incompatible_extensions + stop_and_backup + run_upgrade + run_complete + swap_data + start_pg17 + apply_role_migrations + verify +} + +main "$@" diff --git a/supabase/code/versions.md b/supabase/code/versions.md index af00d5872..9f330dd55 100644 --- a/supabase/code/versions.md +++ b/supabase/code/versions.md @@ -1,5 +1,22 @@ # Docker Image Versions +## 2026-04-27 +- supabase/studio:2026.04.27-sha-5f60601 (prev supabase/studio:2026.04.08-sha-205cbe7) + +## 2026-04-08 +- supabase/studio:2026.04.08-sha-205cbe7 (prev supabase/studio:2026.03.16-sha-5528817) +- postgrest/postgrest:v14.8 (prev postgrest/postgrest:v14.6) +- supabase/storage-api:v1.48.26 (prev supabase/storage-api:v1.44.2) +- supabase/postgres-meta:v0.96.3 (prev supabase/postgres-meta:v0.95.2) +- supabase/logflare:1.36.1 (prev supabase/logflare:1.31.2) + +## 2026-03-16 +- supabase/studio:2026.03.16-sha-5528817 (prev supabase/studio:2026.02.16-sha-26c615c) +- kong/kong:3.9.1 (prev kong:2.8.1) +- postgrest/postgrest:v14.6 (prev postgrest/postgrest:v14.5) +- supabase/storage-api:v1.44.2 (prev supabase/storage-api:v1.37.8) +- supabase/edge-runtime:v1.71.2 (prev supabase/edge-runtime:v1.70.3) + ## 2026-02-16 - supabase/studio:2026.02.16-sha-26c615c (prev supabase/studio:2026.01.27-sha-6aa59ff) - supabase/gotrue:v2.186.0 (prev supabase/gotrue:v2.185.0) diff --git a/supabase/code/volumes/api/envoy/cds.yaml b/supabase/code/volumes/api/envoy/cds.yaml new file mode 100644 index 000000000..2d47842d6 --- /dev/null +++ b/supabase/code/volumes/api/envoy/cds.yaml @@ -0,0 +1,223 @@ +resources: + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: auth + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: auth + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: auth + port_value: 9999 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: /health + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: rest + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: rest + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: rest + port_value: 3000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: / + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: realtime + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: realtime + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: realtime-dev.supabase-realtime + port_value: 4000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: / + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: storage + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: storage + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: storage + port_value: 5000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: /status + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: functions + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: functions + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: functions + port_value: 9000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + tcp_health_check: {} + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: meta + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: meta + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: meta + port_value: 8080 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: /health + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 + + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: studio + connect_timeout: 5s + type: STRICT_DNS + dns_refresh_rate: 5s + dns_failure_refresh_rate: + base_interval: 1s + max_interval: 1s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: studio + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: studio + port_value: 3000 + health_checks: + - timeout: 2s + interval: 5s + unhealthy_threshold: 3 + healthy_threshold: 2 + http_health_check: + path: /project/default + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 10000 + max_pending_requests: 10000 + max_requests: 10000 diff --git a/supabase/code/volumes/api/envoy/docker-entrypoint.sh b/supabase/code/volumes/api/envoy/docker-entrypoint.sh new file mode 100755 index 000000000..783603859 --- /dev/null +++ b/supabase/code/volumes/api/envoy/docker-entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +# Generate SHA1 base64 hash for Envoy basic auth user list +PASSWORD_HASH=$(printf '%s' "${DASHBOARD_PASSWORD}" | openssl sha1 -binary | openssl base64) +DASHBOARD_BASIC_AUTH="${DASHBOARD_USERNAME}:{SHA}${PASSWORD_HASH}" + +echo "Generating Envoy configuration..." + +# Process the lds.yaml template with environment variables using sed +# Using | as delimiter since JWT tokens contain / +sed -e "s|\${ANON_KEY}|${ANON_KEY}|g" \ + -e "s|\${ANON_KEY_ASYMMETRIC}|${ANON_KEY_ASYMMETRIC}|g" \ + -e "s|\${SERVICE_ROLE_KEY}|${SERVICE_ROLE_KEY}|g" \ + -e "s|\${SERVICE_ROLE_KEY_ASYMMETRIC}|${SERVICE_ROLE_KEY_ASYMMETRIC}|g" \ + -e "s|\${SUPABASE_PUBLISHABLE_KEY}|${SUPABASE_PUBLISHABLE_KEY}|g" \ + -e "s|\${SUPABASE_SECRET_KEY}|${SUPABASE_SECRET_KEY}|g" \ + -e "s|\${DASHBOARD_BASIC_AUTH}|${DASHBOARD_BASIC_AUTH}|g" \ + /etc/envoy/lds.template.yaml > /etc/envoy/lds.yaml + +if [ -n "$SUPABASE_SECRET_KEY" ] && \ + [ -n "$SUPABASE_PUBLISHABLE_KEY" ] && \ + [ -n "$SERVICE_ROLE_KEY_ASYMMETRIC" ] && \ + [ -n "$ANON_KEY_ASYMMETRIC" ]; then + echo "Envoy sb_ key translation enabled" +else + echo "Envoy running in legacy API key mode (sb_ keys disabled)" +fi + +echo "Envoy configuration generated successfully" +echo "Starting Envoy..." + +# Start Envoy +exec envoy -c /etc/envoy/envoy.yaml "$@" diff --git a/supabase/code/volumes/api/envoy/envoy.yaml b/supabase/code/volumes/api/envoy/envoy.yaml new file mode 100644 index 000000000..bf3dd4ebf --- /dev/null +++ b/supabase/code/volumes/api/envoy/envoy.yaml @@ -0,0 +1,27 @@ +dynamic_resources: + cds_config: + path_config_source: + path: /etc/envoy/cds.yaml + resource_api_version: V3 + lds_config: + path_config_source: + path: /etc/envoy/lds.yaml + resource_api_version: V3 + +node: + cluster: supabase_cluster + id: supabase_node + +overload_manager: + resource_monitors: + - name: envoy.resource_monitors.global_downstream_max_connections + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 30000 + +admin: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 diff --git a/supabase/code/volumes/api/envoy/lds.template.yaml b/supabase/code/volumes/api/envoy/lds.template.yaml new file mode 100644 index 000000000..4d55f60a0 --- /dev/null +++ b/supabase/code/volumes/api/envoy/lds.template.yaml @@ -0,0 +1,994 @@ +resources: + - '@type': type.googleapis.com/envoy.config.listener.v3.Listener + name: supabase + per_connection_buffer_limit_bytes: 32768 # 32 KiB + + address: + socket_address: + address: 0.0.0.0 + port_value: 8000 + + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + normalize_path: true + merge_slashes: true + path_with_escaped_slashes_action: REJECT_REQUEST + use_remote_address: true + common_http_protocol_options: + headers_with_underscores_action: REJECT_REQUEST + upgrade_configs: + - upgrade_type: websocket + access_log: + - name: envoy.access_loggers.stdout + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog + log_format: + text_format_source: + inline_string: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT% - - [%START_TIME(%d/%b/%Y:%H:%M:%S %z)%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %BYTES_SENT% \"%REQ(REFERER)%\" \"%REQ(USER-AGENT)%\"\n" + + route_config: + name: supabase_route + virtual_hosts: + - name: supabase_host + domains: + - '*' + cors: + allow_origin_string_match: + - safe_regex: + regex: ".*" + allow_methods: "GET,POST,PUT,PATCH,DELETE,OPTIONS,HEAD,CONNECT,TRACE" + allow_headers: "*" + expose_headers: "*" + max_age: "3600" + request_headers_to_add: + - header: + key: X-Forwarded-Host + value: "%REQ(:AUTHORITY)%" + append_action: ADD_IF_ABSENT + - header: + key: X-Forwarded-Port + value: "%DOWNSTREAM_LOCAL_PORT%" + append_action: ADD_IF_ABSENT + routes: + - match: + prefix: /auth/v1/verify + route: + cluster: auth + prefix_rewrite: /verify + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/verify + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /auth/v1/callback + route: + cluster: auth + prefix_rewrite: /callback + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/callback + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /auth/v1/authorize + route: + cluster: auth + prefix_rewrite: /authorize + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/authorize + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /auth/v1/.well-known/jwks.json + route: + cluster: auth + prefix_rewrite: /.well-known/jwks.json + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/.well-known/jwks.json + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /.well-known/oauth-authorization-server + route: + cluster: auth + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /.well-known/oauth-authorization-server + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /sso/saml/acs + route: + cluster: auth + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /sso/saml/acs + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /sso/saml/metadata + route: + cluster: auth + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /sso/saml/metadata + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - name: functions-v1-all + match: + prefix: /functions/v1/ + route: + cluster: functions + prefix_rewrite: / + timeout: 150s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /functions/v1/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /storage/v1/ + route: + cluster: storage + prefix_rewrite: / + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /storage/v1 + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + - name: auth-v1-protected + match: + prefix: /auth/v1/ + route: + cluster: auth + prefix_rewrite: / + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /auth/v1/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: rest-v1-protected + match: + prefix: /rest/v1/ + route: + cluster: rest + prefix_rewrite: / + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /rest/v1/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: graphql-v1-protected + match: + prefix: /graphql/v1 + route: + cluster: rest + prefix_rewrite: /rpc/graphql + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /graphql/v1 + append_action: ADD_IF_ABSENT + - header: + key: Content-Profile + value: graphql_public + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: realtime-v1-api-protected + match: + prefix: /realtime/v1/api + route: + cluster: realtime + prefix_rewrite: /api + timeout: 30s + host_rewrite_literal: realtime-dev.supabase-realtime + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /realtime/v1/api + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: realtime-v1-ws-protected + match: + prefix: /realtime/v1/ + route: + cluster: realtime + prefix_rewrite: /socket/ + timeout: 30s + host_rewrite_literal: realtime-dev.supabase-realtime + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /realtime/v1/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - name: pg-protected + match: + prefix: /pg/ + route: + cluster: meta + prefix_rewrite: / + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /pg/ + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + + - match: + prefix: /api/mcp + route: + cluster: studio + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /api/mcp + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: DENY + policies: + deny_all: + permissions: + - any: true + principals: + - any: true + + - match: + prefix: /mcp + route: + cluster: studio + prefix_rewrite: /api/mcp + timeout: 30s + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: /mcp + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.basic_auth: + '@type': >- + type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + # Block access to /mcp by default + rbac: + rules: + action: DENY + policies: + deny_all: + permissions: + - any: true + principals: + - any: true + # Enable local access (danger zone!) + # 1. Comment out the 'rbac' block above. + # 2. Uncomment and adjust the 'rbac' block below. + # 3. Add or adjust your local IPs in 'principals'. + #rbac: + # rules: + # action: ALLOW + # policies: + # allow_local: + # permissions: + # - any: true + # principals: + # - direct_remote_ip: + # address_prefix: 127.0.0.1 + # prefix_len: 32 + # - direct_remote_ip: + # address_prefix: ::1 + # prefix_len: 128 + + - match: + prefix: / + route: + cluster: studio + timeout: 30s + request_headers_to_remove: + - authorization + request_headers_to_add: + - header: + key: X-Forwarded-Prefix + value: / + append_action: ADD_IF_ABSENT + typed_per_filter_config: + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: ALLOW + policies: + allow_all: + permissions: + - any: true + principals: + - any: true + + http_filters: + - name: envoy.filters.http.cors + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors + + - name: envoy.filters.http.basic_auth + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.basic_auth.v3.BasicAuth + users: + inline_string: '${DASHBOARD_BASIC_AUTH}' + + # Copies ?apikey=... from the URL into the apikey header when clients omit the header. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local FUNCTIONS_ROUTE = "functions-v1-all" + local FUNCTIONS_PREFIX = "/functions/v1/" + + local function is_functions_request(request_handle, headers) + if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then + return true + end + + local path = headers:get(":path") + if path == nil then + return false + end + + return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if is_functions_request(request_handle, headers) then + return + end + + if headers:get("apikey") ~= nil then + return + end + + local path = headers:get(":path") + local query_start = string.find(path, "?", 1, true) + if query_start == nil then + return + end + + local query = string.sub(path, query_start + 1) + for key, value in string.gmatch(query, "([^&]+)=([^&]*)") do + if key == "apikey" and value ~= "" then + headers:add("apikey", value) + return + end + end + end + + # Translates the query parameter apikey into the matching internal JWT and rewrites the URL so only JWTs propagate downstream. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local FUNCTIONS_ROUTE = "functions-v1-all" + local FUNCTIONS_PREFIX = "/functions/v1/" + local SECRET_KEY = "${SUPABASE_SECRET_KEY}" + local PUBLISHABLE_KEY = "${SUPABASE_PUBLISHABLE_KEY}" + local SERVICE_ROLE_JWT = "${SERVICE_ROLE_KEY_ASYMMETRIC}" + local ANON_JWT = "${ANON_KEY_ASYMMETRIC}" + local TRANSLATION_ENABLED = SECRET_KEY ~= "" and PUBLISHABLE_KEY ~= "" and SERVICE_ROLE_JWT ~= "" and ANON_JWT ~= "" + + local function is_functions_request(request_handle, headers) + if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then + return true + end + + local path = headers:get(":path") + if path == nil then + return false + end + + return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX + end + + local function translate_apikey(apikey) + if apikey == nil or apikey == "" then + return nil + end + + if not TRANSLATION_ENABLED then + return nil + end + + if apikey == SECRET_KEY then + return SERVICE_ROLE_JWT + end + + if apikey == PUBLISHABLE_KEY then + return ANON_JWT + end + + return nil + end + + local function extract_query_apikey(path) + if path == nil or path == "" then + return nil + end + + local query_start = string.find(path, "?", 1, true) + if query_start == nil then + return nil + end + + local query = string.sub(path, query_start + 1) + for key, value in string.gmatch(query, "([^&]+)=([^&]*)") do + if key == "apikey" and value ~= "" then + return value + end + end + + return nil + end + + local function replace_query_apikey(path, new_value) + if path == nil or path == "" or new_value == nil or new_value == "" then + return nil + end + + local query_start = string.find(path, "?", 1, true) + if query_start == nil then + return nil + end + + local base = string.sub(path, 1, query_start) + local query = string.sub(path, query_start + 1) + local updated = {} + local replaced = false + + for part in string.gmatch(query, "([^&]+)") do + local key, value = string.match(part, "([^=]+)=(.*)") + if key == "apikey" then + part = key .. "=" .. new_value + replaced = true + end + table.insert(updated, part) + end + + if not replaced then + return nil + end + + return base .. table.concat(updated, "&") + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if is_functions_request(request_handle, headers) then + return + end + + local path = headers:get(":path") + local apikey = extract_query_apikey(path) + local translated = translate_apikey(apikey) + + if translated == nil then + return + end + + headers:replace("apikey", translated) + + local rewritten_path = replace_query_apikey(path, translated) + if rewritten_path ~= nil then + headers:replace(":path", rewritten_path) + end + end + + # Translates an apikey header into the appropriate internal JWT for downstream RBAC checks. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local FUNCTIONS_ROUTE = "functions-v1-all" + local FUNCTIONS_PREFIX = "/functions/v1/" + local SECRET_KEY = "${SUPABASE_SECRET_KEY}" + local PUBLISHABLE_KEY = "${SUPABASE_PUBLISHABLE_KEY}" + local SERVICE_ROLE_JWT = "${SERVICE_ROLE_KEY_ASYMMETRIC}" + local ANON_JWT = "${ANON_KEY_ASYMMETRIC}" + local TRANSLATION_ENABLED = SECRET_KEY ~= "" and PUBLISHABLE_KEY ~= "" and SERVICE_ROLE_JWT ~= "" and ANON_JWT ~= "" + + local function is_functions_request(request_handle, headers) + if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then + return true + end + + local path = headers:get(":path") + if path == nil then + return false + end + + return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX + end + + local function translate_apikey(apikey) + if apikey == nil or apikey == "" then + return nil + end + + if not TRANSLATION_ENABLED then + return nil + end + + if apikey == SECRET_KEY then + return SERVICE_ROLE_JWT + end + + if apikey == PUBLISHABLE_KEY then + return ANON_JWT + end + + return nil + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if is_functions_request(request_handle, headers) then + return + end + + local translated = translate_apikey(headers:get("apikey")) + if translated ~= nil and translated ~= "" then + headers:replace("apikey", translated) + end + end + + # Mirrors apikey into x-api-key for realtime WS compatibility. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local REALTIME_WS_ROUTE = "realtime-v1-ws-protected" + + function envoy_on_request(request_handle) + local route_name = request_handle:streamInfo():routeName() + if route_name ~= REALTIME_WS_ROUTE then + return + end + + local headers = request_handle:headers() + local apikey = headers:get("apikey") + if apikey == nil or apikey == "" then + return + end + + headers:replace("x-api-key", apikey) + end + + # Synthesizes an Authorization header (Bearer …) from apikey when callers don’t provide a real JWT header. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local FUNCTIONS_ROUTE = "functions-v1-all" + local FUNCTIONS_PREFIX = "/functions/v1/" + local REALTIME_WS_ROUTE = "realtime-v1-ws-protected" + + local function is_functions_request(request_handle, headers) + if request_handle:streamInfo():routeName() == FUNCTIONS_ROUTE then + return true + end + + local path = headers:get(":path") + if path == nil then + return false + end + + return string.sub(path, 1, string.len(FUNCTIONS_PREFIX)) == FUNCTIONS_PREFIX + end + + local function has_real_jwt(auth_header) + if auth_header == nil or auth_header == "" then + return false + end + + if string.sub(auth_header, 1, 7) ~= "Bearer " then + return false + end + + return string.sub(auth_header, 1, 10) ~= "Bearer sb_" + end + + local function format_authorization(value) + if value == nil or value == "" then + return nil + end + + if string.sub(value, 1, 7) == "Bearer " then + return value + end + + return "Bearer " .. value + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if request_handle:streamInfo():routeName() == REALTIME_WS_ROUTE then + return + end + + if is_functions_request(request_handle, headers) then + return + end + + if has_real_jwt(headers:get("authorization")) then + return + end + + local apikey = headers:get("apikey") + local authorization_value = format_authorization(apikey) + if authorization_value ~= nil then + headers:replace("authorization", authorization_value) + end + end + + # Returns 401 for missing/invalid API keys on protected API routes. + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + local ANON_KEY = "${ANON_KEY}" + local SERVICE_ROLE_KEY = "${SERVICE_ROLE_KEY}" + local SUPABASE_PUBLISHABLE_KEY = "${SUPABASE_PUBLISHABLE_KEY}" + local SUPABASE_SECRET_KEY = "${SUPABASE_SECRET_KEY}" + local ANON_KEY_ASYMMETRIC = "${ANON_KEY_ASYMMETRIC}" + local SERVICE_ROLE_KEY_ASYMMETRIC = "${SERVICE_ROLE_KEY_ASYMMETRIC}" + local TRANSLATION_ENABLED = SUPABASE_SECRET_KEY ~= "" and SUPABASE_PUBLISHABLE_KEY ~= "" and SERVICE_ROLE_KEY_ASYMMETRIC ~= "" and ANON_KEY_ASYMMETRIC ~= "" + + local PROTECTED_ROUTES = { + ["auth-v1-protected"] = true, + ["rest-v1-protected"] = true, + ["graphql-v1-protected"] = true, + ["realtime-v1-api-protected"] = true, + ["realtime-v1-ws-protected"] = true, + ["pg-protected"] = true, + } + + local function is_protected_route(route_name) + if route_name == nil or route_name == "" then + return false + end + + return PROTECTED_ROUTES[route_name] == true + end + + local function is_valid_apikey(apikey) + if apikey == nil or apikey == "" then + return false + end + + if SERVICE_ROLE_KEY ~= "" and apikey == SERVICE_ROLE_KEY then + return true + end + + if ANON_KEY ~= "" and apikey == ANON_KEY then + return true + end + + if TRANSLATION_ENABLED and apikey == SERVICE_ROLE_KEY_ASYMMETRIC then + return true + end + + if TRANSLATION_ENABLED and apikey == ANON_KEY_ASYMMETRIC then + return true + end + + return false + end + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + local route_name = request_handle:streamInfo():routeName() + if not is_protected_route(route_name) then + return + end + + if is_valid_apikey(headers:get("apikey")) then + return + end + + request_handle:respond({ + [":status"] = "401", + ["content-type"] = "text/plain", + }, "Unauthorized") + end + + - name: envoy.filters.http.rbac + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC + rules: + action: ALLOW + policies: + admin: + permissions: + - url_path: + path: + prefix: /pg/ + principals: + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY}' + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY_ASYMMETRIC}' + apikey: + permissions: + - url_path: + path: + prefix: /auth/v1/ + - url_path: + path: + prefix: /rest/v1/ + - url_path: + path: + prefix: /realtime/v1/api + - url_path: + path: + prefix: /realtime/v1/ + - url_path: + path: + prefix: /graphql/v1 + principals: + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY}' + - header: + name: apikey + string_match: + exact: '${ANON_KEY}' + - header: + name: apikey + string_match: + exact: '${SERVICE_ROLE_KEY_ASYMMETRIC}' + - header: + name: apikey + string_match: + exact: '${ANON_KEY_ASYMMETRIC}' + - name: envoy.filters.http.router + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.router.v3.Router diff --git a/supabase/code/volumes/api/kong-entrypoint.sh b/supabase/code/volumes/api/kong-entrypoint.sh new file mode 100755 index 000000000..d5eee9332 --- /dev/null +++ b/supabase/code/volumes/api/kong-entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Custom entrypoint for Kong that builds Lua expressions for request-transformer +# and performs environment variable substitution in the declarative config. + +# Build Lua expressions for translating opaque API keys to asymmetric JWTs. +# When opaque keys are not configured (empty env vars), expressions fall through +# to legacy-only behavior - just passing apikey as-is. +# +# Full expression logic (when opaque keys are configured): +# 1. If Authorization header exists and is NOT an sb_ key -> pass through (user session JWT) +# 2. If apikey matches secret key -> set service_role asymmetric JWT internal "API key" +# 3. If apikey matches publishable key -> set anon asymmetric JWT internal "API key" +# 4. Fallback: pass apikey as-is (legacy HS256 JWT) + +if [ -n "$SUPABASE_SECRET_KEY" ] && [ -n "$SUPABASE_PUBLISHABLE_KEY" ]; then + # Opaque keys configured -> full translation expressions + export LUA_AUTH_EXPR="\$((headers.authorization ~= nil and headers.authorization:sub(1, 10) ~= 'Bearer sb_' and headers.authorization) or (headers.apikey == '$SUPABASE_SECRET_KEY' and 'Bearer $SERVICE_ROLE_KEY_ASYMMETRIC') or (headers.apikey == '$SUPABASE_PUBLISHABLE_KEY' and 'Bearer $ANON_KEY_ASYMMETRIC') or headers.apikey)" + + # Realtime WebSocket: reads from query_params.apikey (supabase-js sends apikey + # via query string), outputs to x-api-key header which Realtime checks first. + export LUA_RT_WS_EXPR="\$((query_params.apikey == '$SUPABASE_SECRET_KEY' and '$SERVICE_ROLE_KEY_ASYMMETRIC') or (query_params.apikey == '$SUPABASE_PUBLISHABLE_KEY' and '$ANON_KEY_ASYMMETRIC') or query_params.apikey)" +else + # Legacy API keys, not sb_ API keys -> pass apikey through unchanged + export LUA_AUTH_EXPR="\$((headers.authorization ~= nil and headers.authorization:sub(1, 10) ~= 'Bearer sb_' and headers.authorization) or headers.apikey)" + export LUA_RT_WS_EXPR="\$(query_params.apikey)" +fi + +# Substitute environment variables in the Kong declarative config. +# Uses awk instead of eval/echo to preserve YAML quoting (eval strips double +# quotes, breaking "Header: value" patterns that YAML parses as mappings). +awk '{ + result = "" + rest = $0 + while (match(rest, /\$[A-Za-z_][A-Za-z_0-9]*/)) { + varname = substr(rest, RSTART + 1, RLENGTH - 1) + if (varname in ENVIRON) { + result = result substr(rest, 1, RSTART - 1) ENVIRON[varname] + } else { + result = result substr(rest, 1, RSTART + RLENGTH - 1) + } + rest = substr(rest, RSTART + RLENGTH) + } + print result rest +}' /home/kong/temp.yml > "$KONG_DECLARATIVE_CONFIG" + +# Remove empty key-auth credentials (unconfigured opaque keys) +sed -i '/^[[:space:]]*- key:[[:space:]]*$/d' "$KONG_DECLARATIVE_CONFIG" + +exec /entrypoint.sh kong docker-start diff --git a/supabase/code/volumes/api/kong.yml b/supabase/code/volumes/api/kong.yml index ae69cec28..b89e868f5 100644 --- a/supabase/code/volumes/api/kong.yml +++ b/supabase/code/volumes/api/kong.yml @@ -9,9 +9,11 @@ consumers: - username: anon keyauth_credentials: - key: $SUPABASE_ANON_KEY + - key: $SUPABASE_PUBLISHABLE_KEY - username: service_role keyauth_credentials: - key: $SUPABASE_SERVICE_KEY + - key: $SUPABASE_SECRET_KEY ### ### Access Control List @@ -36,6 +38,7 @@ basicauth_credentials: services: ## Open Auth routes - name: auth-v1-open + _comment: 'Auth: /auth/v1/verify* -> http://auth:9999/verify*' url: http://auth:9999/verify routes: - name: auth-v1-open @@ -45,6 +48,7 @@ services: plugins: - name: cors - name: auth-v1-open-callback + _comment: 'Auth: /auth/v1/callback* -> http://auth:9999/callback*' url: http://auth:9999/callback routes: - name: auth-v1-open-callback @@ -54,6 +58,7 @@ services: plugins: - name: cors - name: auth-v1-open-authorize + _comment: 'Auth: /auth/v1/authorize* -> http://auth:9999/authorize*' url: http://auth:9999/authorize routes: - name: auth-v1-open-authorize @@ -62,10 +67,40 @@ services: - /auth/v1/authorize plugins: - name: cors + - name: auth-v1-open-jwks + _comment: 'Auth: /auth/v1/.well-known/jwks.json -> http://auth:9999/.well-known/jwks.json' + url: http://auth:9999/.well-known/jwks.json + routes: + - name: auth-v1-open-jwks + strip_path: true + paths: + - /auth/v1/.well-known/jwks.json + plugins: + - name: cors + + - name: auth-v1-open-sso-acs + url: "http://auth:9999/sso/saml/acs" + routes: + - name: auth-v1-open-sso-acs + strip_path: true + paths: + - /sso/saml/acs + plugins: + - name: cors + + - name: auth-v1-open-sso-metadata + url: "http://auth:9999/sso/saml/metadata" + routes: + - name: auth-v1-open-sso-metadata + strip_path: true + paths: + - /sso/saml/metadata + plugins: + - name: cors ## Secure Auth routes - name: auth-v1 - _comment: 'GoTrue: /auth/v1/* -> http://auth:9999/*' + _comment: 'Auth: /auth/v1/* -> http://auth:9999/*' url: http://auth:9999/ routes: - name: auth-v1-all @@ -77,6 +112,14 @@ services: - name: key-auth config: hide_credentials: false + - name: request-transformer + config: + add: + headers: + - "Authorization: $LUA_AUTH_EXPR" + replace: + headers: + - "Authorization: $LUA_AUTH_EXPR" - name: acl config: hide_groups_header: true @@ -84,7 +127,7 @@ services: - admin - anon - ## Secure REST routes + ## Secure PostgREST routes - name: rest-v1 _comment: 'PostgREST: /rest/v1/* -> http://rest:3000/*' url: http://rest:3000/ @@ -97,7 +140,15 @@ services: - name: cors - name: key-auth config: - hide_credentials: true + hide_credentials: false + - name: request-transformer + config: + add: + headers: + - "Authorization: $LUA_AUTH_EXPR" + replace: + headers: + - "Authorization: $LUA_AUTH_EXPR" - name: acl config: hide_groups_header: true @@ -118,12 +169,16 @@ services: - name: cors - name: key-auth config: - hide_credentials: true + hide_credentials: false - name: request-transformer config: add: headers: - - Content-Profile:graphql_public + - "Content-Profile: graphql_public" + - "Authorization: $LUA_AUTH_EXPR" + replace: + headers: + - "Authorization: $LUA_AUTH_EXPR" - name: acl config: hide_groups_header: true @@ -146,14 +201,23 @@ services: - name: key-auth config: hide_credentials: false + - name: request-transformer + config: + add: + headers: + - "x-api-key:$LUA_RT_WS_EXPR" + replace: + querystring: + - "apikey:$LUA_RT_WS_EXPR" - name: acl config: hide_groups_header: true allow: - admin - anon + - name: realtime-v1-rest - _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' + _comment: 'Realtime: /realtime/v1/api/* -> http://realtime:4000/api/*' url: http://realtime-dev.supabase-realtime:4000/api protocol: http routes: @@ -166,13 +230,30 @@ services: - name: key-auth config: hide_credentials: false + - name: request-transformer + config: + add: + headers: + - "Authorization: $LUA_AUTH_EXPR" + replace: + headers: + - "Authorization: $LUA_AUTH_EXPR" - name: acl config: hide_groups_header: true allow: - admin - anon - ## Storage routes: the storage server manages its own auth + + ## Storage API endpoint (with Authorization header transformation). + ## No key-auth — S3 protocol requests don't carry an apikey header. + ## + ## The request-transformer translates opaque API keys to asymmetric JWTs + ## and passes through existing Authorization headers (user JWTs, AWS SigV4). + ## When no Authorization or apikey header is present (S3 presigned URLs), + ## the Lua expression evaluates to nil which Kong renders as empty string. + ## The post-function strips this empty header so Storage's S3 signature + ## verification falls through to query-parameter parsing. - name: storage-v1 _comment: 'Storage: /storage/v1/* -> http://storage:5000/*' url: http://storage:5000/ @@ -183,11 +264,28 @@ services: - /storage/v1/ plugins: - name: cors + - name: request-transformer + config: + add: + headers: + - "Authorization: $LUA_AUTH_EXPR" + replace: + headers: + - "Authorization: $LUA_AUTH_EXPR" + - name: post-function + config: + access: + - | + local auth = kong.request.get_header("authorization") + if auth == nil or auth == "" or auth:find("^%s*$") then + kong.service.request.clear_header("authorization") + end ## Edge Functions routes - name: functions-v1 _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*' url: http://functions:9000/ + read_timeout: 150000 routes: - name: functions-v1-all strip_path: true @@ -196,6 +294,18 @@ services: plugins: - name: cors + ## OAuth 2.0 Authorization Server Metadata (RFC 8414) + - name: well-known-oauth + _comment: 'Auth: /.well-known/oauth-authorization-server -> http://auth:9999/.well-known/oauth-authorization-server' + url: http://auth:9999/.well-known/oauth-authorization-server + routes: + - name: well-known-oauth + strip_path: true + paths: + - /.well-known/oauth-authorization-server + plugins: + - name: cors + ## Analytics routes ## Not used - Studio and Vector talk directly to analytics via Docker networking. ## If external access is needed, add routes with key-auth matching Logflare's x-api-key auth. diff --git a/supabase/code/volumes/functions/main/index.ts b/supabase/code/volumes/functions/main/index.ts index b593527d6..ebe2061c4 100644 --- a/supabase/code/volumes/functions/main/index.ts +++ b/supabase/code/volumes/functions/main/index.ts @@ -3,8 +3,31 @@ import * as jose from 'https://deno.land/x/jose@v4.14.4/index.ts' console.log('main function started') const JWT_SECRET = Deno.env.get('JWT_SECRET') +const SUPABASE_URL = Deno.env.get('SUPABASE_URL') const VERIFY_JWT = Deno.env.get('VERIFY_JWT') === 'true' +// Create JWKS for ES256/RS256 tokens (newer tokens) +let SUPABASE_JWT_KEYS: ReturnType | null = null +if (SUPABASE_URL) { + try { + SUPABASE_JWT_KEYS = jose.createRemoteJWKSet( + new URL('/auth/v1/.well-known/jwks.json', SUPABASE_URL) + ) + } catch (e) { + console.error('Failed to fetch JWKS from SUPABASE_URL:', e) + } +} + +/** + * Extract JWT token from Authorization header + * + * Parses the Authorization header to extract the Bearer token. + * Expects format: "Bearer " + * + * @param req - The HTTP request object + * @returns The JWT token string + * @throws Error if Authorization header is missing or malformed + */ function getAuthToken(req: Request) { const authHeader = req.headers.get('authorization') if (!authHeader) { @@ -17,23 +40,75 @@ function getAuthToken(req: Request) { return token } -async function verifyJWT(jwt: string): Promise { - const encoder = new TextEncoder() +async function isValidLegacyJWT(jwt: string): Promise { + if (!JWT_SECRET) { + console.error('JWT_SECRET not available for HS256 token verification') + return false + } + + const encoder = new TextEncoder(); const secretKey = encoder.encode(JWT_SECRET) + try { - await jose.jwtVerify(jwt, secretKey) - } catch (err) { - console.error(err) + await jose.jwtVerify(jwt, secretKey); + } catch (e) { + console.error('Symmetric Legacy JWT verification error', e); + return false; + } + return true; +} + +async function isValidJWT(jwt: string): Promise { + if (!SUPABASE_JWT_KEYS) { + console.error('JWKS not available for ES256/RS256 token verification') return false } - return true + + try { + await jose.jwtVerify(jwt, SUPABASE_JWT_KEYS) + } catch (e) { + console.error('Asymmetric JWT verification error', e); + return false + } + + return true; +} + +/** + * Verify JWT token, handling both legacy (HS256) and newer (ES256/RS256) algorithms + * + * This function automatically detects the algorithm used in the token and applies + * the appropriate verification method: + * - HS256: Uses JWT_SECRET (symmetric key) + * - ES256/RS256: Uses JWKS endpoint (asymmetric public keys) + * + * This fix ensures compatibility with both legacy tokens and newer asymmetric tokens, + * resolving the "Key for the ES256 algorithm must be of type CryptoKey" error. + * + * @param jwt - The JWT token string to verify + * @returns Promise resolving to true if verification succeeds, false otherwise + */ +async function isValidHybridJWT(jwt: string): Promise { + const { alg: jwtAlgorithm } = jose.decodeProtectedHeader(jwt) + + if (jwtAlgorithm === 'HS256') { + console.log(`Legacy token type detected, attempting ${jwtAlgorithm} verification.`) + + return await isValidLegacyJWT(jwt) + } + + if (jwtAlgorithm === 'ES256' || jwtAlgorithm === 'RS256') { + return await isValidJWT(jwt) + } + + return false; } Deno.serve(async (req: Request) => { if (req.method !== 'OPTIONS' && VERIFY_JWT) { try { const token = getAuthToken(req) - const isValidJWT = await verifyJWT(token) + const isValidJWT = await isValidHybridJWT(token); if (!isValidJWT) { return new Response(JSON.stringify({ msg: 'Invalid JWT' }), { diff --git a/supabase/code/volumes/logs/vector.yml b/supabase/code/volumes/logs/vector.yml index d600bf286..f63bfd8ad 100644 --- a/supabase/code/volumes/logs/vector.yml +++ b/supabase/code/volumes/logs/vector.yml @@ -30,7 +30,7 @@ transforms: inputs: - project_logs route: - kong: '.appname == "supabase-kong"' + kong: '.appname == "supabase-kong" || .appname == "supabase-envoy"' auth: '.appname == "supabase-auth"' rest: '.appname == "supabase-rest"' realtime: '.appname == "realtime-dev.supabase-realtime"' diff --git a/supabase/code/volumes/pooler/pooler.exs b/supabase/code/volumes/pooler/pooler.exs index 791d61c84..06f05e1b4 100644 --- a/supabase/code/volumes/pooler/pooler.exs +++ b/supabase/code/volumes/pooler/pooler.exs @@ -8,7 +8,7 @@ params = %{ "external_id" => System.get_env("POOLER_TENANT_ID"), - "db_host" => "db", + "db_host" => System.get_env("POSTGRES_HOST") || "db", "db_port" => System.get_env("POSTGRES_PORT"), "db_database" => System.get_env("POSTGRES_DB"), "require_user" => false, diff --git a/supabase/code/volumes/proxy/caddy/Caddyfile b/supabase/code/volumes/proxy/caddy/Caddyfile index d76c3ed36..f1d1c4805 100644 --- a/supabase/code/volumes/proxy/caddy/Caddyfile +++ b/supabase/code/volumes/proxy/caddy/Caddyfile @@ -1,28 +1,10 @@ {$PROXY_DOMAIN} { - @supabase_api path /auth/v1/* /rest/v1/* /graphql/v1/* /realtime/v1/* /functions/v1/* /mcp + @supabase_api path /auth/v1/* /rest/v1/* /graphql/v1 /realtime/v1/* /storage/v1/* /functions/v1/* /mcp /sso/* handle @supabase_api { reverse_proxy kong:8000 } - handle_path /storage/v1/* { - # CORS headers for Storage (bypasses Kong, which normally handles CORS) - @cors_preflight method OPTIONS - handle @cors_preflight { - header Access-Control-Allow-Origin * - header Access-Control-Allow-Methods "GET, HEAD, PUT, PATCH, POST, DELETE, OPTIONS" - header Access-Control-Allow-Headers * - respond 204 - } - - header Access-Control-Allow-Origin * - - reverse_proxy storage:5000 { - # Required for TUS resumable upload Location headers and S3 signature verification. - header_up X-Forwarded-Prefix /{http.request.orig_uri.path.0}/{http.request.orig_uri.path.1} - } - } - handle { basic_auth { # PROXY_AUTH_PASSWORD is overwritten with the bcrypt on startup diff --git a/supabase/code/volumes/proxy/nginx/supabase-nginx.conf.tpl b/supabase/code/volumes/proxy/nginx/supabase-nginx.conf.tpl index f50f23653..24979499c 100644 --- a/supabase/code/volumes/proxy/nginx/supabase-nginx.conf.tpl +++ b/supabase/code/volumes/proxy/nginx/supabase-nginx.conf.tpl @@ -50,6 +50,10 @@ server { proxy_pass http://kong_upstream; } + location /graphql { + proxy_pass http://kong_upstream; + } + location /realtime/v1/ { proxy_pass http://kong_upstream; @@ -67,43 +71,25 @@ server { } location /storage/v1/ { - proxy_pass http://storage:5000/; - proxy_buffering off; + proxy_pass http://kong_upstream; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header X-Forwarded-Port $server_port; + proxy_buffering off; + proxy_request_buffering off; - # Required for TUS resumable upload Location headers and S3 signature verification. - proxy_set_header X-Forwarded-Prefix /storage/v1; + chunked_transfer_encoding off; client_max_body_size 0; - - # CORS headers for Storage (bypasses Kong, which normally handles CORS) - if ($request_method = OPTIONS) { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, HEAD, PUT, PATCH, POST, DELETE, OPTIONS'; - add_header 'Access-Control-Allow-Headers' '*'; - add_header 'Content-Length' 0; - add_header 'Content-Type' 'text/plain charset=UTF-8'; - return 204; - } - - add_header 'Access-Control-Allow-Origin' '*'; } location /functions { proxy_pass http://kong_upstream; } - location /graphql { + location /mcp { proxy_pass http://kong_upstream; } - location /mcp { + location /sso { proxy_pass http://kong_upstream; } }