diff --git a/Dockerfile.postgres b/Dockerfile.postgres new file mode 100644 index 0000000000..7f984cce26 --- /dev/null +++ b/Dockerfile.postgres @@ -0,0 +1,71 @@ +# Fast-startup PostgreSQL for sqlc testing +# +# This image pre-initializes the database at build time so that container +# startup only needs to launch the postgres process — no initdb, no +# entrypoint scripts, no first-run setup. +# +# Build: +# docker build -f Dockerfile.postgres -t sqlc-postgres . +# +# Run: +# docker run -d -p 5432:5432 sqlc-postgres +# +# For fastest I/O, mount the data directory on tmpfs: +# docker run -d -p 5432:5432 --tmpfs /var/lib/postgresql/data:rw,noexec,nosuid,size=256m sqlc-postgres +# +# Connection URI: +# postgres://postgres:mysecretpassword@localhost:5432/postgres?sslmode=disable + +ARG PG_VERSION=18 +FROM postgres:${PG_VERSION} + +ENV POSTGRES_USER=postgres \ + POSTGRES_PASSWORD=mysecretpassword \ + POSTGRES_DB=postgres \ + PGDATA=/var/lib/postgresql/data + +# Pre-initialize the database at build time and apply speed-optimized +# configuration. This eliminates the ~1-2s initdb cost on every container start. +RUN set -eux; \ + mkdir -p "$PGDATA"; \ + chown postgres:postgres "$PGDATA"; \ + echo "$POSTGRES_PASSWORD" > /tmp/pwfile; \ + gosu postgres initdb \ + --username="$POSTGRES_USER" \ + --pwfile=/tmp/pwfile \ + -D "$PGDATA"; \ + rm /tmp/pwfile; \ + \ + # --- Performance settings (unsafe for production, ideal for tests) --- \ + { \ + echo ""; \ + echo "# === sqlc test optimizations ==="; \ + echo "listen_addresses = '*'"; \ + echo "fsync = off"; \ + echo "synchronous_commit = off"; \ + echo "full_page_writes = off"; \ + echo "max_connections = 200"; \ + echo "shared_buffers = 128MB"; \ + echo "wal_level = minimal"; \ + echo "max_wal_senders = 0"; \ + echo "max_wal_size = 256MB"; \ + echo "checkpoint_timeout = 30min"; \ + echo "log_min_messages = FATAL"; \ + echo "log_statement = none"; \ + } >> "$PGDATA/postgresql.conf"; \ + \ + # --- Allow password auth from any host --- \ + echo "host all all 0.0.0.0/0 scram-sha-256" >> "$PGDATA/pg_hba.conf"; \ + echo "host all all ::/0 scram-sha-256" >> "$PGDATA/pg_hba.conf" + +EXPOSE 5432 + +# SIGINT = "fast shutdown": disconnects clients and exits cleanly without +# requiring crash recovery on the next start. Much faster than the default +# SIGTERM ("smart shutdown") which waits for all clients to disconnect. +STOPSIGNAL SIGINT + +USER postgres + +# Start postgres directly, bypassing docker-entrypoint.sh entirely. +CMD ["postgres"] diff --git a/cmd/sqlc-test-setup/main.go b/cmd/sqlc-test-setup/main.go index 3a816f4502..5827a7bc46 100644 --- a/cmd/sqlc-test-setup/main.go +++ b/cmd/sqlc-test-setup/main.go @@ -266,6 +266,10 @@ func startPostgreSQL() error { return fmt.Errorf("configuring pg_hba.conf: %w", err) } + if err := applyFastSettings(); err != nil { + return fmt.Errorf("applying fast postgresql settings: %w", err) + } + log.Println("reloading postgresql configuration") if err := run("sudo", "service", "postgresql", "reload"); err != nil { return fmt.Errorf("reloading postgresql: %w", err) @@ -313,6 +317,78 @@ func ensurePgHBAEntry(hbaPath string) error { return run("bash", "-c", cmd) } +// pgFastSettings are PostgreSQL settings that sacrifice durability for speed. +// They are unsafe for production but ideal for test databases. +var pgFastSettings = map[string]string{ + "fsync": "off", + "synchronous_commit": "off", + "full_page_writes": "off", + "max_connections": "200", + "wal_level": "minimal", + "max_wal_senders": "0", + "max_wal_size": "256MB", + "checkpoint_timeout": "30min", + "log_min_messages": "FATAL", + "log_statement": "none", +} + +// detectPgConfigPath returns the path to postgresql.conf by querying the running server. +func detectPgConfigPath() (string, error) { + out, err := runOutput("bash", "-c", "sudo -u postgres psql -t -c 'SHOW config_file;'") + if err != nil { + return "", fmt.Errorf("querying config_file: %w (output: %s)", err, out) + } + path := strings.TrimSpace(out) + if path == "" { + return "", fmt.Errorf("postgresql.conf path is empty") + } + log.Printf("found postgresql.conf at %s", path) + return path, nil +} + +// applyFastSettings detects postgresql.conf and appends speed-optimized settings +// for test workloads. Settings that are already present are skipped. A restart +// is needed for some settings (e.g. wal_level, max_connections) to take effect. +func applyFastSettings() error { + confPath, err := detectPgConfigPath() + if err != nil { + return err + } + + out, err := runOutput("sudo", "cat", confPath) + if err != nil { + return fmt.Errorf("reading postgresql.conf: %w", err) + } + + // Check if we've already applied settings by looking for our marker comment. + if strings.Contains(out, "# sqlc test optimizations") { + log.Println("fast postgresql settings already applied, skipping") + return nil + } + + log.Println("applying fast postgresql settings for test workloads") + + var block strings.Builder + block.WriteString("\n# sqlc test optimizations\n") + for key, value := range pgFastSettings { + block.WriteString(fmt.Sprintf("%s = %s\n", key, value)) + } + + cmd := fmt.Sprintf("echo '%s' | sudo tee -a %s", block.String(), confPath) + if err := run("bash", "-c", cmd); err != nil { + return fmt.Errorf("appending settings to postgresql.conf: %w", err) + } + + // Some settings (wal_level, max_connections, shared_buffers) require a + // full restart rather than just a reload. + log.Println("restarting postgresql for settings that require a restart") + if err := run("sudo", "service", "postgresql", "restart"); err != nil { + return fmt.Errorf("restarting postgresql: %w", err) + } + + return nil +} + func startMySQL() error { log.Println("--- Starting MySQL ---") diff --git a/docker-compose.yml b/docker-compose.yml index f318d1ed93..dbd9e82075 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,11 +11,9 @@ services: MYSQL_ROOT_HOST: '%' postgresql: - image: "postgres:16" + build: + context: . + dockerfile: Dockerfile.postgres ports: - "5432:5432" restart: always - environment: - POSTGRES_DB: postgres - POSTGRES_PASSWORD: mysecretpassword - POSTGRES_USER: postgres diff --git a/internal/sqltest/docker/postgres.go b/internal/sqltest/docker/postgres.go index 1b2d842c70..240c2c1a46 100644 --- a/internal/sqltest/docker/postgres.go +++ b/internal/sqltest/docker/postgres.go @@ -1,10 +1,13 @@ package docker import ( + "bytes" "context" "fmt" "log/slog" + "os" "os/exec" + "path/filepath" "strings" "time" @@ -13,6 +16,8 @@ import ( var postgresHost string +const postgresImageName = "sqlc-postgres" + func StartPostgreSQLServer(c context.Context) (string, error) { if err := Installed(); err != nil { return "", err @@ -38,11 +43,57 @@ func StartPostgreSQLServer(c context.Context) (string, error) { return data, nil } +// findRepoRoot walks up from the current directory to find the directory +// containing go.mod, which is the repository root. +func findRepoRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("could not find repo root (go.mod)") + } + dir = parent + } +} + +// buildPostgresImage builds the fast-startup PostgreSQL image from +// Dockerfile.postgres. The Dockerfile requires no build context, so we +// pipe it to `docker build -` to avoid sending the repo tree to the daemon. +func buildPostgresImage() error { + root, err := findRepoRoot() + if err != nil { + return err + } + content, err := os.ReadFile(filepath.Join(root, "Dockerfile.postgres")) + if err != nil { + return fmt.Errorf("read Dockerfile.postgres: %w", err) + } + cmd := exec.Command("docker", "build", "-t", postgresImageName, "-") + cmd.Stdin = bytes.NewReader(content) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("docker build sqlc-postgres: %w\n%s", err, output) + } + return nil +} + +// postgresImageExists checks whether the sqlc-postgres image is already built. +func postgresImageExists() bool { + cmd := exec.Command("docker", "image", "inspect", postgresImageName) + return cmd.Run() == nil +} + func startPostgreSQLServer(c context.Context) (string, error) { - { - _, err := exec.Command("docker", "pull", "postgres:16").CombinedOutput() - if err != nil { - return "", fmt.Errorf("docker pull: postgres:16 %w", err) + // Build the fast-startup image if it doesn't already exist. + if !postgresImageExists() { + if err := buildPostgresImage(); err != nil { + return "", err } } @@ -56,14 +107,13 @@ func startPostgreSQLServer(c context.Context) (string, error) { } if !exists { + // The sqlc-postgres image is pre-initialized and pre-configured, + // so no environment variables or extra flags are needed. cmd := exec.Command("docker", "run", "--name", "sqlc_sqltest_docker_postgres", - "-e", "POSTGRES_PASSWORD=mysecretpassword", - "-e", "POSTGRES_USER=postgres", "-p", "5432:5432", "-d", - "postgres:16", - "-c", "max_connections=200", + postgresImageName, ) output, err := cmd.CombinedOutput() @@ -88,7 +138,6 @@ func startPostgreSQLServer(c context.Context) (string, error) { return "", fmt.Errorf("timeout reached: %w", ctx.Err()) case <-ticker.C: - // Run your function here conn, err := pgx.Connect(ctx, uri) if err != nil { slog.Debug("sqltest", "connect", err) diff --git a/internal/sqltest/native/postgres.go b/internal/sqltest/native/postgres.go index f805a40a1c..b46c380b4f 100644 --- a/internal/sqltest/native/postgres.go +++ b/internal/sqltest/native/postgres.go @@ -114,6 +114,21 @@ func startPostgresService() error { return nil } +// pgFastSettings are PostgreSQL settings that sacrifice durability for speed. +// They are unsafe for production but ideal for test databases. +var pgFastSettings = [][2]string{ + {"fsync", "off"}, + {"synchronous_commit", "off"}, + {"full_page_writes", "off"}, + {"max_connections", "200"}, + {"wal_level", "minimal"}, + {"max_wal_senders", "0"}, + {"max_wal_size", "256MB"}, + {"checkpoint_timeout", "30min"}, + {"log_min_messages", "FATAL"}, + {"log_statement", "none"}, +} + func configurePostgres() error { // Set password for postgres user using sudo -u postgres cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER USER postgres PASSWORD 'postgres';") @@ -162,9 +177,86 @@ func configurePostgres() error { } } + // Apply speed-optimized settings for test workloads + if err := applyFastSettings(); err != nil { + slog.Warn("native/postgres", "fast-settings-error", err) + } + return nil } +// applyFastSettings appends speed-optimized settings to postgresql.conf for +// test workloads. Settings that sacrifice durability for speed (fsync=off, etc.) +// are applied once and require a restart to take effect. +func applyFastSettings() error { + output, err := exec.Command("sudo", "-u", "postgres", "psql", "-t", "-c", "SHOW config_file;").CombinedOutput() + if err != nil { + return fmt.Errorf("could not find config_file: %w", err) + } + + confFile := strings.TrimSpace(string(output)) + if confFile == "" { + return fmt.Errorf("empty config_file path") + } + + catOutput, err := exec.Command("sudo", "cat", confFile).CombinedOutput() + if err != nil { + return fmt.Errorf("could not read %s: %w", confFile, err) + } + + // Check if we've already applied settings. + if strings.Contains(string(catOutput), "# sqlc test optimizations") { + slog.Debug("native/postgres", "fast-settings", "already applied") + return nil + } + + slog.Info("native/postgres", "status", "applying fast settings to postgresql.conf") + + var block strings.Builder + block.WriteString("\n# sqlc test optimizations\n") + for _, kv := range pgFastSettings { + fmt.Fprintf(&block, "%s = %s\n", kv[0], kv[1]) + } + + cmd := exec.Command("sudo", "bash", "-c", + fmt.Sprintf("echo '%s' | sudo tee -a %s", block.String(), confFile)) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("appending settings to postgresql.conf: %w\n%s", err, out) + } + + // Some settings (wal_level, max_connections) require a full restart. + slog.Info("native/postgres", "status", "restarting postgresql for fast settings") + if err := restartPostgres(); err != nil { + return fmt.Errorf("restart for fast settings: %w", err) + } + + return nil +} + +func restartPostgres() error { + // Try systemctl restart + cmd := exec.Command("sudo", "systemctl", "restart", "postgresql") + if err := cmd.Run(); err == nil { + return nil + } + + // Try service restart + cmd = exec.Command("sudo", "service", "postgresql", "restart") + if err := cmd.Run(); err == nil { + return nil + } + + // Try pg_ctlcluster restart + output, _ := exec.Command("ls", "/etc/postgresql/").CombinedOutput() + versions := strings.Fields(string(output)) + if len(versions) > 0 { + cmd = exec.Command("sudo", "pg_ctlcluster", versions[0], "main", "restart") + return cmd.Run() + } + + return fmt.Errorf("could not restart PostgreSQL") +} + func reloadPostgres() error { // Try systemctl reload cmd := exec.Command("sudo", "systemctl", "reload", "postgresql")