diff --git a/Dockerfile.builder b/Dockerfile.builder
index cb8eaa9..587adb4 100644
--- a/Dockerfile.builder
+++ b/Dockerfile.builder
@@ -22,6 +22,9 @@ RUN    apt-get update \
 
 ARG REPO_ROOT
 
+ARG UID=1000
+RUN useradd -u ${UID} builder
+
 COPY cmake   /${REPO_ROOT}/cmake
 COPY scripts /${REPO_ROOT}/scripts
 COPY config  /${REPO_ROOT}
diff --git a/README.md b/README.md
index 03e556b..7444ce8 100644
--- a/README.md
+++ b/README.md
@@ -59,6 +59,12 @@ Then build the toolchain by running
 
 The build artifact is `./output/llvm-pauth.squashfs` file.
 
+The build scripts try to detect the UID of the real user running `./build.sh`
+in case this script is executed with `sudo` to make the contents of `./output`
+and `./ccache` directories writable without `sudo` by the host user. This is
+especially useful to prevent the `host-build` performed with the same `./ccache`
+directory from silently falling back to non-cached build.
+
 # Using the toolchain
 
 Mount the produced SquashFS image at `/opt/llvm-pauth`:
diff --git a/build.sh b/build.sh
index bd60348..d98f03d 100755
--- a/build.sh
+++ b/build.sh
@@ -95,10 +95,22 @@ build_in_docker() {
   check_repo_sha "$ROOT/src/llvm" "$LLVM_SHA"
   check_repo_sha "$ROOT/src/musl" "$MUSL_SHA"
 
+  # Try creating a non-privileged user inside the container with the same UID
+  # as the UID of the real user to ensure ./ccache and ./output are writable
+  # without sudo on the host - this is useful to make sure ccache does not
+  # silently fall back to non-cached rebuilds in the 'host-build' mode of build.sh.
+  local UID
+  if [ "x$SUDO_USER" != "x" ]; then
+    UID="$(id -u "$SUDO_USER")"
+  else
+    UID="$(id -u)"
+  fi
+
   $DOCKER_CMD build \
       -t "$DOCKER_IMAGE_NAME" \
       -f Dockerfile.builder \
       --build-arg REPO_ROOT="$REPO_ROOT" \
+      --build-arg UID="$UID" \
       "$ROOT"
   $DOCKER_CMD run -ti --rm \
       --volume "$ROOT/output:$OUTPUT_DIR:rw" \
diff --git a/scripts/build-in-docker.sh b/scripts/build-in-docker.sh
index fb76200..377c260 100755
--- a/scripts/build-in-docker.sh
+++ b/scripts/build-in-docker.sh
@@ -2,6 +2,10 @@
 set -xe
 cd "$(dirname "$0")"
 
+if [ "$1" != "no-switch-user" ]; then
+  exec su builder -c "$0 no-switch-user"
+fi
+
 # This script is an entry point inside the Docker container.
 # Its location is expected to be $REPO_ROOT/scripts/build-in-docker.sh.