TEZ-4682: [Cloud] Tez AM docker image

.gitignore

@@ -10,3 +10,4 @@
 .settings
 pom.xml.versionsBackup
 target
+tez-dist/src/docker/cache/

tez-dag/src/main/resources/tez-container-log4j.properties

@@ -21,9 +21,16 @@ log4j.rootLogger=${tez.root.logger}
 log4j.threshold=ALL
 
 #
-# ContainerLog Appender
+# Console Appender
 #
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.out
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} [%p] [%t] |%c{2}|: %m%n
 
+#
+# ContainerLog Appender
+#
 log4j.appender.CLA=org.apache.tez.common.TezContainerLogAppender
 log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir}
 

tez-dist/pom.xml

@@ -118,6 +118,34 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>docker</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>build-docker-image</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>/bin/bash</executable>
+                  <arguments>
+                    <argument>${project.basedir}/src/docker/tez-am/build-tez-am-docker.sh</argument>
+                    <argument>-tez ${project.version}</argument>
+                    <argument>-repo apache</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
   <build>

tez-dist/src/docker/tez-am/Dockerfile.tez_am

@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG BUILD_ENV=unarchive
+
+# hadolint ignore=DL3006
+FROM ubuntu AS unarchive
+# hadolint ignore=DL3010
+ONBUILD COPY tez-*.tar.gz /opt
+
+# hadolint ignore=DL3006
+FROM ${BUILD_ENV} AS env
+ARG TEZ_VERSION
+
+RUN mkdir -p /opt/tez \
+    && tar -xzv \
+        -f /opt/tez-$TEZ_VERSION.tar.gz \
+        -C /opt/tez \
+    && rm -rf /opt/tez-$TEZ_VERSION.tar.gz
+
+FROM eclipse-temurin:21-jdk-ubi9-minimal AS run
+
+ARG UID=1000
+ARG TEZ_VERSION
+
+# Install dependencies
+# hadolint ignore=DL3041
+RUN set -ex; \
+    microdnf update -y; \
+    microdnf -y install procps gettext findutils hostname; \
+    microdnf clean all; \
+    useradd --no-create-home -s /sbin/nologin -c "" --uid $UID tez
+
+# Set necessary environment variables
+ENV TEZ_HOME=/opt/tez \
+    TEZ_CONF_DIR=/opt/tez/conf
+
+ENV TEZ_CLIENT_VERSION=$TEZ_VERSION
+
+ENV PATH=$TEZ_HOME/bin:$PATH
+
+COPY --from=env --chown=tez /opt/tez $TEZ_HOME
+
+RUN mkdir -p $TEZ_CONF_DIR && chown tez:tez $TEZ_CONF_DIR
+
+COPY --chown=tez tez-am-entrypoint.sh /
+COPY --chown=tez conf $TEZ_CONF_DIR
+
+# Create Extension Point Directory
+RUN mkdir -p /opt/tez/plugins && chown tez:tez /opt/tez/plugins && chmod 755 /opt/tez/plugins
+
+RUN chmod +x /tez-am-entrypoint.sh
+
+USER tez
+WORKDIR $TEZ_HOME
+
+ENTRYPOINT ["/tez-am-entrypoint.sh"]

tez-dist/src/docker/tez-am/README.md

@@ -0,0 +1,124 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+# Tez AM Docker
+
+1. Building the docker image:
+
+    ```bash
+    mvn clean install -DskipTests -Pdocker
+    ```
+
+2. Install zookeeper in mac:
+
+    a. Via brew: set the `tez.am.zookeeper.quorum` value as
+    `host.docker.internal:2181` in `tez-site.xml`
+
+    ```bash
+    brew install zookeeper
+    zkServer start
+    ```
+
+    b. Use Zookeeper docker image (Refer to docker compose yml):
+
+    ```bash
+    docker pull zookeeper:3.8.4
+
+    docker run -d \
+        --name zookeeper-server \
+        -p 2181:2181 \
+        -p 8080:8080 \
+        -e ZOO_MY_ID=1 \
+        zookeeper:3.8.4
+    ```
+
+3. Running the Tez AM container explicitly:
+
+    ```bash
+    export TEZ_VERSION=1.0.0-SNAPSHOT
+
+    docker run --rm \
+        -p 10001:10001 \
+        --env-file tez-dist/src/docker/tez-am/tez-am.env \
+        --name tez-am \
+        --hostname localhost \
+        apache/tez-am:$TEZ_VERSION
+    ```
+
+   * `TEZ_VERSION` corresponds to the Maven `${project.version}`.
+       Set this environment variable in your shell before running the commands.
+   * Expose ports using the `-p` flag based on the
+     `tez.am.client.am.port-range` property in `tez-site.xml`.
+   * The `--hostname` flag configures the container's hostname, allowing
+      services on the host (e.g., macOS) to connect to it.
+   * Ensure the `--env-file` flag is included, or at a minimum, pass
+     `-e TEZ_FRAMEWORK_MODE=STANDALONE_ZOOKEEPER` to the `docker run` command.
+
+4. Debugging the Tez AM container:
+Uncomment the `JAVA_TOOL_OPTIONS` in `tez-am.env` and expose 5005 port
+using `-p` flag
+
+    ```bash
+    docker run --rm \
+        -p 10001:10001 -p 5005:5005 \
+        --env-file tez-dist/src/docker/tez-am/tez-am.env \
+        --name tez-am \
+        --hostname localhost \
+        apache/tez-am:$TEZ_VERSION
+    ```
+
+5. To override the tez-site.xml in docker image use:
+   * Set the `TEZ_CUSTOM_CONF_DIR` environment variable in `tez-am.env`
+      or via the `docker run` command (e.g., `/opt/tez/custom-conf`).
+
+    ```bash
+    export TEZ_SITE_PATH=$(pwd)/tez-dist/src/docker/conf/tez-site.xml
+
+    docker run --rm \
+    -p 10001:10001 \
+    --env-file tez-dist/src/docker/tez-am/tez-am.env \
+    -v "$TEZ_SITE_PATH:/opt/tez/custom-conf/tez-site.xml" \
+    --name tez-am \
+    --hostname localhost \
+    apache/tez-am:$TEZ_VERSION
+    ```
+
+6. To add plugin jars in docker image use:
+   * The plugin directory path inside the Docker container is fixed at `/opt/tez/plugins`.
+
+    ```bash
+    docker run --rm \
+    -p 10001:10001 \
+    --env-file tez-dist/src/docker/tez-am/tez-am.env \
+    -v "/path/to/your/local/plugins:/opt/tez/plugins" \
+    --name tez-am \
+    --hostname localhost \
+    apache/tez-am:$TEZ_VERSION
+    ```
+
+7. Using Docker Compose:
+    * Refer to the `docker-compose.yml` file in this directory for
+    an example of how to run both the Tez AM and Zookeeper containers
+    together using Docker Compose.
+
+     ```bash
+     docker-compose -f tez-dist/src/docker/tez-am/docker-compose.yml up -d --build
+     ```
+
+    * This command will start both the Tez AM, Zookeeper, Minimal
+    Hadoop containers as defined in the `docker-compose.yml` file.

tez-dist/src/docker/tez-am/build-tez-am-docker.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -xeou pipefail
+
+TEZ_VERSION=
+REPO=
+
+usage() {
+  cat <<EOF 1>&2
+Usage: $0 [-h] [-tez <Tez version>] [-repo <Docker repo>]
+Build the Apache Tez AM Docker image
+-help                Display help
+-tez                 Build image with the specified Tez version
+-repo                Docker repository
+EOF
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+  -h)
+    usage
+    exit 0
+    ;;
+  -tez)
+    shift
+    TEZ_VERSION=$1
+    shift
+    ;;
+  -repo)
+    shift
+    REPO=$1
+    shift
+    ;;
+  *)
+    shift
+    ;;
+  esac
+done
+
+SCRIPT_DIR=$(
+  cd "$(dirname "$0")"
+  pwd
+)
+
+DIST_DIR=${DIST_DIR:-"$SCRIPT_DIR/../../.."}
+PROJECT_ROOT=${PROJECT_ROOT:-"$SCRIPT_DIR/../../../.."}
+
+REPO=${REPO:-apache}
+WORK_DIR="$(mktemp -d)"
+CACHE_DIR="$SCRIPT_DIR/cache"
+mkdir -p "$CACHE_DIR"
+
+# Defaults Tez versions from pom.xml if not provided
+TEZ_VERSION=${TEZ_VERSION:-$(mvn -f "$PROJECT_ROOT/pom.xml" -q help:evaluate -Dexpression=project.version -DforceStdout)}
+
+#####################################
+# Pick tez tarball from local build #
+#####################################
+TEZ_FILE_NAME="tez-$TEZ_VERSION.tar.gz"
+LOCAL_DIST_PATH="$DIST_DIR/target/$TEZ_FILE_NAME"
+
+if [ -f "$LOCAL_DIST_PATH" ]; then
+  echo "--> Found local Tez build artifact at: $LOCAL_DIST_PATH"
+  cp "$LOCAL_DIST_PATH" "$WORK_DIR/"
+else
+  echo "--> Error: Local Tez artifact not found at $LOCAL_DIST_PATH"
+  echo "--> Please build the project first (e.g., mvn clean install -DskipTests)."
+  exit 1
+fi
+
+# -------------------------------------------------------------------------
+# BUILD CONTEXT PREPARATION
+# -------------------------------------------------------------------------
+cp -R "$SCRIPT_DIR/conf" "$WORK_DIR/" 2>/dev/null || mkdir -p "$WORK_DIR/conf"
+cp "$SCRIPT_DIR/tez-am-entrypoint.sh" "$WORK_DIR/"
+cp "$SCRIPT_DIR/Dockerfile.tez_am" "$WORK_DIR/"
+
+echo "Building Docker image..."
+docker build \
+  "$WORK_DIR" \
+  -f "$WORK_DIR/Dockerfile.tez_am" \
+  -t "$REPO/tez-am:$TEZ_VERSION" \
+  --build-arg "BUILD_ENV=unarchive" \
+  --build-arg "TEZ_VERSION=$TEZ_VERSION"
+
+rm -r "${WORK_DIR}"
+echo "Docker image $REPO/tez-am:$TEZ_VERSION built successfully."