CASSANDRA-20997: Upgrade testing docker image to Ubuntu 22.04 and add Python 3.12/3.13 support

cassandra-release/finish_release.sh

@@ -282,6 +282,6 @@ echo ' 7) update #cassandra topic on slack'
 echo ' 8) tweet from @cassandra'
 echo ' 9) release version in JIRA'
 echo ' 10) remove old version (eg: `svn rm https://dist.apache.org/repos/dist/release/cassandra/<previous_version>`)'
-echo ' 11) increment build.xml (base.version), CHANGES.txt, and  ubuntu2004_test.docker (ccm installed) for the next release'
+echo ' 11) increment build.xml (base.version), CHANGES.txt, and  ubuntu_test.docker (ccm installed) for the next release'
 echo ' 12) Add release in https://reporter.apache.org/addrelease.html?cassandra (same as instructions in email you will receive from the \"Apache Reporter Service\")'
 echo ' 13) update current_ version in cassandra-dtest/upgrade_tests/upgrade_manifest.py'

contribulyze/contribulyze.sh

@@ -4,7 +4,7 @@
 #
 # Example run in docker (from the cassandra-builds/contribulyze directory)
 #
-# docker run -t -v`pwd`/build/html:/tmp/contribulyze-html -v`pwd`:/contribulyze apache/cassandra-testing-ubuntu2004-java11-w-dependencies bash -lc 'pip3 install --quiet python-dateutil ; cd /contribulyze ; bash contribulyze.sh '
+# docker run -t -v`pwd`/build/html:/tmp/contribulyze-html -v`pwd`:/contribulyze apache/cassandra-testing-ubuntu-java11-w-dependencies bash -lc 'pip3 install --quiet python-dateutil ; cd /contribulyze ; bash contribulyze.sh '
 #
 
 set -e

docker/testing/README.md

@@ -20,11 +20,11 @@ Build images from the parent directory using the following commands. Change tag
 
 Base image:
 
-    docker buildx build --platform linux/amd64,linux/arm64 -t apache/cassandra-testing-ubuntu2004-java11:$(date +"%Y%m%d") -t apache/cassandra-testing-ubuntu2004-java11:latest -f ubuntu2004_j11.docker --provenance=true --sbom=true --push .
+    docker buildx build --platform linux/amd64,linux/arm64 -t apache/cassandra-testing-ubuntu-java11:$(date +"%Y%m%d") -t apache/cassandra-testing-ubuntu-java11:latest -f ubuntu_j11.docker --provenance=true --sbom=true --push .
 
 Caching image:
 
-    docker buildx build --platform linux/amd64,linux/arm64 --no-cache -t apache/cassandra-testing-ubuntu2004-java11-w-dependencies:$(date +"%Y%m%d") -t apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest -f ubuntu2004_j11_w_dependencies.docker --provenance=true --sbom=true --push .
+    docker buildx build --platform linux/amd64,linux/arm64 --no-cache -t apache/cassandra-testing-ubuntu-java11-w-dependencies:$(date +"%Y%m%d") -t apache/cassandra-testing-ubuntu-java11-w-dependencies:latest -f ubuntu_j11_w_dependencies.docker --provenance=true --sbom=true --push .
 
 Please make sure to always tag also by date, so we can go back to that version in case anything breaks after the next update!
 

docker/testing/ubuntu2004_j11.docker → docker/testing/ubuntu_j11.docker

@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 LABEL org.opencontainers.image.authors="Apache Cassandra <dev@cassandra.apache.org>"
 
 # configure apt to retry downloads
@@ -19,22 +19,49 @@ RUN echo 'Acquire::http::Timeout "60";' > /etc/apt/apt.conf.d/80proxy.conf
 RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf
 
 # install our python dependencies and some other stuff we need
-# libev4 libev-dev are for the python driver / libssl-dev is for python3.6
-
+# libev4 libev-dev are for the python driver / libssl-dev is for python3.6 and modern pythons
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install -y --no-install-recommends software-properties-common apt-utils vim && \
-    apt-get install -y lsof
+    apt-get install -y --no-install-recommends software-properties-common apt-utils vim gnupg && \
+    apt-get install -y lsof build-essential zlib1g-dev libncurses5-dev libgdbm-dev \
+                       libnss3-dev libssl-dev libreadline-dev libffi-dev libsqlite3-dev \
+                       wget libbz2-dev gcc-9 g++-9
 
+# 1. Compile OpenSSL 1.1.1w from source using GCC-9 (Required for Python 3.6 on Jammy)
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    wget https://www.openssl.org/source/openssl-1.1.1w.tar.gz && \
+    tar xzf openssl-1.1.1w.tar.gz && \
+    cd openssl-1.1.1w && \
+    CC=gcc-9 CXX=g++-9 ./config shared --prefix=/opt/openssl-1.1 --openssldir=/opt/openssl-1.1 && \
+    make -j$(nproc) && make install && \
+    cd .. && rm -rf openssl-1.1.1w*
+
+# 2. Compile Python 3.6.15 from source using custom OpenSSL and Glibc patch
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    wget https://www.python.org/ftp/python/3.6.15/Python-3.6.15.tgz && \
+    tar xzf Python-3.6.15.tgz && \
+    cd Python-3.6.15 && \
+    sed -i 's/MINSIGSTKSZ/2048/g' Modules/faulthandler.c && \
+    sed -i 's/SIGSTKSZ/8192/g' Modules/faulthandler.c && \
+    CC=gcc-9 CXX=g++-9 \
+    LDFLAGS="-Wl,-rpath=/opt/openssl-1.1/lib -L/opt/openssl-1.1/lib" \
+    CPPFLAGS="-I/opt/openssl-1.1/include" \
+    ./configure --with-openssl=/opt/openssl-1.1 --prefix=/usr && \
+    make -j$(nproc) altinstall && \
+    cd .. && rm -rf Python-3.6.15*
+
+# Install other Pythons via deadsnakes
 RUN export DEBIAN_FRONTEND=noninteractive && \
     add-apt-repository -y ppa:deadsnakes/ppa && \
     apt-get update && \
-    apt-get install -y curl git-core python2.7 python-dev python3-pip \
-        python3.6 python3.6-venv python3.6-dev \
+    apt-get install -y curl git-core python2.7 python2.7-dev python3-pip \
         python3.7 python3.7-venv python3.7-dev \
         python3.8 python3.8-venv python3.8-dev \
+        python3.10 python3.10-venv python3.10-dev \
         python3.11 python3.11-venv python3.11-dev \
-        virtualenv net-tools libev4 libev-dev wget gcc libssl-dev libxml2 libxslt1-dev
+        python3.12 python3.12-venv python3.12-dev \
+        python3.13 python3.13-venv python3.13-dev \
+        virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev
 
 # install python2-pip
 RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py
@@ -45,14 +72,21 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 3
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 4
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 5
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 5
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 6
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.12 7
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.13 8
+
 RUN python2.7 -m pip install --upgrade pip
-RUN python3.6 -m pip install --upgrade pip
+RUN curl -sS https://bootstrap.pypa.io/pip/3.6/get-pip.py | python3.6
 RUN python3.7 -m pip install --upgrade pip
 RUN python3.8 -m pip install --upgrade pip
 
-# 3.11 needs newer pip
+# Newer Pythons need newer pip
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
 RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13
 
 # solves warning: "jemalloc shared library could not be preloaded to speed up memory allocations"
 RUN export DEBIAN_FRONTEND=noninteractive && \
@@ -97,7 +131,7 @@ RUN find /etc -type f -name java.security -exec sed -i 's/3DES_EDE_CBC$/3DES_EDE
 # setup our user -- if we don't do this docker will default to root and Cassandra will fail to start
 # as we appear to have a check that the user isn't starting Cassandra as root
 RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get install sudo && \
+    apt-get install -y sudo && \
     adduser --disabled-password --gecos "" cassandra && \
     echo "cassandra ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/cassandra && \
     chmod 0440 /etc/sudoers.d/cassandra
@@ -126,7 +160,7 @@ ENV PATH="$PATH:/usr/local/go/bin"
 
 # switch to the cassandra user... we are all done running things as root
 USER cassandra
-ENV HOME /home/cassandra
+ENV HOME=/home/cassandra
 WORKDIR /home/cassandra
 
 # Add environment variables for Ant and Java and add them to the PATH
@@ -139,7 +173,7 @@ RUN echo 'export ANT_HOME=/usr/share/ant' >> /home/cassandra/.bashrc && \
 ENV ANT_HOME=/usr/share/ant
 
 # run pip commands and setup virtualenv (note we do this after we switch to cassandra user so we
-# setup the virtualenv for the cassandra user and not the root user by accident) for Python 3.6/3.7/3.8
+# setup the virtualenv for the cassandra user and not the root user by accident) for Python 3.6/3.7/3.8/3.10/3.11/3.12/3.13
 # Don't build cython extensions when installing cassandra-driver. During test execution the driver
 # dependency is refreshed via pip install --upgrade, so that driver changes can be pulled in without
 # requiring the image to be rebuilt. Rebuilding compiled extensions is costly and is disabled by
@@ -163,9 +197,31 @@ RUN virtualenv --python=python3.8 env3.8
 RUN chmod +x env3.8/bin/activate
 RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 && source ~/env3.8/bin/activate && pip3 install --upgrade pip && pip3 install -r /opt/requirements.txt && pip3 freeze --user"
 
+RUN virtualenv --python=python3.10 env3.10
+RUN chmod +x env3.10/bin/activate
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 && source ~/env3.10/bin/activate && pip3 install \"pip<25.0\" \"setuptools==60.8.2\" wheel && pip3 install --no-build-isolation -r /opt/requirements.txt && pip3 freeze --user"
+
 RUN virtualenv --python=python3.11 env3.11
 RUN chmod +x env3.11/bin/activate
-RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 && source ~/env3.11/bin/activate && pip3 install --upgrade pip && pip3 install -r /opt/requirements.txt && pip3 freeze --user"
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 && source ~/env3.11/bin/activate && pip3 install \"pip<25.0\" \"setuptools==60.8.2\" wheel && pip3 install --no-build-isolation -r /opt/requirements.txt && pip3 freeze --user"
+
+RUN virtualenv --python=python3.12 env3.12
+RUN chmod +x env3.12/bin/activate
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 && \
+    source ~/env3.12/bin/activate && \
+    curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 - \"pip<25.0\" \"setuptools==60.8.2\" wheel && \
+    sed -i 's/pkgutil.ImpImporter/type(\"ImpImporter\", (object,), {})/g' ~/env3.12/lib/python3.12/site-packages/pkg_resources/__init__.py && \
+    pip3 install --no-build-isolation -r /opt/requirements.txt && \
+    pip3 freeze --user"
+
+RUN virtualenv --python=python3.13 env3.13
+RUN chmod +x env3.13/bin/activate
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 && \
+    source ~/env3.13/bin/activate && \
+    curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 - \"pip<25.0\" \"setuptools==60.8.2\" wheel && \
+    sed -i 's/pkgutil.ImpImporter/type(\"ImpImporter\", (object,), {})/g' ~/env3.13/lib/python3.13/site-packages/pkg_resources/__init__.py && \
+    pip3 install --no-build-isolation -r /opt/requirements.txt && \
+    pip3 freeze --user"
 
 # we need to make SSH less strict to prevent various dtests from failing when they attempt to
 # git clone a given commit/tag/etc

docker/testing/ubuntu2004_j11_w_dependencies.docker → docker/testing/ubuntu_j11_w_dependencies.docker

@@ -11,7 +11,7 @@
 # limitations under the License.
 
 # base things off the testing image without dependencies warmed up
-FROM apache/cassandra-testing-ubuntu2004-java11
+FROM apache/cassandra-testing-ubuntu-java11
 LABEL org.opencontainers.image.authors="Apache Cassandra <dev@cassandra.apache.org>"
 
 USER cassandra

jenkins-dsl/cassandra_job_dsl_seed.groovy

@@ -40,11 +40,11 @@ def cassandraBranches = ['cassandra-5.0', 'trunk']
 def legacyCassandraBranches = ['cassandra-2.2', 'cassandra-3.0', 'cassandra-3.11', 'cassandra-4.0', 'cassandra-4.1']
 // Ant test targets
 def testTargets = ['test', 'test-burn', 'test-cdc', 'test-compression', 'stress-test', 'fqltool-test', 'long-test', 'jvm-dtest', 'jvm-dtest-upgrade', 'microbench']
-def testDockerImage = 'apache/cassandra-testing-ubuntu2004-java11-w-dependencies'
+def testDockerImage = 'apache/cassandra-testing-ubuntu-java11-w-dependencies'
 
 // Dtest test targets
 def dtestTargets = ['dtest', 'dtest-novnode', 'dtest-offheap', 'dtest-large', 'dtest-large-novnode', 'dtest-upgrade']
-def dtestDockerImage = 'apache/cassandra-testing-ubuntu2004-java11'
+def dtestDockerImage = 'apache/cassandra-testing-ubuntu-java11'
 
 // expected longest job runtime
 def maxJobHours = 12
@@ -1454,7 +1454,7 @@ job('contribulyze') {
         buildDescription('', buildDescStr)
         shell("""
                 mkdir -p build/html ; chmod -R 777 build/html
-                docker run -t -v`pwd`/build/html:/tmp/contribulyze-html -v`pwd`/contribulyze:/contribulyze apache/cassandra-testing-ubuntu2004-java11-w-dependencies bash -lc 'pip3 install --quiet python-dateutil ; cd /contribulyze ; bash contribulyze.sh '
+                docker run -t -v`pwd`/build/html:/tmp/contribulyze-html -v`pwd`/contribulyze:/contribulyze apache/cassandra-testing-ubuntu-java11-w-dependencies bash -lc 'pip3 install --quiet python-dateutil ; cd /contribulyze ; bash contribulyze.sh '
               """)
     }
     publishers {