From 5013d16cfb06e2954aaa54eb5171d057cc1bcc35 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Sun, 22 Feb 2026 20:00:26 -0500 Subject: [PATCH] Migrate to Django 5 storages --- .ci/assets/ci_constraints.txt | 3 +- .ci/scripts/check_release.py | 11 +++- .github/workflows/ci.yml | 8 +-- .github/workflows/scripts/install.sh | 4 +- pulp_python/app/models.py | 4 +- pulp_python/app/utils.py | 5 +- template_config.yml | 88 ++++++++++++++++------------ 7 files changed, 72 insertions(+), 51 deletions(-) diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index c8070aff..c9198f19 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -1,5 +1,6 @@ # Pulpcore versions without the openapi command do no longer work in the CI -pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* +# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70 +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.* tablib!=3.6.0 diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 81bae1fd..86e250e4 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -1,11 +1,20 @@ #!/usr/bin/env python +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# "pyyaml>=6.0.3,<6.1.0", +# ] +# /// import argparse import re import os import tomllib -import yaml from pathlib import Path + +import yaml from packaging.version import Version from git import Repo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e9afb1f..8c4bcf91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,13 +94,13 @@ jobs: run_docs: ${{ needs.check-changes.outputs.run_docs }} lint: - needs: - - "check-changes" - if: needs.check-changes.outputs.run_tests == '1' uses: "./.github/workflows/lint.yml" build: - needs: "lint" + needs: + - "check-changes" + - "lint" + if: needs.check-changes.outputs.run_tests == '1' uses: "./.github/workflows/build.yml" test: diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 87a53dae..08594e38 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -95,7 +95,7 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": ""}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}}\ pulp_scenario_env: {}\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" @@ -109,7 +109,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": ""}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 7f69aa9b..1adcfc74 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -4,7 +4,6 @@ from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ObjectDoesNotExist from django.db import models -from django.conf import settings from pulpcore.plugin.models import ( Content, Publication, @@ -13,6 +12,7 @@ Repository, ) from pulpcore.plugin.responses import ArtifactResponse +from pulpcore.plugin.util import get_domain from pathlib import PurePath from .utils import ( @@ -77,7 +77,7 @@ def content_handler(self, path): elif len(path.parts) and path.parts[0] == "simple": # Temporary fix for PublishedMetadata not being properly served from remote storage # https://github.com/pulp/pulp_python/issues/413 - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if get_domain().storage_class != "pulpcore.app.models.storage.FileSystem": if self.publication or self.repository: try: publication = self.publication or Publication.objects.filter( diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index eb1082de..db6acdd0 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -3,7 +3,6 @@ import tempfile import json from collections import defaultdict -from django.core.files.storage import default_storage as storage from django.conf import settings from jinja2 import Template from packaging.utils import canonicalize_name @@ -144,8 +143,8 @@ def get_project_metadata_from_artifact(filename, artifact): # because pkginfo validates that the filename has a valid extension before # reading it with tempfile.NamedTemporaryFile('wb', dir=".", suffix=filename) as temp_file: - artifact_file = storage.open(artifact.file.name) - shutil.copyfileobj(artifact_file, temp_file) + artifact.file.seek(0) + shutil.copyfileobj(artifact.file, temp_file) temp_file.flush() metadata = DIST_TYPES[packagetype](temp_file.name) metadata.packagetype = packagetype diff --git a/template_config.yml b/template_config.yml index 33c3d396..80335113 100644 --- a/template_config.yml +++ b/template_config.yml @@ -2,18 +2,21 @@ # were not present before running plugin-template have been added with their default values. # generated with plugin_template +# +# After editing this file please always reapply the plugin template before committing any changes. -api_root: /pulp/ +--- +api_root: "/pulp/" black: false check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true -ci_base_image: ghcr.io/pulp/pulp-ci-centos9 +ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" ci_env: {} -ci_trigger: '{pull_request: {branches: [''*'']}}' -cli_package: pulp-cli -cli_repo: https://github.com/pulp/pulp-cli.git +ci_trigger: "{pull_request: {branches: ['*']}}" +cli_package: "pulp-cli" +cli_repo: "https://github.com/pulp/pulp-cli.git" core_import_allowed: [] deploy_client_to_pypi: true deploy_client_to_rubygems: true @@ -23,54 +26,62 @@ docker_fixtures: false extra_files: [] flake8: true flake8_ignore: [] -github_org: pulp +github_org: "pulp" latest_release_branch: null lint_requirements: true os_required_packages: [] parallel_test_workers: 8 -plugin_app_label: python -plugin_default_branch: main -plugin_name: pulp_python +plugin_app_label: "python" +plugin_default_branch: "main" +plugin_name: "pulp_python" plugins: -- app_label: python - name: pulp_python -post_job_template: null -pre_job_template: null + - app_label: "python" + name: "pulp_python" pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} pulp_env_s3: {} -pulp_scheme: https +pulp_scheme: "https" pulp_settings: - allowed_export_paths: /tmp - allowed_import_paths: /tmp + allowed_export_paths: "/tmp" + allowed_import_paths: "/tmp" orphan_protection_time: 0 - pypi_api_hostname: https://pulp:443 + pypi_api_hostname: "https://pulp:443" pulp_settings_azure: - AZURE_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== - AZURE_ACCOUNT_NAME: devstoreaccount1 - AZURE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; - AZURE_CONTAINER: pulp-test - AZURE_LOCATION: pulp3 - AZURE_OVERWRITE_FILES: true - AZURE_URL_EXPIRATION_SECS: 120 - DEFAULT_FILE_STORAGE: storages.backends.azure_storage.AzureStorage - MEDIA_ROOT: '' + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.azure_storage.AzureStorage" + OPTIONS: + account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + account_name: "devstoreaccount1" + azure_container: "pulp-test" + connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;" + expiration_secs: 120 + location: "pulp3" + overwrite_files: true + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" pulp_settings_gcp: null pulp_settings_s3: - AWS_ACCESS_KEY_ID: AKIAIT2Z5TDYPX3ARJBA - AWS_DEFAULT_ACL: '@none None' - AWS_S3_ADDRESSING_STYLE: path - AWS_S3_ENDPOINT_URL: http://minio:9000 - AWS_S3_REGION_NAME: eu-central-1 - AWS_S3_SIGNATURE_VERSION: s3v4 - AWS_SECRET_ACCESS_KEY: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS - AWS_STORAGE_BUCKET_NAME: pulp3 - DEFAULT_FILE_STORAGE: storages.backends.s3boto3.S3Boto3Storage - MEDIA_ROOT: '' + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.s3boto3.S3Boto3Storage" + OPTIONS: + access_key: "AKIAIT2Z5TDYPX3ARJBA" + addressing_style: "path" + bucket_name: "pulp3" + default_acl: "@none" + endpoint_url: "http://minio:9000" + region_name: "eu-central-1" + secret_key: "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS" + signature_version: "s3v4" + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" pydocstyle: true -release_email: pulp-infra@redhat.com -release_user: pulpbot +release_email: "pulp-infra@redhat.com" +release_user: "pulpbot" stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 @@ -86,4 +97,5 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true +...