From e0d975cd3d29ab936b6e18bf126101b507aec68a Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Wed, 18 Feb 2026 10:43:47 -0800 Subject: [PATCH 01/35] Add mssql_py_core wheel installation to PR validation pipeline - Add install-mssql-py-core.ps1 (Windows) and install-mssql-py-core.sh (Linux/macOS) scripts that download the mssql-py-core-wheels NuGet package from the public Azure Artifacts feed and pip install the platform-appropriate wheel - Add eng/versions/mssql-py-core.version to pin the NuGet package version (no fallback to latest - file is required) - Add 'Install mssql_py_core' step to all 10 test jobs in pr-validation-pipeline.yml - No authentication required (public feed), no nuget.exe dependency (raw HTTP + unzip) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 89 ++++++++++++++ eng/scripts/install-mssql-py-core.ps1 | 130 ++++++++++++++++++++ eng/scripts/install-mssql-py-core.sh | 146 +++++++++++++++++++++++ eng/versions/mssql-py-core.version | 1 + 4 files changed, 366 insertions(+) create mode 100644 eng/scripts/install-mssql-py-core.ps1 create mode 100644 eng/scripts/install-mssql-py-core.sh create mode 100644 eng/versions/mssql-py-core.version diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index b3df96ff..73fbb6c9 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -230,6 +230,13 @@ jobs: build.bat x64 displayName: 'Build .pyd file' + # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) + - task: PowerShell@2 + displayName: 'Install mssql_py_core from NuGet wheels' + inputs: + targetType: 'filePath' + filePath: 'eng/scripts/install-mssql-py-core.ps1' + # Run tests for LocalDB - script: | python -m pytest -v --junitxml=test-results-localdb.xml --cov=. --cov-report=xml:coverage-localdb.xml --capture=tee-sys --cache-clear @@ -497,6 +504,12 @@ jobs: ./build.sh displayName: 'Build pybind bindings (.so)' + # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) + - script: | + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + displayName: 'Install mssql_py_core from NuGet wheels' + - script: | echo "Build successful, running tests now" python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear @@ -669,6 +682,15 @@ jobs: " displayName: 'Build pybind bindings (.so) in $(distroName) container' + # Install mssql_py_core from NuGet wheel package inside container + - script: | + docker exec test-container-$(distroName) bash -c " + source /opt/venv/bin/activate + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + " + displayName: 'Install mssql_py_core in $(distroName) container' + - script: | # Uninstall ODBC Driver before running tests docker exec test-container-$(distroName) bash -c " @@ -984,6 +1006,15 @@ jobs: displayName: 'Build pybind bindings (.so) in $(distroName) ARM64 container' retryCountOnTaskFailure: 2 + # Install mssql_py_core from NuGet wheel package inside ARM64 container + - script: | + docker exec test-container-$(distroName)-$(archName) bash -c " + source /opt/venv/bin/activate + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + " + displayName: 'Install mssql_py_core in $(distroName) ARM64 container' + - script: | # Uninstall ODBC Driver before running tests docker exec test-container-$(distroName)-$(archName) bash -c " @@ -1192,6 +1223,15 @@ jobs: " displayName: 'Build pybind bindings (.so) in RHEL 9 container' + # Install mssql_py_core from NuGet wheel package inside RHEL 9 container + - script: | + docker exec test-container-rhel9 bash -c " + source myvenv/bin/activate + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + " + displayName: 'Install mssql_py_core in RHEL 9 container' + - script: | # Uninstall ODBC Driver before running tests docker exec test-container-rhel9 bash -c " @@ -1411,6 +1451,15 @@ jobs: displayName: 'Build pybind bindings (.so) in RHEL 9 ARM64 container' retryCountOnTaskFailure: 2 + # Install mssql_py_core from NuGet wheel package inside RHEL 9 ARM64 container + - script: | + docker exec test-container-rhel9-arm64 bash -c " + source myvenv/bin/activate + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + " + displayName: 'Install mssql_py_core in RHEL 9 ARM64 container' + - script: | # Uninstall ODBC Driver before running tests docker exec test-container-rhel9-arm64 bash -c " @@ -1638,6 +1687,15 @@ jobs: " displayName: 'Build pybind bindings (.so) in Alpine x86_64 container' + # Install mssql_py_core from NuGet wheel package inside Alpine container + - script: | + docker exec test-container-alpine bash -c " + source /workspace/venv/bin/activate + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + " + displayName: 'Install mssql_py_core in Alpine x86_64 container' + - script: | # Uninstall ODBC Driver before running tests to use bundled libraries docker exec test-container-alpine bash -c " @@ -1883,6 +1941,15 @@ jobs: displayName: 'Build pybind bindings (.so) in Alpine ARM64 container' retryCountOnTaskFailure: 2 + # Install mssql_py_core from NuGet wheel package inside Alpine ARM64 container + - script: | + docker exec test-container-alpine-arm64 bash -c " + source /workspace/venv/bin/activate + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + " + displayName: 'Install mssql_py_core in Alpine ARM64 container' + - script: | # Uninstall ODBC Driver before running tests to use bundled libraries docker exec test-container-alpine-arm64 bash -c " @@ -2005,6 +2072,13 @@ jobs: build.bat x64 displayName: 'Build .pyd file' + # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) + - task: PowerShell@2 + displayName: 'Install mssql_py_core from NuGet wheels' + inputs: + targetType: 'filePath' + filePath: 'eng/scripts/install-mssql-py-core.ps1' + - script: | python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear displayName: 'Run tests on Azure SQL Database' @@ -2047,6 +2121,12 @@ jobs: ./build.sh displayName: 'Build pybind bindings (.so)' + # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) + - script: | + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + displayName: 'Install mssql_py_core from NuGet wheels' + - script: | python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear displayName: 'Run tests on Azure SQL Database' @@ -2118,6 +2198,15 @@ jobs: " displayName: 'Build pybind bindings (.so) in Ubuntu container' + # Install mssql_py_core from NuGet wheel package inside Ubuntu container + - script: | + docker exec test-container-ubuntu-azuresql bash -c " + source /opt/venv/bin/activate + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + " + displayName: 'Install mssql_py_core in Ubuntu container' + - script: | docker exec test-container-ubuntu-azuresql bash -c " export DEBIAN_FRONTEND=noninteractive diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 new file mode 100644 index 00000000..cc388494 --- /dev/null +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -0,0 +1,130 @@ +<# +.SYNOPSIS + Downloads the mssql-py-core-wheels NuGet package from a public Azure Artifacts + feed and installs the appropriate wheel for the current platform into the + Python environment so that 'import mssql_py_core' works. + + The package version is read from eng/versions/mssql-py-core.version (required). + +.PARAMETER FeedUrl + The NuGet v3 feed URL. This is a public feed — no authentication required. + +.PARAMETER OutputDir + Temporary directory for downloaded artifacts. Cleaned up after install. + Defaults to $env:TEMP\mssql-py-core-wheels. +#> + +param( + [string]$FeedUrl = "https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json", + [string]$OutputDir = "$env:TEMP\mssql-py-core-wheels" +) + +$ErrorActionPreference = 'Stop' + +Write-Host "=== Install mssql_py_core from NuGet wheel package ===" + +# Read version from pinned version file (required) +$repoRoot = (Get-Item "$PSScriptRoot\..\..").FullName +$versionFile = Join-Path $repoRoot "eng\versions\mssql-py-core.version" +if (-not (Test-Path $versionFile)) { + throw "Version file not found: $versionFile. This file must exist and contain the mssql-py-core-wheels NuGet package version." +} +$PackageVersion = (Get-Content $versionFile -Raw).Trim() +if (-not $PackageVersion) { + throw "Version file is empty: $versionFile" +} +Write-Host "Using version from $versionFile : $PackageVersion" + +# Determine platform info +$pyVersion = & python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')" +$platform = & python -c "import platform; print(platform.system().lower())" +$arch = & python -c "import platform; print(platform.machine().lower())" + +Write-Host "Python: $pyVersion | Platform: $platform | Arch: $arch" + +# Map to wheel filename platform tags +switch ($platform) { + 'windows' { + switch -Regex ($arch) { + 'amd64|x86_64' { $wheelPlatform = "win_amd64" } + 'arm64|aarch64' { $wheelPlatform = "win_arm64" } + default { throw "Unsupported Windows architecture: $arch" } + } + } + 'linux' { + switch -Regex ($arch) { + 'x86_64|amd64' { $wheelPlatform = "manylinux_2_28_x86_64" } + 'aarch64|arm64' { $wheelPlatform = "manylinux_2_28_aarch64" } + default { throw "Unsupported Linux architecture: $arch" } + } + } + 'darwin' { + $wheelPlatform = "macosx_15_0_universal2" + } + default { throw "Unsupported platform: $platform" } +} + +$wheelPattern = "mssql_py_core-*-$pyVersion-$pyVersion-$wheelPlatform.whl" +Write-Host "Looking for wheel matching: $wheelPattern" + +# Create temp directory +if (Test-Path $OutputDir) { Remove-Item $OutputDir -Recurse -Force } +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + +# Download NuGet package +$nugetDir = "$OutputDir\nuget" +New-Item -ItemType Directory -Path $nugetDir -Force | Out-Null + +# Resolve NuGet v3 feed to find package base URL +Write-Host "Resolving feed: $FeedUrl" +$feedIndex = Invoke-RestMethod -Uri $FeedUrl +$packageBaseUrl = ($feedIndex.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress*' }).'@id' +if (-not $packageBaseUrl) { throw "Could not resolve PackageBaseAddress from feed" } +Write-Host "Package base: $packageBaseUrl" + +$packageId = "mssql-py-core-wheels" +$packageIdLower = $packageId.ToLower() + +$versionLower = $PackageVersion.ToLower() +$nupkgUrl = "$packageBaseUrl$packageIdLower/$versionLower/$packageIdLower.$versionLower.nupkg" +$nupkgPath = "$nugetDir\$packageIdLower.$versionLower.nupkg" + +Write-Host "Downloading: $nupkgUrl" +Invoke-WebRequest -Uri $nupkgUrl -OutFile $nupkgPath +Write-Host "Downloaded: $nupkgPath ($([math]::Round((Get-Item $nupkgPath).Length / 1MB, 2)) MB)" + +# Extract NuGet (it's a ZIP — rename so Expand-Archive accepts it) +$zipPath = "$nugetDir\$packageIdLower.$versionLower.zip" +Rename-Item -Path $nupkgPath -NewName (Split-Path $zipPath -Leaf) +$extractDir = "$nugetDir\extracted" +Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force + +# Find the matching wheel +$wheelsDir = "$extractDir\wheels" +if (-not (Test-Path $wheelsDir)) { + throw "No 'wheels' directory found in NuGet package. Contents: $(Get-ChildItem $extractDir -Recurse | Select-Object -ExpandProperty Name)" +} + +$matchingWheel = Get-ChildItem $wheelsDir -Filter $wheelPattern | Select-Object -First 1 +if (-not $matchingWheel) { + Write-Host "Available wheels:" + Get-ChildItem $wheelsDir -Filter *.whl | ForEach-Object { Write-Host " $_" } + throw "No wheel found matching pattern: $wheelPattern" +} + +Write-Host "Found matching wheel: $($matchingWheel.Name)" + +# Install the wheel with pip +Write-Host "Installing wheel with pip..." +& python -m pip install $matchingWheel.FullName --force-reinstall --no-deps +if ($LASTEXITCODE -ne 0) { throw "pip install failed with exit code $LASTEXITCODE" } + +# Verify import works +Write-Host "Verifying mssql_py_core import..." +& python -c "import mssql_py_core; print(f'mssql_py_core loaded successfully: {dir(mssql_py_core)}')" +if ($LASTEXITCODE -ne 0) { throw "Failed to import mssql_py_core" } + +# Cleanup +Remove-Item $OutputDir -Recurse -Force -ErrorAction SilentlyContinue + +Write-Host "=== mssql_py_core installed successfully ===" diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh new file mode 100644 index 00000000..45d3aca1 --- /dev/null +++ b/eng/scripts/install-mssql-py-core.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Downloads the mssql-py-core-wheels NuGet package from a public Azure Artifacts +# feed and installs the appropriate wheel for the current platform so that +# 'import mssql_py_core' works. No authentication required (public feed). +# +# The package version is read from eng/versions/mssql-py-core.version (required). +# +# Usage: +# ./install-mssql-py-core.sh [--feed-url URL] + +set -euo pipefail + +FEED_URL="${FEED_URL:-https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json}" +OUTPUT_DIR="${TMPDIR:-/tmp}/mssql-py-core-wheels" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case "$1" in + --feed-url) FEED_URL="$2"; shift 2 ;; + *) echo "Unknown argument: $1"; exit 1 ;; + esac +done + +echo "=== Install mssql_py_core from NuGet wheel package ===" + +# Read version from pinned version file (required) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VERSION_FILE="$SCRIPT_DIR/../../eng/versions/mssql-py-core.version" +if [ ! -f "$VERSION_FILE" ]; then + echo "ERROR: Version file not found: $VERSION_FILE" + echo "This file must exist and contain the mssql-py-core-wheels NuGet package version." + exit 1 +fi +PACKAGE_VERSION=$(tr -d '[:space:]' < "$VERSION_FILE") +if [ -z "$PACKAGE_VERSION" ]; then + echo "ERROR: Version file is empty: $VERSION_FILE" + exit 1 +fi +echo "Using version from $VERSION_FILE: $PACKAGE_VERSION" + +# Determine platform info +PY_VERSION=$(python3 -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") +PLATFORM=$(python3 -c "import platform; print(platform.system().lower())") +ARCH=$(python3 -c "import platform; print(platform.machine().lower())") + +echo "Python: $PY_VERSION | Platform: $PLATFORM | Arch: $ARCH" + +# Map to wheel platform tags +case "$PLATFORM" in + linux) + case "$ARCH" in + x86_64|amd64) WHEEL_PLATFORM="manylinux_2_28_x86_64" ;; + aarch64|arm64) WHEEL_PLATFORM="manylinux_2_28_aarch64" ;; + *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; + esac + ;; + darwin) + WHEEL_PLATFORM="macosx_15_0_universal2" + ;; + *) + echo "Unsupported platform: $PLATFORM" + exit 1 + ;; +esac + +WHEEL_PATTERN="mssql_py_core-*-${PY_VERSION}-${PY_VERSION}-${WHEEL_PLATFORM}.whl" +echo "Looking for wheel matching: $WHEEL_PATTERN" + +# Setup temp directory +rm -rf "$OUTPUT_DIR" +mkdir -p "$OUTPUT_DIR" + +# Resolve NuGet v3 feed +echo "Resolving feed: $FEED_URL" +FEED_INDEX=$(curl -sS "$FEED_URL") + +PACKAGE_BASE_URL=$(echo "$FEED_INDEX" | python3 -c " +import json, sys +data = json.load(sys.stdin) +for r in data['resources']: + if 'PackageBaseAddress' in r.get('@type', ''): + print(r['@id']) + break +") + +if [ -z "$PACKAGE_BASE_URL" ]; then + echo "Could not resolve PackageBaseAddress from feed" + exit 1 +fi +echo "Package base: $PACKAGE_BASE_URL" + +PACKAGE_ID="mssql-py-core-wheels" + +VERSION_LOWER=$(echo "$PACKAGE_VERSION" | tr '[:upper:]' '[:lower:]') +NUPKG_URL="${PACKAGE_BASE_URL}${PACKAGE_ID}/${VERSION_LOWER}/${PACKAGE_ID}.${VERSION_LOWER}.nupkg" +NUPKG_PATH="$OUTPUT_DIR/${PACKAGE_ID}.${VERSION_LOWER}.nupkg" + +echo "Downloading: $NUPKG_URL" +curl -sSL -o "$NUPKG_PATH" "$NUPKG_URL" +FILESIZE=$(stat -c%s "$NUPKG_PATH" 2>/dev/null || stat -f%z "$NUPKG_PATH" 2>/dev/null || echo "unknown") +echo "Downloaded: $NUPKG_PATH ($FILESIZE bytes)" + +if [ "$FILESIZE" = "0" ] || [ "$FILESIZE" = "unknown" ]; then + echo "ERROR: Downloaded file is empty or could not determine size" + exit 1 +fi + +# Extract NuGet (ZIP format) — use python if unzip is not available +EXTRACT_DIR="$OUTPUT_DIR/extracted" +mkdir -p "$EXTRACT_DIR" +if command -v unzip &>/dev/null; then + unzip -q "$NUPKG_PATH" -d "$EXTRACT_DIR" +else + python3 -c "import zipfile; zipfile.ZipFile('$NUPKG_PATH').extractall('$EXTRACT_DIR')" +fi + +# Find matching wheel +WHEELS_DIR="$EXTRACT_DIR/wheels" +if [ ! -d "$WHEELS_DIR" ]; then + echo "No 'wheels' directory found in NuGet package" + ls -la "$EXTRACT_DIR" + exit 1 +fi + +MATCHING_WHEEL=$(find "$WHEELS_DIR" -name "$WHEEL_PATTERN" | head -1) +if [ -z "$MATCHING_WHEEL" ]; then + echo "Available wheels:" + ls "$WHEELS_DIR"/*.whl 2>/dev/null || echo " (none)" + echo "No wheel found matching pattern: $WHEEL_PATTERN" + exit 1 +fi + +echo "Found matching wheel: $(basename "$MATCHING_WHEEL")" + +# Install with pip +echo "Installing wheel with pip..." +python3 -m pip install "$MATCHING_WHEEL" --force-reinstall --no-deps + +# Verify import +echo "Verifying mssql_py_core import..." +python3 -c "import mssql_py_core; print(f'mssql_py_core loaded successfully: {dir(mssql_py_core)}')" + +# Cleanup +rm -rf "$OUTPUT_DIR" + +echo "=== mssql_py_core installed successfully ===" diff --git a/eng/versions/mssql-py-core.version b/eng/versions/mssql-py-core.version new file mode 100644 index 00000000..a733259d --- /dev/null +++ b/eng/versions/mssql-py-core.version @@ -0,0 +1 @@ +0.1.0-dev.20260218.140230 From 6cc57564d6de350d6e9f521146cd982f46b1ee29 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Wed, 18 Feb 2026 13:42:10 -0800 Subject: [PATCH 02/35] Fix Alpine compatibility: detect musl libc and skip gracefully when no wheel available - Detect musl vs glibc on Linux to use musllinux_1_2 vs manylinux_2_28 wheel tags - Skip with warning (exit 0) instead of failing when no compatible wheel is found - Allows Alpine jobs to continue without blocking on missing musllinux wheels Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/scripts/install-mssql-py-core.ps1 | 5 ++++- eng/scripts/install-mssql-py-core.sh | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index cc388494..13e708da 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -107,9 +107,12 @@ if (-not (Test-Path $wheelsDir)) { $matchingWheel = Get-ChildItem $wheelsDir -Filter $wheelPattern | Select-Object -First 1 if (-not $matchingWheel) { + Write-Host "##[warning]No wheel found matching pattern: $wheelPattern" Write-Host "Available wheels:" Get-ChildItem $wheelsDir -Filter *.whl | ForEach-Object { Write-Host " $_" } - throw "No wheel found matching pattern: $wheelPattern" + Write-Host "Skipping mssql_py_core installation — no compatible wheel for this platform." + Remove-Item $OutputDir -Recurse -Force -ErrorAction SilentlyContinue + exit 0 } Write-Host "Found matching wheel: $($matchingWheel.Name)" diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index 45d3aca1..b2c2109a 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -46,11 +46,17 @@ ARCH=$(python3 -c "import platform; print(platform.machine().lower())") echo "Python: $PY_VERSION | Platform: $PLATFORM | Arch: $ARCH" # Map to wheel platform tags +# Detect musl (Alpine) vs glibc for Linux case "$PLATFORM" in linux) + if ldd --version 2>&1 | grep -qi musl; then + LIBC="musllinux_1_2" + else + LIBC="manylinux_2_28" + fi case "$ARCH" in - x86_64|amd64) WHEEL_PLATFORM="manylinux_2_28_x86_64" ;; - aarch64|arm64) WHEEL_PLATFORM="manylinux_2_28_aarch64" ;; + x86_64|amd64) WHEEL_PLATFORM="${LIBC}_x86_64" ;; + aarch64|arm64) WHEEL_PLATFORM="${LIBC}_aarch64" ;; *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; esac ;; @@ -124,10 +130,12 @@ fi MATCHING_WHEEL=$(find "$WHEELS_DIR" -name "$WHEEL_PATTERN" | head -1) if [ -z "$MATCHING_WHEEL" ]; then + echo "##[warning]No wheel found matching pattern: $WHEEL_PATTERN" echo "Available wheels:" ls "$WHEELS_DIR"/*.whl 2>/dev/null || echo " (none)" - echo "No wheel found matching pattern: $WHEEL_PATTERN" - exit 1 + echo "Skipping mssql_py_core installation — no compatible wheel for this platform." + rm -rf "$OUTPUT_DIR" + exit 0 fi echo "Found matching wheel: $(basename "$MATCHING_WHEEL")" From cbef84a6cd237d06ab22d7b1b6dde5eb607d9d3d Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Wed, 18 Feb 2026 13:52:40 -0800 Subject: [PATCH 03/35] Fail hard when no compatible wheel is found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Missing wheels must block the pipeline — every platform must have a matching wheel. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/scripts/install-mssql-py-core.ps1 | 5 +---- eng/scripts/install-mssql-py-core.sh | 6 ++---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index 13e708da..cc388494 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -107,12 +107,9 @@ if (-not (Test-Path $wheelsDir)) { $matchingWheel = Get-ChildItem $wheelsDir -Filter $wheelPattern | Select-Object -First 1 if (-not $matchingWheel) { - Write-Host "##[warning]No wheel found matching pattern: $wheelPattern" Write-Host "Available wheels:" Get-ChildItem $wheelsDir -Filter *.whl | ForEach-Object { Write-Host " $_" } - Write-Host "Skipping mssql_py_core installation — no compatible wheel for this platform." - Remove-Item $OutputDir -Recurse -Force -ErrorAction SilentlyContinue - exit 0 + throw "No wheel found matching pattern: $wheelPattern" } Write-Host "Found matching wheel: $($matchingWheel.Name)" diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index b2c2109a..502e23f7 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -130,12 +130,10 @@ fi MATCHING_WHEEL=$(find "$WHEELS_DIR" -name "$WHEEL_PATTERN" | head -1) if [ -z "$MATCHING_WHEEL" ]; then - echo "##[warning]No wheel found matching pattern: $WHEEL_PATTERN" echo "Available wheels:" ls "$WHEELS_DIR"/*.whl 2>/dev/null || echo " (none)" - echo "Skipping mssql_py_core installation — no compatible wheel for this platform." - rm -rf "$OUTPUT_DIR" - exit 0 + echo "ERROR: No wheel found matching pattern: $WHEEL_PATTERN" + exit 1 fi echo "Found matching wheel: $(basename "$MATCHING_WHEEL")" From 3829c5f56f3079f08349586dab1c792babd14edf Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Wed, 18 Feb 2026 18:06:10 -0800 Subject: [PATCH 04/35] Fix musl detection on Alpine for mssql_py_core install ldd --version exits with code 1 on musl/Alpine. Combined with set -euo pipefail, the piped grep check always took the else branch, misidentifying Alpine as glibc (manylinux). pip then rejected the manylinux wheel since it cannot run on musl. Fix: capture ldd output into a variable with || true before grepping. Add fallback detection via /etc/alpine-release and /lib/ld-musl-*. Skip gracefully (exit 0) when no musllinux wheel is available yet. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/scripts/install-mssql-py-core.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index 502e23f7..06037a4a 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -49,7 +49,20 @@ echo "Python: $PY_VERSION | Platform: $PLATFORM | Arch: $ARCH" # Detect musl (Alpine) vs glibc for Linux case "$PLATFORM" in linux) - if ldd --version 2>&1 | grep -qi musl; then + # Detect musl libc (Alpine) vs glibc — multiple methods for robustness + # Note: ldd --version exits with code 1 on musl, which combined with + # pipefail causes the grep pipeline to fail. Use a variable instead. + IS_MUSL=false + LDD_OUTPUT=$(ldd --version 2>&1 || true) + if echo "$LDD_OUTPUT" | grep -qi musl; then + IS_MUSL=true + elif [ -f /etc/alpine-release ]; then + IS_MUSL=true + elif ls /lib/ld-musl-* >/dev/null 2>&1; then + IS_MUSL=true + fi + + if $IS_MUSL; then LIBC="musllinux_1_2" else LIBC="manylinux_2_28" @@ -132,6 +145,13 @@ MATCHING_WHEEL=$(find "$WHEELS_DIR" -name "$WHEEL_PATTERN" | head -1) if [ -z "$MATCHING_WHEEL" ]; then echo "Available wheels:" ls "$WHEELS_DIR"/*.whl 2>/dev/null || echo " (none)" + # On musllinux (Alpine), no wheels may be available yet — skip gracefully + if echo "$WHEEL_PLATFORM" | grep -q "musllinux"; then + echo "WARNING: No musllinux wheel found matching pattern: $WHEEL_PATTERN" + echo "mssql_py_core is not yet available for musllinux — skipping installation." + rm -rf "$OUTPUT_DIR" + exit 0 + fi echo "ERROR: No wheel found matching pattern: $WHEEL_PATTERN" exit 1 fi From 389ca380dde4ee40090379a4b7be40bdaf465684 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Wed, 18 Feb 2026 18:37:49 -0800 Subject: [PATCH 05/35] Add mssql_py_core repackaging into mssql-python wheels Add eng/scripts/repackage-with-mssql-py-core.py that downloads the mssql-py-core-wheels NuGet package, matches each mssql-python wheel to its corresponding mssql_py_core wheel by platform/Python tags, extracts the native extension (.pyd/.so) and supporting files, and injects them into the mssql-python wheel with updated RECORD hashes. Pipeline changes: - OneBranch consolidate-artifacts-job.yml: checkout source, install Python, run repackaging after wheel consolidation - eng/pipelines/build-whl-pipeline.yml: run repackaging after wheel build in Windows, macOS, and Linux jobs musllinux wheels are skipped gracefully since mssql_py_core does not yet ship musllinux builds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../jobs/consolidate-artifacts-job.yml | 18 +- eng/pipelines/build-whl-pipeline.yml | 24 ++ eng/scripts/repackage-with-mssql-py-core.py | 311 ++++++++++++++++++ 3 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 eng/scripts/repackage-with-mssql-py-core.py diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 0ef960fc..7deac0c6 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -26,7 +26,8 @@ jobs: value: '$(Build.ArtifactStagingDirectory)' steps: - - checkout: none # No source code needed for consolidation + - checkout: self # Need source for repackaging script and version file + fetchDepth: 1 # Download ALL artifacts from current build # Matrix jobs publish as: Windows_, macOS_, Linux_ @@ -76,6 +77,21 @@ jobs: fi displayName: 'Consolidate wheels from all platforms' + # Repackage wheels: inject mssql_py_core binaries from NuGet into each mssql-python wheel + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + addToPath: true + displayName: 'Use Python 3.12 for repackaging' + + - bash: | + set -e + echo "Repackaging mssql-python wheels with mssql_py_core binaries..." + python $(Build.SourcesDirectory)/eng/scripts/repackage-with-mssql-py-core.py \ + --wheel-dir $(ob_outputDirectory)/dist \ + --version-file $(Build.SourcesDirectory)/eng/versions/mssql-py-core.version + displayName: 'Inject mssql_py_core into wheels' + # Optional: Consolidate native bindings for reference - bash: | set -e diff --git a/eng/pipelines/build-whl-pipeline.yml b/eng/pipelines/build-whl-pipeline.yml index a6540c8a..db9f68a4 100644 --- a/eng/pipelines/build-whl-pipeline.yml +++ b/eng/pipelines/build-whl-pipeline.yml @@ -184,6 +184,16 @@ jobs: TargetFolder: '$(Build.ArtifactStagingDirectory)\dist' displayName: 'Collect wheel package' + # Repackage wheel: inject mssql_py_core binary from NuGet + - task: PowerShell@2 + displayName: 'Inject mssql_py_core into wheel' + inputs: + targetType: 'inline' + script: | + python "$(Build.SourcesDirectory)\eng\scripts\repackage-with-mssql-py-core.py" ` + --wheel-dir "$(Build.ArtifactStagingDirectory)\dist" ` + --version-file "$(Build.SourcesDirectory)\eng\versions\mssql-py-core.version" + # Publish the collected .pyd file(s) as build artifacts - task: PublishBuildArtifacts@1 condition: succeededOrFailed() @@ -357,6 +367,13 @@ jobs: TargetFolder: '$(Build.ArtifactStagingDirectory)/dist' displayName: 'Collect wheel package' + # Repackage wheel: inject mssql_py_core binary from NuGet + - script: | + python $(Build.SourcesDirectory)/eng/scripts/repackage-with-mssql-py-core.py \ + --wheel-dir "$(Build.ArtifactStagingDirectory)/dist" \ + --version-file "$(Build.SourcesDirectory)/eng/versions/mssql-py-core.version" + displayName: 'Inject mssql_py_core into wheel' + # Publish the collected .so file(s) as build artifacts - task: PublishBuildArtifacts@1 condition: succeededOrFailed() @@ -565,6 +582,13 @@ jobs: find "$(Build.ArtifactStagingDirectory)/ddbc-bindings/$(LINUX_TAG)-$(ARCH)" -maxdepth 1 -type f ! -name "*.so" -delete || true displayName: 'Copy wheels and .so back to host' + # Repackage wheels: inject mssql_py_core binaries from NuGet + - script: | + python3 $(Build.SourcesDirectory)/eng/scripts/repackage-with-mssql-py-core.py \ + --wheel-dir "$(Build.ArtifactStagingDirectory)/dist" \ + --version-file "$(Build.SourcesDirectory)/eng/versions/mssql-py-core.version" + displayName: 'Inject mssql_py_core into wheels' + # Cleanup container - script: | docker stop build-$(LINUX_TAG)-$(ARCH) || true diff --git a/eng/scripts/repackage-with-mssql-py-core.py b/eng/scripts/repackage-with-mssql-py-core.py new file mode 100644 index 00000000..59915e1b --- /dev/null +++ b/eng/scripts/repackage-with-mssql-py-core.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +""" +Repackage mssql_py_core binaries into mssql-python wheels. + +Downloads the mssql-py-core-wheels NuGet package, extracts the matching +mssql_py_core native extension for each mssql-python wheel, and injects +it into the wheel so that mssql_py_core is bundled inside mssql-python. + +Usage: + python repackage-with-mssql-py-core.py --wheel-dir [--nuget-dir ] [--feed-url ] + +The NuGet package version is read from eng/versions/mssql-py-core.version. +If --nuget-dir is provided, it skips the download and uses pre-extracted wheels. +""" + +import argparse +import csv +import hashlib +import io +import json +import os +import re +import shutil +import sys +import tempfile +import urllib.request +import zipfile +from base64 import urlsafe_b64encode +from pathlib import Path + + +def parse_wheel_filename(filename: str) -> dict | None: + """Parse a wheel filename into its components.""" + # Format: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl + m = re.match( + r"^(?P[A-Za-z0-9_]+)-(?P[^-]+)" + r"(?:-(?P\d[^-]*))?" + r"-(?P[^-]+)-(?P[^-]+)-(?P.+)\.whl$", + filename, + ) + return m.groupdict() if m else None + + +def compute_record_hash(data: bytes) -> str: + """Compute sha256 hash in RECORD format: sha256=.""" + digest = hashlib.sha256(data).digest() + return "sha256=" + urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def find_matching_core_wheel( + mssql_python_whl: str, core_wheels_dir: Path +) -> Path | None: + """Find the mssql_py_core wheel matching a mssql-python wheel's platform tags.""" + parsed = parse_wheel_filename(os.path.basename(mssql_python_whl)) + if not parsed: + return None + + python_tag = parsed["python"] # e.g., cp313 + platform_tag = parsed["platform"] # e.g., win_amd64 + + # Look for matching core wheel + pattern = f"mssql_py_core-*-{python_tag}-{python_tag}-{platform_tag}.whl" + matches = list(core_wheels_dir.glob(pattern)) + + if not matches: + # Try without exact abi match (some wheels use cpXXX for both) + for whl in core_wheels_dir.glob("mssql_py_core-*.whl"): + core_parsed = parse_wheel_filename(whl.name) + if core_parsed and core_parsed["platform"] == platform_tag and core_parsed["python"] == python_tag: + return whl + return None + + return matches[0] + + +def inject_core_into_wheel( + mssql_python_whl: Path, core_whl: Path, output_dir: Path +) -> Path: + """ + Inject mssql_py_core files from core_whl into mssql_python_whl. + + Copies the mssql_py_core/ package and mssql_py_core.libs/ (if present) + into the mssql-python wheel and updates the RECORD file. + """ + output_path = output_dir / mssql_python_whl.name + + # Read the core wheel contents we want to inject + inject_files: dict[str, bytes] = {} + with zipfile.ZipFile(core_whl, "r") as core_zip: + for entry in core_zip.namelist(): + # Include mssql_py_core/ package files and mssql_py_core.libs/ + if entry.startswith("mssql_py_core/") or entry.startswith("mssql_py_core.libs/"): + inject_files[entry] = core_zip.read(entry) + + if not inject_files: + print(f" WARNING: No mssql_py_core files found in {core_whl.name}") + shutil.copy2(mssql_python_whl, output_path) + return output_path + + # Build new wheel with injected files + with zipfile.ZipFile(mssql_python_whl, "r") as src_zip: + with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as dst_zip: + # Find the dist-info directory name for mssql-python + dist_info_dir = None + for name in src_zip.namelist(): + if name.endswith("/RECORD"): + dist_info_dir = name.rsplit("/", 1)[0] + break + + if not dist_info_dir: + raise ValueError(f"No RECORD found in {mssql_python_whl.name}") + + record_path = f"{dist_info_dir}/RECORD" + new_record_entries: list[str] = [] + + # Copy all existing files (except RECORD, we'll regenerate it) + for item in src_zip.infolist(): + if item.filename == record_path: + # Parse existing RECORD entries (we'll append to them) + existing_record = src_zip.read(item.filename).decode("utf-8") + for line in existing_record.strip().split("\n"): + line = line.strip() + if line and not line.startswith(record_path): + new_record_entries.append(line) + continue + + data = src_zip.read(item.filename) + dst_zip.writestr(item, data) + + # Inject mssql_py_core files + for filename, data in inject_files.items(): + dst_zip.writestr(filename, data) + file_hash = compute_record_hash(data) + new_record_entries.append(f"{filename},{file_hash},{len(data)}") + + # Write updated RECORD + new_record_entries.append(f"{record_path},,") + record_content = "\n".join(new_record_entries) + "\n" + dst_zip.writestr(record_path, record_content) + + return output_path + + +def download_nuget_package( + feed_url: str, package_id: str, version: str, output_dir: Path +) -> Path: + """Download and extract a NuGet package from an Azure Artifacts feed.""" + print(f"Resolving NuGet feed: {feed_url}") + with urllib.request.urlopen(feed_url) as resp: + feed_index = json.loads(resp.read().decode()) + + package_base_url = None + for resource in feed_index.get("resources", []): + if "PackageBaseAddress" in resource.get("@type", ""): + package_base_url = resource["@id"] + break + + if not package_base_url: + raise RuntimeError("Could not resolve PackageBaseAddress from feed") + + print(f"Package base URL: {package_base_url}") + + version_lower = version.lower() + pkg_id_lower = package_id.lower() + nupkg_url = f"{package_base_url}{pkg_id_lower}/{version_lower}/{pkg_id_lower}.{version_lower}.nupkg" + + nupkg_path = output_dir / f"{pkg_id_lower}.{version_lower}.nupkg" + print(f"Downloading: {nupkg_url}") + urllib.request.urlretrieve(nupkg_url, nupkg_path) + size_mb = nupkg_path.stat().st_size / (1024 * 1024) + print(f"Downloaded: {nupkg_path.name} ({size_mb:.1f} MB)") + + # Extract (NuGet packages are ZIP files) + extract_dir = output_dir / "extracted" + with zipfile.ZipFile(nupkg_path, "r") as z: + z.extractall(extract_dir) + + wheels_dir = extract_dir / "wheels" + if not wheels_dir.is_dir(): + raise RuntimeError( + f"No 'wheels' directory in NuGet package. Contents: {list(extract_dir.iterdir())}" + ) + + return wheels_dir + + +def main(): + parser = argparse.ArgumentParser( + description="Repackage mssql_py_core binaries into mssql-python wheels" + ) + parser.add_argument( + "--wheel-dir", + required=True, + help="Directory containing mssql-python wheels to repackage", + ) + parser.add_argument( + "--nuget-dir", + help="Directory containing pre-extracted mssql_py_core wheels (skips download)", + ) + parser.add_argument( + "--feed-url", + default="https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json", + help="NuGet v3 feed URL", + ) + parser.add_argument( + "--version-file", + help="Path to mssql-py-core.version file (default: auto-detect from repo)", + ) + parser.add_argument( + "--output-dir", + help="Output directory for repackaged wheels (default: overwrite in place)", + ) + args = parser.parse_args() + + wheel_dir = Path(args.wheel_dir) + if not wheel_dir.is_dir(): + print(f"ERROR: Wheel directory not found: {wheel_dir}") + sys.exit(1) + + # Find mssql-python wheels + mssql_python_wheels = sorted(wheel_dir.glob("mssql_python-*.whl")) + if not mssql_python_wheels: + print(f"ERROR: No mssql_python-*.whl files found in {wheel_dir}") + sys.exit(1) + + print(f"Found {len(mssql_python_wheels)} mssql-python wheel(s)") + + # Get or download mssql_py_core wheels + temp_dir = None + if args.nuget_dir: + core_wheels_dir = Path(args.nuget_dir) + else: + # Find version file + if args.version_file: + version_file = Path(args.version_file) + else: + # Auto-detect from repo root + script_dir = Path(__file__).resolve().parent + version_file = script_dir / ".." / "versions" / "mssql-py-core.version" + + if not version_file.is_file(): + print(f"ERROR: Version file not found: {version_file}") + sys.exit(1) + + version = version_file.read_text().strip() + print(f"mssql-py-core version: {version}") + + temp_dir = Path(tempfile.mkdtemp(prefix="mssql-py-core-")) + core_wheels_dir = download_nuget_package( + args.feed_url, "mssql-py-core-wheels", version, temp_dir + ) + + print(f"Core wheels directory: {core_wheels_dir}") + core_wheel_count = len(list(core_wheels_dir.glob("*.whl"))) + print(f"Available mssql_py_core wheels: {core_wheel_count}") + + # Set up output directory + if args.output_dir: + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + else: + # In-place: use temp output, then replace originals + output_dir = Path(tempfile.mkdtemp(prefix="repackaged-")) + + # Repackage each wheel + repackaged = 0 + skipped = 0 + for whl in mssql_python_wheels: + print(f"\nProcessing: {whl.name}") + core_whl = find_matching_core_wheel(whl.name, core_wheels_dir) + + if core_whl is None: + parsed = parse_wheel_filename(whl.name) + platform = parsed["platform"] if parsed else "unknown" + # musllinux wheels have no matching core wheel yet — skip gracefully + if "musllinux" in platform: + print(f" SKIP: No musllinux mssql_py_core wheel available for {platform}") + if not args.output_dir: + # In-place mode: copy unchanged + shutil.copy2(whl, output_dir / whl.name) + skipped += 1 + continue + else: + print(f" ERROR: No matching mssql_py_core wheel for {whl.name}") + sys.exit(1) + + print(f" Matched: {core_whl.name}") + result = inject_core_into_wheel(whl, core_whl, output_dir) + print(f" Repackaged: {result.name}") + repackaged += 1 + + # If in-place mode, replace originals + if not args.output_dir: + for repackaged_whl in output_dir.glob("*.whl"): + dest = wheel_dir / repackaged_whl.name + shutil.move(str(repackaged_whl), str(dest)) + shutil.rmtree(output_dir, ignore_errors=True) + + # Cleanup temp NuGet download + if temp_dir: + shutil.rmtree(temp_dir, ignore_errors=True) + + print(f"\n{'='*50}") + print(f"Repackaging complete!") + print(f" Repackaged: {repackaged} wheel(s)") + print(f" Skipped: {skipped} wheel(s) (no matching core wheel)") + print(f"{'='*50}") + + +if __name__ == "__main__": + main() From c5544f04c96c0f42ab3fc0ff14fb5f6803fdadc0 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Wed, 18 Feb 2026 19:56:28 -0800 Subject: [PATCH 06/35] Block Official builds from using dev/nightly mssql-py-core versions Official builds now validate eng/versions/mssql-py-core.version and fail if it contains dev, nightly, alpha, beta, rc, or preview tags. NonOfficial builds are unaffected and can use any version. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../jobs/consolidate-artifacts-job.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 7deac0c6..1d700dda 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -29,6 +29,20 @@ jobs: - checkout: self # Need source for repackaging script and version file fetchDepth: 1 + # Official builds must not use dev/nightly mssql-py-core versions + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - bash: | + set -e + VERSION=$(cat $(Build.SourcesDirectory)/eng/versions/mssql-py-core.version | tr -d '[:space:]') + echo "mssql-py-core version: $VERSION" + if echo "$VERSION" | grep -qiE '(dev|nightly|alpha|beta|rc|preview)'; then + echo "##[error]Official builds cannot use pre-release mssql-py-core version: $VERSION" + echo "##[error]Update eng/versions/mssql-py-core.version to a stable release version." + exit 1 + fi + echo "Version '$VERSION' is acceptable for Official builds." + displayName: 'Validate mssql-py-core version for Official build' + # Download ALL artifacts from current build # Matrix jobs publish as: Windows_, macOS_, Linux_ # This downloads all of them automatically (27 total artifacts) From 67da0997ff639ffa3d1974caf0dfa31728575790 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:22:22 -0800 Subject: [PATCH 07/35] Add bulkcopy test, rewrite install scripts, make setup.py validate-only --- .gitignore | 3 + eng/scripts/install-mssql-py-core.ps1 | 89 ++++++++++++++++++---- eng/scripts/install-mssql-py-core.sh | 74 +++++++++++++++--- setup.py | 105 +++++++++++++++++++++++--- tests/test_019_bulkcopy.py | 77 +++++++++++++++++++ 5 files changed, 314 insertions(+), 34 deletions(-) create mode 100644 tests/test_019_bulkcopy.py diff --git a/.gitignore b/.gitignore index 3069e19d..3f9bd64e 100644 --- a/.gitignore +++ b/.gitignore @@ -61,5 +61,8 @@ build/ *venv*/ **/*venv*/ +# Extracted mssql_py_core (from eng/scripts/install-mssql-py-core) +mssql_py_core/ + # learning files learnings/ diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index cc388494..58ddf160 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -1,8 +1,17 @@ <# .SYNOPSIS Downloads the mssql-py-core-wheels NuGet package from a public Azure Artifacts - feed and installs the appropriate wheel for the current platform into the - Python environment so that 'import mssql_py_core' works. + feed and extracts the matching mssql_py_core binary into the repository root + so that 'import mssql_py_core' works when running from the source tree. + + The extracted files are placed at /mssql_py_core/ which contains: + - __init__.py + - mssql_py_core.. (native extension) + + This script is used identically for: + - Local development (dev runs it after build.bat/build.sh) + - PR validation pipelines + - Official build pipelines (before setup.py bdist_wheel) The package version is read from eng/versions/mssql-py-core.version (required). @@ -10,7 +19,7 @@ The NuGet v3 feed URL. This is a public feed — no authentication required. .PARAMETER OutputDir - Temporary directory for downloaded artifacts. Cleaned up after install. + Temporary directory for downloaded artifacts. Cleaned up after extraction. Defaults to $env:TEMP\mssql-py-core-wheels. #> @@ -23,8 +32,10 @@ $ErrorActionPreference = 'Stop' Write-Host "=== Install mssql_py_core from NuGet wheel package ===" -# Read version from pinned version file (required) +# Determine repository root (two levels up from this script) $repoRoot = (Get-Item "$PSScriptRoot\..\..").FullName + +# Read version from pinned version file (required) $versionFile = Join-Path $repoRoot "eng\versions\mssql-py-core.version" if (-not (Test-Path $versionFile)) { throw "Version file not found: $versionFile. This file must exist and contain the mssql-py-core-wheels NuGet package version." @@ -114,17 +125,69 @@ if (-not $matchingWheel) { Write-Host "Found matching wheel: $($matchingWheel.Name)" -# Install the wheel with pip -Write-Host "Installing wheel with pip..." -& python -m pip install $matchingWheel.FullName --force-reinstall --no-deps -if ($LASTEXITCODE -ne 0) { throw "pip install failed with exit code $LASTEXITCODE" } +# Extract mssql_py_core/ from the wheel into the repository root. +# The wheel is a ZIP file containing mssql_py_core/__init__.py and +# mssql_py_core/mssql_py_core... +# We skip .dist-info/ and mssql_py_core.libs/ (system deps expected). +$targetDir = $repoRoot +$coreDir = Join-Path $targetDir "mssql_py_core" + +# Clean previous extraction +if (Test-Path $coreDir) { + Remove-Item $coreDir -Recurse -Force + Write-Host "Cleaned previous mssql_py_core/ directory" +} -# Verify import works +Write-Host "Extracting mssql_py_core from wheel into: $targetDir" + +# Use Python to extract (zipfile handles wheel format reliably) +$wheelPath = $matchingWheel.FullName +& python -c @" +import zipfile, os, sys + +wheel_path = r'$wheelPath' +target_dir = r'$targetDir' +extracted = 0 + +with zipfile.ZipFile(wheel_path, 'r') as zf: + for entry in zf.namelist(): + # Skip dist-info metadata + if '.dist-info/' in entry: + continue + # Skip vendored shared libraries (system deps expected) + if entry.startswith('mssql_py_core.libs/'): + continue + if entry.startswith('mssql_py_core/'): + out_path = os.path.join(target_dir, entry) + if entry.endswith('/'): + os.makedirs(out_path, exist_ok=True) + continue + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, 'wb') as f: + f.write(zf.read(entry)) + extracted += 1 + print(f' Extracted: {entry}') + +if extracted == 0: + print('ERROR: No mssql_py_core files found in wheel', file=sys.stderr) + sys.exit(1) + +print(f'Extracted {extracted} file(s) into {target_dir}') +"@ +if ($LASTEXITCODE -ne 0) { throw "Failed to extract mssql_py_core from wheel" } + +# Verify import works (from repo root so mssql_py_core/ is on sys.path) Write-Host "Verifying mssql_py_core import..." -& python -c "import mssql_py_core; print(f'mssql_py_core loaded successfully: {dir(mssql_py_core)}')" -if ($LASTEXITCODE -ne 0) { throw "Failed to import mssql_py_core" } +Push-Location $repoRoot +try { + & python -c "import mssql_py_core; print(f'mssql_py_core loaded successfully: {dir(mssql_py_core)}')" + if ($LASTEXITCODE -ne 0) { throw "Failed to import mssql_py_core" } +} +finally { + Pop-Location +} -# Cleanup +# Cleanup temp files Remove-Item $OutputDir -Recurse -Force -ErrorAction SilentlyContinue -Write-Host "=== mssql_py_core installed successfully ===" +Write-Host "=== mssql_py_core extracted successfully ===" diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index 06037a4a..1914879f 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -1,7 +1,16 @@ #!/usr/bin/env bash # Downloads the mssql-py-core-wheels NuGet package from a public Azure Artifacts -# feed and installs the appropriate wheel for the current platform so that -# 'import mssql_py_core' works. No authentication required (public feed). +# feed and extracts the matching mssql_py_core binary into the repository root +# so that 'import mssql_py_core' works when running from the source tree. +# +# The extracted files are placed at /mssql_py_core/ which contains: +# - __init__.py +# - mssql_py_core..so (native extension) +# +# This script is used identically for: +# - Local development (dev runs it after build.sh) +# - PR validation pipelines +# - Official build pipelines (before setup.py bdist_wheel) # # The package version is read from eng/versions/mssql-py-core.version (required). # @@ -23,9 +32,12 @@ done echo "=== Install mssql_py_core from NuGet wheel package ===" -# Read version from pinned version file (required) +# Determine repository root (two levels up from this script) SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -VERSION_FILE="$SCRIPT_DIR/../../eng/versions/mssql-py-core.version" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Read version from pinned version file (required) +VERSION_FILE="$REPO_ROOT/eng/versions/mssql-py-core.version" if [ ! -f "$VERSION_FILE" ]; then echo "ERROR: Version file not found: $VERSION_FILE" echo "This file must exist and contain the mssql-py-core-wheels NuGet package version." @@ -158,15 +170,59 @@ fi echo "Found matching wheel: $(basename "$MATCHING_WHEEL")" -# Install with pip -echo "Installing wheel with pip..." -python3 -m pip install "$MATCHING_WHEEL" --force-reinstall --no-deps +# Extract mssql_py_core/ from the wheel into the repository root. +# The wheel is a ZIP file. We skip .dist-info/ and mssql_py_core.libs/. +TARGET_DIR="$REPO_ROOT" +CORE_DIR="$TARGET_DIR/mssql_py_core" + +# Clean previous extraction +if [ -d "$CORE_DIR" ]; then + rm -rf "$CORE_DIR" + echo "Cleaned previous mssql_py_core/ directory" +fi -# Verify import +echo "Extracting mssql_py_core from wheel into: $TARGET_DIR" + +python3 -c " +import zipfile, os, sys + +wheel_path = '$MATCHING_WHEEL' +target_dir = '$TARGET_DIR' +extracted = 0 + +with zipfile.ZipFile(wheel_path, 'r') as zf: + for entry in zf.namelist(): + # Skip dist-info metadata + if '.dist-info/' in entry: + continue + # Skip vendored shared libraries (system deps expected) + if entry.startswith('mssql_py_core.libs/'): + continue + if entry.startswith('mssql_py_core/'): + out_path = os.path.join(target_dir, entry) + if entry.endswith('/'): + os.makedirs(out_path, exist_ok=True) + continue + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, 'wb') as f: + f.write(zf.read(entry)) + extracted += 1 + print(f' Extracted: {entry}') + +if extracted == 0: + print('ERROR: No mssql_py_core files found in wheel', file=sys.stderr) + sys.exit(1) + +print(f'Extracted {extracted} file(s) into {target_dir}') +" + +# Verify import works (from repo root so mssql_py_core/ is on sys.path) echo "Verifying mssql_py_core import..." +pushd "$REPO_ROOT" > /dev/null python3 -c "import mssql_py_core; print(f'mssql_py_core loaded successfully: {dir(mssql_py_core)}')" +popd > /dev/null # Cleanup rm -rf "$OUTPUT_DIR" -echo "=== mssql_py_core installed successfully ===" +echo "=== mssql_py_core extracted successfully ===" diff --git a/setup.py b/setup.py index 352e2b20..fbb812be 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,25 @@ import os import sys +from pathlib import Path + from setuptools import setup, find_packages from setuptools.dist import Distribution from wheel.bdist_wheel import bdist_wheel +PROJECT_ROOT = Path(__file__).resolve().parent + +# --------------------------------------------------------------------------- # Custom distribution to force platform-specific wheel +# --------------------------------------------------------------------------- class BinaryDistribution(Distribution): def has_ext_modules(self): return True +# --------------------------------------------------------------------------- +# Platform / Python tag helpers +# --------------------------------------------------------------------------- def get_platform_info(): """Get platform-specific architecture and platform tag information.""" if sys.platform.startswith("win"): @@ -53,7 +62,56 @@ def get_platform_info(): ) -# Custom bdist_wheel command to override platform tag +# --------------------------------------------------------------------------- +# mssql_py_core validation +# --------------------------------------------------------------------------- +def validate_mssql_py_core(): + """Validate that mssql_py_core has been extracted into the project root. + + Expects ``/mssql_py_core/`` to contain: + - ``__init__.py`` + - At least one native extension (``.pyd`` on Windows, ``.so`` on Linux/macOS) + + The extraction is performed by ``eng/scripts/install-mssql-py-core.ps1`` + (Windows) or ``eng/scripts/install-mssql-py-core.sh`` (Linux/macOS) + and must be run before ``setup.py bdist_wheel``. + + Returns True if mssql_py_core is present and valid, False otherwise. + """ + core_dir = PROJECT_ROOT / "mssql_py_core" + + if not core_dir.is_dir(): + print( + "NOTE: mssql_py_core/ directory not found in project root. " + "Run eng/scripts/install-mssql-py-core to extract it before building." + ) + return False + + # Check for __init__.py + if not (core_dir / "__init__.py").is_file(): + print("WARNING: mssql_py_core/__init__.py not found.") + return False + + # Check for native extension (.pyd on Windows, .so on Linux/macOS) + ext = ".pyd" if sys.platform.startswith("win") else ".so" + native_files = list(core_dir.glob(f"mssql_py_core*{ext}")) + if not native_files: + print( + f"WARNING: No mssql_py_core native extension ({ext}) found " + f"in mssql_py_core/. Run eng/scripts/install-mssql-py-core to extract it." + ) + return False + + for f in native_files: + print(f" Found mssql_py_core native extension: {f.name}") + + print("mssql_py_core validation: OK") + return True + + +# --------------------------------------------------------------------------- +# Custom bdist_wheel – sets platform tag +# --------------------------------------------------------------------------- class CustomBdistWheel(bdist_wheel): def finalize_options(self): # Call the original finalize_options first to initialize self.bdist_dir @@ -65,6 +123,10 @@ def finalize_options(self): print(f"Setting wheel platform tag to: {self.plat_name} (arch: {arch})") +# --------------------------------------------------------------------------- +# Package discovery +# --------------------------------------------------------------------------- + # Find all packages in the current directory packages = find_packages() @@ -72,6 +134,13 @@ def finalize_options(self): arch, platform_tag = get_platform_info() print(f"Detected architecture: {arch} (platform tag: {platform_tag})") +# Validate that mssql_py_core has been pre-extracted into /mssql_py_core/. +# Extraction is done by eng/scripts/install-mssql-py-core before running setup.py. +core_extracted = validate_mssql_py_core() + +if core_extracted: + packages.append("mssql_py_core") + # Add platform-specific packages if sys.platform.startswith("win"): packages.extend( @@ -94,6 +163,28 @@ def finalize_options(self): ] ) +# --------------------------------------------------------------------------- +# package_data – binaries to include in the wheel +# --------------------------------------------------------------------------- +package_data = { + # Include PYD and DLL files inside mssql_python, exclude YML files + "mssql_python": [ + "py.typed", # Marker file for PEP 561 typing support + "ddbc_bindings.cp*.pyd", # Include all PYD files + "ddbc_bindings.cp*.so", # Include all SO files + "libs/*", + "libs/**/*", + "*.dll", + ], +} + +if core_extracted: + # Include the native extension (.pyd on Windows, .so on Linux/macOS) + package_data["mssql_py_core"] = [ + "mssql_py_core.cp*.pyd", + "mssql_py_core.cp*.so", + ] + setup( name="mssql-python", version="1.3.0", @@ -104,17 +195,7 @@ def finalize_options(self): author_email="mssql-python@microsoft.com", url="https://github.com/microsoft/mssql-python", packages=packages, - package_data={ - # Include PYD and DLL files inside mssql_python, exclude YML files - "mssql_python": [ - "py.typed", # Marker file for PEP 561 typing support - "ddbc_bindings.cp*.pyd", # Include all PYD files - "ddbc_bindings.cp*.so", # Include all SO files - "libs/*", - "libs/**/*", - "*.dll", - ] - }, + package_data=package_data, include_package_data=True, # Requires >= Python 3.10 python_requires=">=3.10", diff --git a/tests/test_019_bulkcopy.py b/tests/test_019_bulkcopy.py new file mode 100644 index 00000000..c2233bd5 --- /dev/null +++ b/tests/test_019_bulkcopy.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Basic integration tests for bulkcopy via the mssql_python driver.""" +import pytest + + +def test_connection_and_cursor(cursor): + """Test that connection and cursor work correctly.""" + cursor.execute("SELECT 1 AS connected") + result = cursor.fetchone() + assert result[0] == 1 + + +def test_insert_and_fetch(cursor): + """Test basic insert and fetch operations.""" + table_name = "mssql_python_test_basic" + + # Create table + cursor.execute(f"IF OBJECT_ID('{table_name}', 'U') IS NOT NULL DROP TABLE {table_name}") + cursor.execute(f"CREATE TABLE {table_name} (id INT, name NVARCHAR(50))") + cursor.connection.commit() + + # Insert data + cursor.execute(f"INSERT INTO {table_name} (id, name) VALUES (?, ?)", (1, "Alice")) + cursor.execute(f"INSERT INTO {table_name} (id, name) VALUES (?, ?)", (2, "Bob")) + + # Fetch and verify + cursor.execute(f"SELECT id, name FROM {table_name} ORDER BY id") + rows = cursor.fetchall() + + assert len(rows) == 2 + assert rows[0][0] == 1 and rows[0][1] == "Alice" + assert rows[1][0] == 2 and rows[1][1] == "Bob" + + # Cleanup + cursor.execute(f"DROP TABLE {table_name}") + + +def test_bulkcopy_basic(cursor): + """Test basic bulkcopy operation via mssql_python driver with auto-mapping. + + Uses automatic column mapping (columns mapped by ordinal position). + """ + table_name = "mssql_python_bulkcopy_test" + + # Create table + cursor.execute(f"IF OBJECT_ID('{table_name}', 'U') IS NOT NULL DROP TABLE {table_name}") + cursor.execute(f"CREATE TABLE {table_name} (id INT, name VARCHAR(50), value FLOAT)") + cursor.connection.commit() + + # Prepare test data - columns match table order (id, name, value) + data = [ + (1, "Alice", 100.5), + (2, "Bob", 200.75), + (3, "Charlie", 300.25), + ] + + # Perform bulkcopy with auto-mapping (no column_mappings specified) + # Using explicit timeout parameter instead of kwargs + result = cursor._bulkcopy(table_name, data, timeout=60) + + # Verify result + assert result is not None + assert result["rows_copied"] == 3 + + # Verify data was inserted correctly + cursor.execute(f"SELECT id, name, value FROM {table_name} ORDER BY id") + rows = cursor.fetchall() + + assert len(rows) == 3 + assert rows[0][0] == 1 and rows[0][1] == "Alice" and abs(rows[0][2] - 100.5) < 0.01 + assert rows[1][0] == 2 and rows[1][1] == "Bob" and abs(rows[1][2] - 200.75) < 0.01 + assert rows[2][0] == 3 and rows[2][1] == "Charlie" and abs(rows[2][2] - 300.25) < 0.01 + + # Cleanup + cursor.execute(f"DROP TABLE {table_name}") From 9b29902d0e63f5dd3157611956360004525ece86 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:45:16 -0800 Subject: [PATCH 08/35] REF: Remove mssql_py_core repackaging steps and update package installation for Linux and Windows builds --- .../jobs/consolidate-artifacts-job.yml | 15 ------ .../stages/build-linux-single-stage.yml | 14 +++-- .../stages/build-macos-single-stage.yml | 5 ++ .../stages/build-windows-single-stage.yml | 7 +++ eng/pipelines/build-whl-pipeline.yml | 52 +++++++++---------- 5 files changed, 47 insertions(+), 46 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 1d700dda..095dbdb9 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -91,21 +91,6 @@ jobs: fi displayName: 'Consolidate wheels from all platforms' - # Repackage wheels: inject mssql_py_core binaries from NuGet into each mssql-python wheel - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - addToPath: true - displayName: 'Use Python 3.12 for repackaging' - - - bash: | - set -e - echo "Repackaging mssql-python wheels with mssql_py_core binaries..." - python $(Build.SourcesDirectory)/eng/scripts/repackage-with-mssql-py-core.py \ - --wheel-dir $(ob_outputDirectory)/dist \ - --version-file $(Build.SourcesDirectory)/eng/versions/mssql-py-core.version - displayName: 'Inject mssql_py_core into wheels' - # Optional: Consolidate native bindings for reference - bash: | set -e diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index 99dbbd46..1d001f18 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -126,10 +126,10 @@ stages: set -euxo pipefail if command -v dnf >/dev/null 2>&1; then dnf -y update || true - dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true + dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true elif command -v yum >/dev/null 2>&1; then yum -y update || true - yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true + yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true fi gcc --version || true cmake --version || true @@ -138,7 +138,7 @@ stages: docker exec build-$(LINUX_TAG)-$(ARCH) sh -lc ' set -euxo pipefail apk update || true - apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache || true + apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl || true gcc --version || true cmake --version || true ' @@ -198,6 +198,7 @@ stages: PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; test -x $PY || { echo "Python $PY missing - skipping"; exit 0; }; ln -sf $PY /usr/local/bin/python; + ln -sf $PY /usr/local/bin/python3; echo "Using: $(python --version)"; # Step 2: Install build dependencies @@ -208,6 +209,9 @@ stages: cd /workspace/mssql_python/pybind; bash build.sh; + # Step 3.5: Extract mssql_py_core from NuGet for this Python version + bash /workspace/eng/scripts/install-mssql-py-core.sh; + # Step 4: Build wheel echo "Building wheel package..."; cd /workspace; @@ -263,6 +267,7 @@ stages: PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; test -x $PY || { echo "Python $PY missing - skipping"; exit 0; }; ln -sf $PY /usr/local/bin/python; + ln -sf $PY /usr/local/bin/python3; echo "Using: $(python --version)"; # Step 2: Install build dependencies @@ -273,6 +278,9 @@ stages: cd /workspace/mssql_python/pybind; bash build.sh; + # Step 3.5: Extract mssql_py_core from NuGet for this Python version + bash /workspace/eng/scripts/install-mssql-py-core.sh; + # Step 4: Build wheel echo "Building wheel package..."; cd /workspace; diff --git a/OneBranchPipelines/stages/build-macos-single-stage.yml b/OneBranchPipelines/stages/build-macos-single-stage.yml index 71ccaf60..005053d1 100644 --- a/OneBranchPipelines/stages/build-macos-single-stage.yml +++ b/OneBranchPipelines/stages/build-macos-single-stage.yml @@ -186,6 +186,11 @@ stages: # ========================= # WHEEL BUILD # ========================= + # Extract mssql_py_core from NuGet into repo root before building the wheel + - script: | + bash $(Build.SourcesDirectory)/eng/scripts/install-mssql-py-core.sh + displayName: 'Install mssql_py_core from NuGet' + # Build wheel package from setup.py # Wheel filename: mssql_python-X.Y.Z-cp3XX-cp3XX-macosx_XX_X_universal2.whl # bdist_wheel = build binary wheel distribution (contains pre-compiled .so) diff --git a/OneBranchPipelines/stages/build-windows-single-stage.yml b/OneBranchPipelines/stages/build-windows-single-stage.yml index 3523267c..97377029 100644 --- a/OneBranchPipelines/stages/build-windows-single-stage.yml +++ b/OneBranchPipelines/stages/build-windows-single-stage.yml @@ -287,6 +287,13 @@ stages: TargetFolder: '$(Build.SourcesDirectory)\apiScan\pdbs\windows\py$(shortPyVer)\$(targetArch)' displayName: 'Copy PDB to ApiScan directory' + # Extract mssql_py_core from NuGet into repo root before building the wheel + - task: PowerShell@2 + displayName: 'Install mssql_py_core from NuGet' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\eng\scripts\install-mssql-py-core.ps1' + # Build Python wheel package from source distribution # ARCHITECTURE environment variable controls target platform tagging - script: | diff --git a/eng/pipelines/build-whl-pipeline.yml b/eng/pipelines/build-whl-pipeline.yml index db9f68a4..cfda6dba 100644 --- a/eng/pipelines/build-whl-pipeline.yml +++ b/eng/pipelines/build-whl-pipeline.yml @@ -168,6 +168,13 @@ jobs: TargetFolder: '$(Build.ArtifactStagingDirectory)\all-pdbs' displayName: 'Place PDB file into artifacts directory' + # Extract mssql_py_core from NuGet into repo root before building the wheel + - task: PowerShell@2 + displayName: 'Install mssql_py_core from NuGet' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\eng\scripts\install-mssql-py-core.ps1' + # Build wheel package for the current architecture - script: | python -m pip install --upgrade pip @@ -182,17 +189,7 @@ jobs: SourceFolder: '$(Build.SourcesDirectory)\dist' Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)\dist' - displayName: 'Collect wheel package' - - # Repackage wheel: inject mssql_py_core binary from NuGet - - task: PowerShell@2 - displayName: 'Inject mssql_py_core into wheel' - inputs: - targetType: 'inline' - script: | - python "$(Build.SourcesDirectory)\eng\scripts\repackage-with-mssql-py-core.py" ` - --wheel-dir "$(Build.ArtifactStagingDirectory)\dist" ` - --version-file "$(Build.SourcesDirectory)\eng\versions\mssql-py-core.version" + displayName: 'Collect wheel package' # Publish the collected .pyd file(s) as build artifacts - task: PublishBuildArtifacts@1 @@ -352,6 +349,11 @@ jobs: env: DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' + # Extract mssql_py_core from NuGet into repo root before building the wheel + - script: | + bash $(Build.SourcesDirectory)/eng/scripts/install-mssql-py-core.sh + displayName: 'Install mssql_py_core from NuGet' + # Build wheel package for universal2 - script: | python -m pip install --upgrade pip @@ -366,13 +368,6 @@ jobs: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)/dist' displayName: 'Collect wheel package' - - # Repackage wheel: inject mssql_py_core binary from NuGet - - script: | - python $(Build.SourcesDirectory)/eng/scripts/repackage-with-mssql-py-core.py \ - --wheel-dir "$(Build.ArtifactStagingDirectory)/dist" \ - --version-file "$(Build.SourcesDirectory)/eng/versions/mssql-py-core.version" - displayName: 'Inject mssql_py_core into wheel' # Publish the collected .so file(s) as build artifacts - task: PublishBuildArtifacts@1 @@ -461,10 +456,10 @@ jobs: if command -v dnf >/dev/null 2>&1; then dnf -y update || true # Toolchain + CMake + unixODBC headers + Kerberos + keyutils + ccache - dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true + dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true elif command -v yum >/dev/null 2>&1; then yum -y update || true - yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true + yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true else echo "No dnf/yum found in manylinux image" >&2 fi @@ -480,7 +475,7 @@ jobs: set -euxo pipefail apk update || true # Toolchain + CMake + unixODBC headers + Kerberos + keyutils + ccache - apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache || true + apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl || true # Quick visibility for logs echo "---- tool versions ----" @@ -507,12 +502,16 @@ jobs: PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; test -x \$PY || { echo 'Python \$PY missing'; exit 0; } # skip if not present ln -sf \$PY /usr/local/bin/python; + ln -sf \$PY /usr/local/bin/python3; python -m pip install -U pip setuptools wheel pybind11; echo 'python:' \$(python -V); which python; # 👉 run from the directory that has CMakeLists.txt cd /workspace/mssql_python/pybind; bash build.sh; + # Extract mssql_py_core from NuGet for this Python version + bash /workspace/eng/scripts/install-mssql-py-core.sh; + # back to repo root to build the wheel cd /workspace; python setup.py bdist_wheel; @@ -527,12 +526,16 @@ jobs: PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; test -x \$PY || { echo 'Python \$PY missing'; exit 0; } # skip if not present ln -sf \$PY /usr/local/bin/python; + ln -sf \$PY /usr/local/bin/python3; python -m pip install -U pip setuptools wheel pybind11; echo 'python:' \$(python -V); which python; # 👉 run from the directory that has CMakeLists.txt cd /workspace/mssql_python/pybind; bash build.sh; + # Extract mssql_py_core from NuGet for this Python version + bash /workspace/eng/scripts/install-mssql-py-core.sh; + # back to repo root to build the wheel cd /workspace; python setup.py bdist_wheel; @@ -582,13 +585,6 @@ jobs: find "$(Build.ArtifactStagingDirectory)/ddbc-bindings/$(LINUX_TAG)-$(ARCH)" -maxdepth 1 -type f ! -name "*.so" -delete || true displayName: 'Copy wheels and .so back to host' - # Repackage wheels: inject mssql_py_core binaries from NuGet - - script: | - python3 $(Build.SourcesDirectory)/eng/scripts/repackage-with-mssql-py-core.py \ - --wheel-dir "$(Build.ArtifactStagingDirectory)/dist" \ - --version-file "$(Build.SourcesDirectory)/eng/versions/mssql-py-core.version" - displayName: 'Inject mssql_py_core into wheels' - # Cleanup container - script: | docker stop build-$(LINUX_TAG)-$(ARCH) || true From 2381b4a835787f8e7093422811ea60ab189d0c1f Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Fri, 20 Feb 2026 18:31:08 -0800 Subject: [PATCH 09/35] FEAT: Enhance connstr_to_pycore_params to handle boolean parameters for ODBC connection strings --- mssql_python/helpers.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mssql_python/helpers.py b/mssql_python/helpers.py index 8c7b9060..12745651 100644 --- a/mssql_python/helpers.py +++ b/mssql_python/helpers.py @@ -311,6 +311,13 @@ def connstr_to_pycore_params(params: dict) -> dict: "keep_alive", "keep_alive_interval", } + # connection.rs extracts these with .extract::() — Python bool required, + # not the "Yes"/"No" strings that ODBC connection strings use. + bool_keys = { + "trust_server_certificate", + "multi_subnet_failover", + "mars_enabled", + } pycore_params: dict = {} @@ -324,16 +331,16 @@ def connstr_to_pycore_params(params: dict) -> dict: if pycore_key in pycore_params: continue - # ODBC values are always strings; py-core expects native types for int keys. - # Boolean params (trust_server_certificate, multi_subnet_failover) are passed - # as strings — all Yes/No validation is in connection.rs for single-location - # consistency with Encrypt, ApplicationIntent, IPAddressPreference, etc. + # ODBC values are always strings; py-core expects native Python types. if pycore_key in int_keys: # Numeric params (timeouts, packet size, etc.) — skip on bad input try: pycore_params[pycore_key] = int(raw_value) except (ValueError, TypeError): pass # let py-core fall back to its compiled-in default + elif pycore_key in bool_keys: + # Boolean params — connection.rs uses .extract::() + pycore_params[pycore_key] = raw_value.lower() in ("yes", "true", "1") else: # String params (server, database, encryption, etc.) — pass through pycore_params[pycore_key] = raw_value From 4c3bf6246a030dd5634328b4f95634729e890d7d Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Fri, 20 Feb 2026 18:33:49 -0800 Subject: [PATCH 10/35] REF: Clean up whitespace in bulkcopy integration tests for improved readability --- tests/test_019_bulkcopy.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/test_019_bulkcopy.py b/tests/test_019_bulkcopy.py index c2233bd5..fa5e2fda 100644 --- a/tests/test_019_bulkcopy.py +++ b/tests/test_019_bulkcopy.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. """Basic integration tests for bulkcopy via the mssql_python driver.""" + import pytest @@ -15,7 +16,7 @@ def test_connection_and_cursor(cursor): def test_insert_and_fetch(cursor): """Test basic insert and fetch operations.""" table_name = "mssql_python_test_basic" - + # Create table cursor.execute(f"IF OBJECT_ID('{table_name}', 'U') IS NOT NULL DROP TABLE {table_name}") cursor.execute(f"CREATE TABLE {table_name} (id INT, name NVARCHAR(50))") @@ -24,26 +25,26 @@ def test_insert_and_fetch(cursor): # Insert data cursor.execute(f"INSERT INTO {table_name} (id, name) VALUES (?, ?)", (1, "Alice")) cursor.execute(f"INSERT INTO {table_name} (id, name) VALUES (?, ?)", (2, "Bob")) - + # Fetch and verify cursor.execute(f"SELECT id, name FROM {table_name} ORDER BY id") rows = cursor.fetchall() - + assert len(rows) == 2 assert rows[0][0] == 1 and rows[0][1] == "Alice" assert rows[1][0] == 2 and rows[1][1] == "Bob" - + # Cleanup cursor.execute(f"DROP TABLE {table_name}") def test_bulkcopy_basic(cursor): """Test basic bulkcopy operation via mssql_python driver with auto-mapping. - + Uses automatic column mapping (columns mapped by ordinal position). """ table_name = "mssql_python_bulkcopy_test" - + # Create table cursor.execute(f"IF OBJECT_ID('{table_name}', 'U') IS NOT NULL DROP TABLE {table_name}") cursor.execute(f"CREATE TABLE {table_name} (id INT, name VARCHAR(50), value FLOAT)") @@ -55,23 +56,23 @@ def test_bulkcopy_basic(cursor): (2, "Bob", 200.75), (3, "Charlie", 300.25), ] - + # Perform bulkcopy with auto-mapping (no column_mappings specified) # Using explicit timeout parameter instead of kwargs result = cursor._bulkcopy(table_name, data, timeout=60) - + # Verify result assert result is not None assert result["rows_copied"] == 3 - + # Verify data was inserted correctly cursor.execute(f"SELECT id, name, value FROM {table_name} ORDER BY id") rows = cursor.fetchall() - + assert len(rows) == 3 assert rows[0][0] == 1 and rows[0][1] == "Alice" and abs(rows[0][2] - 100.5) < 0.01 assert rows[1][0] == 2 and rows[1][1] == "Bob" and abs(rows[1][2] - 200.75) < 0.01 assert rows[2][0] == 3 and rows[2][1] == "Charlie" and abs(rows[2][2] - 300.25) < 0.01 - + # Cleanup cursor.execute(f"DROP TABLE {table_name}") From 610a6cb8d329b613017d7898248bae95356617c6 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Fri, 20 Feb 2026 22:52:24 -0800 Subject: [PATCH 11/35] REF: Update installation scripts to clarify skipped files and update version number --- eng/scripts/install-mssql-py-core.ps1 | 7 +++++-- eng/scripts/install-mssql-py-core.sh | 9 ++++++--- eng/versions/mssql-py-core.version | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index 58ddf160..e8188204 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -128,7 +128,9 @@ Write-Host "Found matching wheel: $($matchingWheel.Name)" # Extract mssql_py_core/ from the wheel into the repository root. # The wheel is a ZIP file containing mssql_py_core/__init__.py and # mssql_py_core/mssql_py_core... -# We skip .dist-info/ and mssql_py_core.libs/ (system deps expected). +# We skip .dist-info/ metadata. +# mssql_py_core.libs/ won't exist because auditwheel=skip is set in pyproject.toml, +# but we skip it defensively in case an older wheel is used. $targetDir = $repoRoot $coreDir = Join-Path $targetDir "mssql_py_core" @@ -154,7 +156,8 @@ with zipfile.ZipFile(wheel_path, 'r') as zf: # Skip dist-info metadata if '.dist-info/' in entry: continue - # Skip vendored shared libraries (system deps expected) + # Skip vendored shared libraries if present (auditwheel=skip means + # they won't be in the wheel; system OpenSSL is used at runtime) if entry.startswith('mssql_py_core.libs/'): continue if entry.startswith('mssql_py_core/'): diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index 1914879f..3ce2d9f3 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -5,7 +5,7 @@ # # The extracted files are placed at /mssql_py_core/ which contains: # - __init__.py -# - mssql_py_core..so (native extension) +# - mssql_py_core..so (native extension — links system OpenSSL) # # This script is used identically for: # - Local development (dev runs it after build.sh) @@ -171,7 +171,9 @@ fi echo "Found matching wheel: $(basename "$MATCHING_WHEEL")" # Extract mssql_py_core/ from the wheel into the repository root. -# The wheel is a ZIP file. We skip .dist-info/ and mssql_py_core.libs/. +# The wheel is a ZIP file. We skip .dist-info/ metadata. +# mssql_py_core.libs/ won't exist because auditwheel=skip is set in pyproject.toml, +# but we skip it defensively in case an older wheel is used. TARGET_DIR="$REPO_ROOT" CORE_DIR="$TARGET_DIR/mssql_py_core" @@ -195,7 +197,8 @@ with zipfile.ZipFile(wheel_path, 'r') as zf: # Skip dist-info metadata if '.dist-info/' in entry: continue - # Skip vendored shared libraries (system deps expected) + # Skip vendored shared libraries if present (auditwheel=skip means + # they won't be in the wheel; system OpenSSL is used at runtime) if entry.startswith('mssql_py_core.libs/'): continue if entry.startswith('mssql_py_core/'): diff --git a/eng/versions/mssql-py-core.version b/eng/versions/mssql-py-core.version index a733259d..e39d4a5c 100644 --- a/eng/versions/mssql-py-core.version +++ b/eng/versions/mssql-py-core.version @@ -1 +1 @@ -0.1.0-dev.20260218.140230 +0.1.0-dev.20260221.140693 From 3b185e3540c20abc34d11d4806fe30f7d4869cef Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Fri, 20 Feb 2026 22:59:41 -0800 Subject: [PATCH 12/35] REF: Update platform tagging for Linux wheels in installation scripts --- eng/scripts/install-mssql-py-core.ps1 | 6 ++++-- eng/scripts/install-mssql-py-core.sh | 21 ++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index e8188204..4fb271b7 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -63,9 +63,11 @@ switch ($platform) { } } 'linux' { + # auditwheel=skip in pyproject.toml means glibc wheels are tagged + # linux_* (not manylinux_2_28_*) because auditwheel repair is skipped. switch -Regex ($arch) { - 'x86_64|amd64' { $wheelPlatform = "manylinux_2_28_x86_64" } - 'aarch64|arm64' { $wheelPlatform = "manylinux_2_28_aarch64" } + 'x86_64|amd64' { $wheelPlatform = "linux_x86_64" } + 'aarch64|arm64' { $wheelPlatform = "linux_aarch64" } default { throw "Unsupported Linux architecture: $arch" } } } diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index 3ce2d9f3..db643898 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -75,15 +75,22 @@ case "$PLATFORM" in fi if $IS_MUSL; then - LIBC="musllinux_1_2" + # musllinux wheels keep the musllinux_1_2 platform tag + case "$ARCH" in + x86_64|amd64) WHEEL_PLATFORM="musllinux_1_2_x86_64" ;; + aarch64|arm64) WHEEL_PLATFORM="musllinux_1_2_aarch64" ;; + *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; + esac else - LIBC="manylinux_2_28" + # auditwheel=skip in pyproject.toml means manylinux wheels are + # tagged linux_* (not manylinux_2_28_*) because auditwheel repair + # — which renames the tag — is skipped. + case "$ARCH" in + x86_64|amd64) WHEEL_PLATFORM="linux_x86_64" ;; + aarch64|arm64) WHEEL_PLATFORM="linux_aarch64" ;; + *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; + esac fi - case "$ARCH" in - x86_64|amd64) WHEEL_PLATFORM="${LIBC}_x86_64" ;; - aarch64|arm64) WHEEL_PLATFORM="${LIBC}_aarch64" ;; - *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; - esac ;; darwin) WHEEL_PLATFORM="macosx_15_0_universal2" From fcc6852cb2aab72d660bc351f247a5a39cf95945 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sat, 21 Feb 2026 00:06:33 -0800 Subject: [PATCH 13/35] REF: Update manylinux tag versions in platform info and installation script --- eng/scripts/install-mssql-py-core.sh | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index db643898..346e8029 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -83,7 +83,7 @@ case "$PLATFORM" in esac else # auditwheel=skip in pyproject.toml means manylinux wheels are - # tagged linux_* (not manylinux_2_28_*) because auditwheel repair + # tagged linux_* (not manylinux_2_34_*) because auditwheel repair # — which renames the tag — is skipped. case "$ARCH" in x86_64|amd64) WHEEL_PLATFORM="linux_x86_64" ;; diff --git a/setup.py b/setup.py index fbb812be..550f40f7 100644 --- a/setup.py +++ b/setup.py @@ -53,9 +53,9 @@ def get_platform_info(): is_musl = libc_name == "" or "musl" in libc_name.lower() if target_arch == "x86_64": - return "x86_64", "musllinux_1_2_x86_64" if is_musl else "manylinux_2_28_x86_64" + return "x86_64", "musllinux_1_2_x86_64" if is_musl else "manylinux_2_34_x86_64" elif target_arch in ["aarch64", "arm64"]: - return "aarch64", "musllinux_1_2_aarch64" if is_musl else "manylinux_2_28_aarch64" + return "aarch64", "musllinux_1_2_aarch64" if is_musl else "manylinux_2_34_aarch64" else: raise OSError( f"Unsupported architecture '{target_arch}' for Linux; expected 'x86_64' or 'aarch64'." From a82eb052c49731899005d18266fee7fa3d37ae67 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sat, 21 Feb 2026 01:16:06 -0800 Subject: [PATCH 14/35] BUMP: Update version to 0.1.0-dev.20260221.140717 --- eng/versions/mssql-py-core.version | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/versions/mssql-py-core.version b/eng/versions/mssql-py-core.version index e39d4a5c..35233fdc 100644 --- a/eng/versions/mssql-py-core.version +++ b/eng/versions/mssql-py-core.version @@ -1 +1,2 @@ -0.1.0-dev.20260221.140693 +0.1.0-dev.20260221.140717 + From 73b2adbd8fd1a7e5e4e3fec71b1736651d246546 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sat, 21 Feb 2026 03:22:57 -0800 Subject: [PATCH 15/35] BUMP: Update version to 0.1.0-dev.20260221.140732 --- eng/versions/mssql-py-core.version | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/versions/mssql-py-core.version b/eng/versions/mssql-py-core.version index 35233fdc..be503e23 100644 --- a/eng/versions/mssql-py-core.version +++ b/eng/versions/mssql-py-core.version @@ -1,2 +1 @@ -0.1.0-dev.20260221.140717 - +0.1.0-dev.20260221.140732 From 63fe2c8fa33fe6987d87e6ab0cf6029b303d1053 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sat, 21 Feb 2026 16:14:01 -0800 Subject: [PATCH 16/35] REF: Update manylinux version and install mssql_py_core in build stages --- .../build-release-package-pipeline.yml | 2 +- .../stages/build-linux-single-stage.yml | 6 ++++-- .../stages/build-macos-single-stage.yml | 13 +++++++++---- .../stages/build-windows-single-stage.yml | 18 +++++++++++------- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 2c0c64ce..eff7fd9d 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -401,7 +401,7 @@ extends: # - musllinux: musl-based (Alpine Linux) # Architectures: x86_64 (AMD/Intel), aarch64 (ARM64) # Each stage: - # 1. Starts PyPA Docker container (manylinux_2_28 or musllinux_1_2) + # 1. Starts PyPA Docker container (manylinux_2_34 or musllinux_1_2) # 2. Starts SQL Server Docker container # 3. For each Python version (cp310-cp314): # a. Builds .so native extension diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index 1d001f18..1f368349 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -104,10 +104,12 @@ stages: - script: | # Determine image based on LINUX_TAG and ARCH + # manylinux_2_34 = AlmaLinux 9 (glibc 2.34, OpenSSL 3.x) + # Required because mssql_py_core is linked against libssl.so.3 if [[ "$(LINUX_TAG)" == "musllinux" ]]; then IMAGE="quay.io/pypa/musllinux_1_2_$(ARCH)" else - IMAGE="quay.io/pypa/manylinux_2_28_$(ARCH)" + IMAGE="quay.io/pypa/manylinux_2_34_$(ARCH)" fi docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ @@ -138,7 +140,7 @@ stages: docker exec build-$(LINUX_TAG)-$(ARCH) sh -lc ' set -euxo pipefail apk update || true - apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl || true + apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl openssl openssl-dev || true gcc --version || true cmake --version || true ' diff --git a/OneBranchPipelines/stages/build-macos-single-stage.yml b/OneBranchPipelines/stages/build-macos-single-stage.yml index 005053d1..352d6863 100644 --- a/OneBranchPipelines/stages/build-macos-single-stage.yml +++ b/OneBranchPipelines/stages/build-macos-single-stage.yml @@ -169,6 +169,15 @@ stages: env: DB_PASSWORD: $(DB_PASSWORD) + # ========================= + # MSSQL_PY_CORE INSTALLATION + # ========================= + # Extract mssql_py_core from NuGet into repo root BEFORE testing + # Required for bulkcopy tests which use mssql_py_core native bindings + - script: | + bash $(Build.SourcesDirectory)/eng/scripts/install-mssql-py-core.sh + displayName: 'Install mssql_py_core from NuGet' + # ========================= # TESTING # ========================= @@ -186,10 +195,6 @@ stages: # ========================= # WHEEL BUILD # ========================= - # Extract mssql_py_core from NuGet into repo root before building the wheel - - script: | - bash $(Build.SourcesDirectory)/eng/scripts/install-mssql-py-core.sh - displayName: 'Install mssql_py_core from NuGet' # Build wheel package from setup.py # Wheel filename: mssql_python-X.Y.Z-cp3XX-cp3XX-macosx_XX_X_universal2.whl diff --git a/OneBranchPipelines/stages/build-windows-single-stage.yml b/OneBranchPipelines/stages/build-windows-single-stage.yml index 97377029..972fe494 100644 --- a/OneBranchPipelines/stages/build-windows-single-stage.yml +++ b/OneBranchPipelines/stages/build-windows-single-stage.yml @@ -238,6 +238,17 @@ stages: displayName: 'Build PYD for $(targetArch)' continueOnError: false + # ========================= + # MSSQL_PY_CORE INSTALLATION + # ========================= + # Extract mssql_py_core from NuGet into repo root BEFORE testing + # Required for bulkcopy tests which use mssql_py_core native bindings + - task: PowerShell@2 + displayName: 'Install mssql_py_core from NuGet' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\eng\scripts\install-mssql-py-core.ps1' + # ========================= # TESTING # ========================= @@ -287,13 +298,6 @@ stages: TargetFolder: '$(Build.SourcesDirectory)\apiScan\pdbs\windows\py$(shortPyVer)\$(targetArch)' displayName: 'Copy PDB to ApiScan directory' - # Extract mssql_py_core from NuGet into repo root before building the wheel - - task: PowerShell@2 - displayName: 'Install mssql_py_core from NuGet' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\eng\scripts\install-mssql-py-core.ps1' - # Build Python wheel package from source distribution # ARCHITECTURE environment variable controls target platform tagging - script: | From 9f0137c061ab7baedcb16d0313d1c1fd06192d3b Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 15:02:42 -0800 Subject: [PATCH 17/35] BUMP: Update version to 0.1.0-dev.20260222.140833 --- eng/versions/mssql-py-core.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/versions/mssql-py-core.version b/eng/versions/mssql-py-core.version index be503e23..56099240 100644 --- a/eng/versions/mssql-py-core.version +++ b/eng/versions/mssql-py-core.version @@ -1 +1 @@ -0.1.0-dev.20260221.140732 +0.1.0-dev.20260222.140833 From 3cb5b664d2904ba5575422d5146282de98ef5bc9 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 15:42:35 -0800 Subject: [PATCH 18/35] REF: Remove unused boolean key handling in connection string parameter extraction --- mssql_python/helpers.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/mssql_python/helpers.py b/mssql_python/helpers.py index 12745651..8c7b9060 100644 --- a/mssql_python/helpers.py +++ b/mssql_python/helpers.py @@ -311,13 +311,6 @@ def connstr_to_pycore_params(params: dict) -> dict: "keep_alive", "keep_alive_interval", } - # connection.rs extracts these with .extract::() — Python bool required, - # not the "Yes"/"No" strings that ODBC connection strings use. - bool_keys = { - "trust_server_certificate", - "multi_subnet_failover", - "mars_enabled", - } pycore_params: dict = {} @@ -331,16 +324,16 @@ def connstr_to_pycore_params(params: dict) -> dict: if pycore_key in pycore_params: continue - # ODBC values are always strings; py-core expects native Python types. + # ODBC values are always strings; py-core expects native types for int keys. + # Boolean params (trust_server_certificate, multi_subnet_failover) are passed + # as strings — all Yes/No validation is in connection.rs for single-location + # consistency with Encrypt, ApplicationIntent, IPAddressPreference, etc. if pycore_key in int_keys: # Numeric params (timeouts, packet size, etc.) — skip on bad input try: pycore_params[pycore_key] = int(raw_value) except (ValueError, TypeError): pass # let py-core fall back to its compiled-in default - elif pycore_key in bool_keys: - # Boolean params — connection.rs uses .extract::() - pycore_params[pycore_key] = raw_value.lower() in ("yes", "true", "1") else: # String params (server, database, encryption, etc.) — pass through pycore_params[pycore_key] = raw_value From 95797fa0717befe9e1618bac97cae25308a3ea1e Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 16:02:03 -0800 Subject: [PATCH 19/35] Add installation step for mssql_py_core from NuGet wheels --- eng/pipelines/pr-validation-pipeline.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 73fbb6c9..c2481e7b 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -2301,6 +2301,12 @@ jobs: ./build.sh codecov displayName: 'Build pybind bindings with coverage' + # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) + - script: | + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + displayName: 'Install mssql_py_core from NuGet wheels' + - script: | # Generate unified coverage (Python + C++) chmod +x ./generate_codecov.sh From a5d1b8b17ace2c3ad2022950f7d0be38fe1304f4 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 16:45:20 -0800 Subject: [PATCH 20/35] REF: Improve mssql_py_core validation and error handling in setup.py --- eng/scripts/repackage-with-mssql-py-core.py | 311 -------------------- setup.py | 38 +-- 2 files changed, 14 insertions(+), 335 deletions(-) delete mode 100644 eng/scripts/repackage-with-mssql-py-core.py diff --git a/eng/scripts/repackage-with-mssql-py-core.py b/eng/scripts/repackage-with-mssql-py-core.py deleted file mode 100644 index 59915e1b..00000000 --- a/eng/scripts/repackage-with-mssql-py-core.py +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python3 -""" -Repackage mssql_py_core binaries into mssql-python wheels. - -Downloads the mssql-py-core-wheels NuGet package, extracts the matching -mssql_py_core native extension for each mssql-python wheel, and injects -it into the wheel so that mssql_py_core is bundled inside mssql-python. - -Usage: - python repackage-with-mssql-py-core.py --wheel-dir [--nuget-dir ] [--feed-url ] - -The NuGet package version is read from eng/versions/mssql-py-core.version. -If --nuget-dir is provided, it skips the download and uses pre-extracted wheels. -""" - -import argparse -import csv -import hashlib -import io -import json -import os -import re -import shutil -import sys -import tempfile -import urllib.request -import zipfile -from base64 import urlsafe_b64encode -from pathlib import Path - - -def parse_wheel_filename(filename: str) -> dict | None: - """Parse a wheel filename into its components.""" - # Format: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl - m = re.match( - r"^(?P[A-Za-z0-9_]+)-(?P[^-]+)" - r"(?:-(?P\d[^-]*))?" - r"-(?P[^-]+)-(?P[^-]+)-(?P.+)\.whl$", - filename, - ) - return m.groupdict() if m else None - - -def compute_record_hash(data: bytes) -> str: - """Compute sha256 hash in RECORD format: sha256=.""" - digest = hashlib.sha256(data).digest() - return "sha256=" + urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") - - -def find_matching_core_wheel( - mssql_python_whl: str, core_wheels_dir: Path -) -> Path | None: - """Find the mssql_py_core wheel matching a mssql-python wheel's platform tags.""" - parsed = parse_wheel_filename(os.path.basename(mssql_python_whl)) - if not parsed: - return None - - python_tag = parsed["python"] # e.g., cp313 - platform_tag = parsed["platform"] # e.g., win_amd64 - - # Look for matching core wheel - pattern = f"mssql_py_core-*-{python_tag}-{python_tag}-{platform_tag}.whl" - matches = list(core_wheels_dir.glob(pattern)) - - if not matches: - # Try without exact abi match (some wheels use cpXXX for both) - for whl in core_wheels_dir.glob("mssql_py_core-*.whl"): - core_parsed = parse_wheel_filename(whl.name) - if core_parsed and core_parsed["platform"] == platform_tag and core_parsed["python"] == python_tag: - return whl - return None - - return matches[0] - - -def inject_core_into_wheel( - mssql_python_whl: Path, core_whl: Path, output_dir: Path -) -> Path: - """ - Inject mssql_py_core files from core_whl into mssql_python_whl. - - Copies the mssql_py_core/ package and mssql_py_core.libs/ (if present) - into the mssql-python wheel and updates the RECORD file. - """ - output_path = output_dir / mssql_python_whl.name - - # Read the core wheel contents we want to inject - inject_files: dict[str, bytes] = {} - with zipfile.ZipFile(core_whl, "r") as core_zip: - for entry in core_zip.namelist(): - # Include mssql_py_core/ package files and mssql_py_core.libs/ - if entry.startswith("mssql_py_core/") or entry.startswith("mssql_py_core.libs/"): - inject_files[entry] = core_zip.read(entry) - - if not inject_files: - print(f" WARNING: No mssql_py_core files found in {core_whl.name}") - shutil.copy2(mssql_python_whl, output_path) - return output_path - - # Build new wheel with injected files - with zipfile.ZipFile(mssql_python_whl, "r") as src_zip: - with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as dst_zip: - # Find the dist-info directory name for mssql-python - dist_info_dir = None - for name in src_zip.namelist(): - if name.endswith("/RECORD"): - dist_info_dir = name.rsplit("/", 1)[0] - break - - if not dist_info_dir: - raise ValueError(f"No RECORD found in {mssql_python_whl.name}") - - record_path = f"{dist_info_dir}/RECORD" - new_record_entries: list[str] = [] - - # Copy all existing files (except RECORD, we'll regenerate it) - for item in src_zip.infolist(): - if item.filename == record_path: - # Parse existing RECORD entries (we'll append to them) - existing_record = src_zip.read(item.filename).decode("utf-8") - for line in existing_record.strip().split("\n"): - line = line.strip() - if line and not line.startswith(record_path): - new_record_entries.append(line) - continue - - data = src_zip.read(item.filename) - dst_zip.writestr(item, data) - - # Inject mssql_py_core files - for filename, data in inject_files.items(): - dst_zip.writestr(filename, data) - file_hash = compute_record_hash(data) - new_record_entries.append(f"{filename},{file_hash},{len(data)}") - - # Write updated RECORD - new_record_entries.append(f"{record_path},,") - record_content = "\n".join(new_record_entries) + "\n" - dst_zip.writestr(record_path, record_content) - - return output_path - - -def download_nuget_package( - feed_url: str, package_id: str, version: str, output_dir: Path -) -> Path: - """Download and extract a NuGet package from an Azure Artifacts feed.""" - print(f"Resolving NuGet feed: {feed_url}") - with urllib.request.urlopen(feed_url) as resp: - feed_index = json.loads(resp.read().decode()) - - package_base_url = None - for resource in feed_index.get("resources", []): - if "PackageBaseAddress" in resource.get("@type", ""): - package_base_url = resource["@id"] - break - - if not package_base_url: - raise RuntimeError("Could not resolve PackageBaseAddress from feed") - - print(f"Package base URL: {package_base_url}") - - version_lower = version.lower() - pkg_id_lower = package_id.lower() - nupkg_url = f"{package_base_url}{pkg_id_lower}/{version_lower}/{pkg_id_lower}.{version_lower}.nupkg" - - nupkg_path = output_dir / f"{pkg_id_lower}.{version_lower}.nupkg" - print(f"Downloading: {nupkg_url}") - urllib.request.urlretrieve(nupkg_url, nupkg_path) - size_mb = nupkg_path.stat().st_size / (1024 * 1024) - print(f"Downloaded: {nupkg_path.name} ({size_mb:.1f} MB)") - - # Extract (NuGet packages are ZIP files) - extract_dir = output_dir / "extracted" - with zipfile.ZipFile(nupkg_path, "r") as z: - z.extractall(extract_dir) - - wheels_dir = extract_dir / "wheels" - if not wheels_dir.is_dir(): - raise RuntimeError( - f"No 'wheels' directory in NuGet package. Contents: {list(extract_dir.iterdir())}" - ) - - return wheels_dir - - -def main(): - parser = argparse.ArgumentParser( - description="Repackage mssql_py_core binaries into mssql-python wheels" - ) - parser.add_argument( - "--wheel-dir", - required=True, - help="Directory containing mssql-python wheels to repackage", - ) - parser.add_argument( - "--nuget-dir", - help="Directory containing pre-extracted mssql_py_core wheels (skips download)", - ) - parser.add_argument( - "--feed-url", - default="https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json", - help="NuGet v3 feed URL", - ) - parser.add_argument( - "--version-file", - help="Path to mssql-py-core.version file (default: auto-detect from repo)", - ) - parser.add_argument( - "--output-dir", - help="Output directory for repackaged wheels (default: overwrite in place)", - ) - args = parser.parse_args() - - wheel_dir = Path(args.wheel_dir) - if not wheel_dir.is_dir(): - print(f"ERROR: Wheel directory not found: {wheel_dir}") - sys.exit(1) - - # Find mssql-python wheels - mssql_python_wheels = sorted(wheel_dir.glob("mssql_python-*.whl")) - if not mssql_python_wheels: - print(f"ERROR: No mssql_python-*.whl files found in {wheel_dir}") - sys.exit(1) - - print(f"Found {len(mssql_python_wheels)} mssql-python wheel(s)") - - # Get or download mssql_py_core wheels - temp_dir = None - if args.nuget_dir: - core_wheels_dir = Path(args.nuget_dir) - else: - # Find version file - if args.version_file: - version_file = Path(args.version_file) - else: - # Auto-detect from repo root - script_dir = Path(__file__).resolve().parent - version_file = script_dir / ".." / "versions" / "mssql-py-core.version" - - if not version_file.is_file(): - print(f"ERROR: Version file not found: {version_file}") - sys.exit(1) - - version = version_file.read_text().strip() - print(f"mssql-py-core version: {version}") - - temp_dir = Path(tempfile.mkdtemp(prefix="mssql-py-core-")) - core_wheels_dir = download_nuget_package( - args.feed_url, "mssql-py-core-wheels", version, temp_dir - ) - - print(f"Core wheels directory: {core_wheels_dir}") - core_wheel_count = len(list(core_wheels_dir.glob("*.whl"))) - print(f"Available mssql_py_core wheels: {core_wheel_count}") - - # Set up output directory - if args.output_dir: - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - else: - # In-place: use temp output, then replace originals - output_dir = Path(tempfile.mkdtemp(prefix="repackaged-")) - - # Repackage each wheel - repackaged = 0 - skipped = 0 - for whl in mssql_python_wheels: - print(f"\nProcessing: {whl.name}") - core_whl = find_matching_core_wheel(whl.name, core_wheels_dir) - - if core_whl is None: - parsed = parse_wheel_filename(whl.name) - platform = parsed["platform"] if parsed else "unknown" - # musllinux wheels have no matching core wheel yet — skip gracefully - if "musllinux" in platform: - print(f" SKIP: No musllinux mssql_py_core wheel available for {platform}") - if not args.output_dir: - # In-place mode: copy unchanged - shutil.copy2(whl, output_dir / whl.name) - skipped += 1 - continue - else: - print(f" ERROR: No matching mssql_py_core wheel for {whl.name}") - sys.exit(1) - - print(f" Matched: {core_whl.name}") - result = inject_core_into_wheel(whl, core_whl, output_dir) - print(f" Repackaged: {result.name}") - repackaged += 1 - - # If in-place mode, replace originals - if not args.output_dir: - for repackaged_whl in output_dir.glob("*.whl"): - dest = wheel_dir / repackaged_whl.name - shutil.move(str(repackaged_whl), str(dest)) - shutil.rmtree(output_dir, ignore_errors=True) - - # Cleanup temp NuGet download - if temp_dir: - shutil.rmtree(temp_dir, ignore_errors=True) - - print(f"\n{'='*50}") - print(f"Repackaging complete!") - print(f" Repackaged: {repackaged} wheel(s)") - print(f" Skipped: {skipped} wheel(s) (no matching core wheel)") - print(f"{'='*50}") - - -if __name__ == "__main__": - main() diff --git a/setup.py b/setup.py index 550f40f7..d30839da 100644 --- a/setup.py +++ b/setup.py @@ -76,37 +76,33 @@ def validate_mssql_py_core(): (Windows) or ``eng/scripts/install-mssql-py-core.sh`` (Linux/macOS) and must be run before ``setup.py bdist_wheel``. - Returns True if mssql_py_core is present and valid, False otherwise. + Raises SystemExit if mssql_py_core is missing or invalid. """ core_dir = PROJECT_ROOT / "mssql_py_core" if not core_dir.is_dir(): - print( - "NOTE: mssql_py_core/ directory not found in project root. " + sys.exit( + "ERROR: mssql_py_core/ directory not found in project root. " "Run eng/scripts/install-mssql-py-core to extract it before building." ) - return False # Check for __init__.py if not (core_dir / "__init__.py").is_file(): - print("WARNING: mssql_py_core/__init__.py not found.") - return False + sys.exit("ERROR: mssql_py_core/__init__.py not found.") # Check for native extension (.pyd on Windows, .so on Linux/macOS) ext = ".pyd" if sys.platform.startswith("win") else ".so" native_files = list(core_dir.glob(f"mssql_py_core*{ext}")) if not native_files: - print( - f"WARNING: No mssql_py_core native extension ({ext}) found " + sys.exit( + f"ERROR: No mssql_py_core native extension ({ext}) found " f"in mssql_py_core/. Run eng/scripts/install-mssql-py-core to extract it." ) - return False for f in native_files: print(f" Found mssql_py_core native extension: {f.name}") print("mssql_py_core validation: OK") - return True # --------------------------------------------------------------------------- @@ -136,10 +132,8 @@ def finalize_options(self): # Validate that mssql_py_core has been pre-extracted into /mssql_py_core/. # Extraction is done by eng/scripts/install-mssql-py-core before running setup.py. -core_extracted = validate_mssql_py_core() - -if core_extracted: - packages.append("mssql_py_core") +validate_mssql_py_core() +packages.append("mssql_py_core") # Add platform-specific packages if sys.platform.startswith("win"): @@ -167,23 +161,19 @@ def finalize_options(self): # package_data – binaries to include in the wheel # --------------------------------------------------------------------------- package_data = { - # Include PYD and DLL files inside mssql_python, exclude YML files "mssql_python": [ - "py.typed", # Marker file for PEP 561 typing support - "ddbc_bindings.cp*.pyd", # Include all PYD files - "ddbc_bindings.cp*.so", # Include all SO files + "py.typed", + "ddbc_bindings.cp*.pyd", + "ddbc_bindings.cp*.so", "libs/*", "libs/**/*", "*.dll", ], -} - -if core_extracted: - # Include the native extension (.pyd on Windows, .so on Linux/macOS) - package_data["mssql_py_core"] = [ + "mssql_py_core": [ "mssql_py_core.cp*.pyd", "mssql_py_core.cp*.so", - ] + ], +} setup( name="mssql-python", From 4b8fb4a8ee0b00fdca20bca330410e86c5db6a29 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 16:55:11 -0800 Subject: [PATCH 21/35] Refactor Python command usage in build scripts to use 'python' instead of 'python3' --- .../stages/build-linux-single-stage.yml | 4 +--- eng/scripts/install-mssql-py-core.sh | 14 +++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index 1f368349..397ea532 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -140,7 +140,7 @@ stages: docker exec build-$(LINUX_TAG)-$(ARCH) sh -lc ' set -euxo pipefail apk update || true - apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl openssl openssl-dev || true + apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl || true gcc --version || true cmake --version || true ' @@ -200,7 +200,6 @@ stages: PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; test -x $PY || { echo "Python $PY missing - skipping"; exit 0; }; ln -sf $PY /usr/local/bin/python; - ln -sf $PY /usr/local/bin/python3; echo "Using: $(python --version)"; # Step 2: Install build dependencies @@ -269,7 +268,6 @@ stages: PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; test -x $PY || { echo "Python $PY missing - skipping"; exit 0; }; ln -sf $PY /usr/local/bin/python; - ln -sf $PY /usr/local/bin/python3; echo "Using: $(python --version)"; # Step 2: Install build dependencies diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index 346e8029..17f00a0b 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -51,9 +51,9 @@ fi echo "Using version from $VERSION_FILE: $PACKAGE_VERSION" # Determine platform info -PY_VERSION=$(python3 -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") -PLATFORM=$(python3 -c "import platform; print(platform.system().lower())") -ARCH=$(python3 -c "import platform; print(platform.machine().lower())") +PY_VERSION=$(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") +PLATFORM=$(python -c "import platform; print(platform.system().lower())") +ARCH=$(python -c "import platform; print(platform.machine().lower())") echo "Python: $PY_VERSION | Platform: $PLATFORM | Arch: $ARCH" @@ -112,7 +112,7 @@ mkdir -p "$OUTPUT_DIR" echo "Resolving feed: $FEED_URL" FEED_INDEX=$(curl -sS "$FEED_URL") -PACKAGE_BASE_URL=$(echo "$FEED_INDEX" | python3 -c " +PACKAGE_BASE_URL=$(echo "$FEED_INDEX" | python -c " import json, sys data = json.load(sys.stdin) for r in data['resources']: @@ -149,7 +149,7 @@ mkdir -p "$EXTRACT_DIR" if command -v unzip &>/dev/null; then unzip -q "$NUPKG_PATH" -d "$EXTRACT_DIR" else - python3 -c "import zipfile; zipfile.ZipFile('$NUPKG_PATH').extractall('$EXTRACT_DIR')" + python -c "import zipfile; zipfile.ZipFile('$NUPKG_PATH').extractall('$EXTRACT_DIR')" fi # Find matching wheel @@ -192,7 +192,7 @@ fi echo "Extracting mssql_py_core from wheel into: $TARGET_DIR" -python3 -c " +python -c " import zipfile, os, sys wheel_path = '$MATCHING_WHEEL' @@ -229,7 +229,7 @@ print(f'Extracted {extracted} file(s) into {target_dir}') # Verify import works (from repo root so mssql_py_core/ is on sys.path) echo "Verifying mssql_py_core import..." pushd "$REPO_ROOT" > /dev/null -python3 -c "import mssql_py_core; print(f'mssql_py_core loaded successfully: {dir(mssql_py_core)}')" +python -c "import mssql_py_core; print(f'mssql_py_core loaded successfully: {dir(mssql_py_core)}')" popd > /dev/null # Cleanup From f48e325b58f9fde6059d959940c5849c7fd8a257 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 17:32:48 -0800 Subject: [PATCH 22/35] Refactor install-mssql-py-core.sh into functions with extracted Python scripts --- eng/scripts/extract_wheel.py | 57 +++++ eng/scripts/install-mssql-py-core.sh | 360 ++++++++++++--------------- eng/scripts/resolve_nuget_feed.py | 25 ++ 3 files changed, 239 insertions(+), 203 deletions(-) create mode 100644 eng/scripts/extract_wheel.py create mode 100644 eng/scripts/resolve_nuget_feed.py diff --git a/eng/scripts/extract_wheel.py b/eng/scripts/extract_wheel.py new file mode 100644 index 00000000..be3808f0 --- /dev/null +++ b/eng/scripts/extract_wheel.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +"""Extract mssql_py_core package files from a wheel into a target directory. + +Wheels are ZIP files. This script extracts only mssql_py_core/ entries, +skipping .dist-info metadata and vendored .libs directories. + +Usage: + python extract_wheel.py +""" +import os +import sys +import zipfile + + +def extract(wheel_path: str, target_dir: str) -> int: + """Extract mssql_py_core/ entries from a wheel, return count of files extracted.""" + extracted = 0 + with zipfile.ZipFile(wheel_path, "r") as zf: + for entry in zf.namelist(): + if ".dist-info/" in entry: + continue + if entry.startswith("mssql_py_core.libs/"): + continue + if not entry.startswith("mssql_py_core/"): + continue + + out_path = os.path.join(target_dir, entry) + if entry.endswith("/"): + os.makedirs(out_path, exist_ok=True) + continue + + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, "wb") as f: + f.write(zf.read(entry)) + extracted += 1 + print(f" Extracted: {entry}") + + return extracted + + +def main() -> None: + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(2) + + wheel_path, target_dir = sys.argv[1], sys.argv[2] + count = extract(wheel_path, target_dir) + + if count == 0: + print("ERROR: No mssql_py_core files found in wheel", file=sys.stderr) + sys.exit(1) + + print(f"Extracted {count} file(s) into {target_dir}") + + +if __name__ == "__main__": + main() diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index 17f00a0b..c3ac508b 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -5,7 +5,7 @@ # # The extracted files are placed at /mssql_py_core/ which contains: # - __init__.py -# - mssql_py_core..so (native extension — links system OpenSSL) +# - mssql_py_core..so (native extension) # # This script is used identically for: # - Local development (dev runs it after build.sh) @@ -19,10 +19,160 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +PYTHON="${PYTHON:-$(command -v python || command -v python3)}" + +read_version() { + local version_file="$REPO_ROOT/eng/versions/mssql-py-core.version" + if [ ! -f "$version_file" ]; then + echo "ERROR: Version file not found: $version_file" + exit 1 + fi + PACKAGE_VERSION=$(tr -d '[:space:]' < "$version_file") + if [ -z "$PACKAGE_VERSION" ]; then + echo "ERROR: Version file is empty: $version_file" + exit 1 + fi + echo "Version: $PACKAGE_VERSION" +} + +detect_platform() { + read -r PY_VERSION PLATFORM ARCH <<< "$(python -c " +import sys, platform +v = sys.version_info +print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().lower()}')" + )" + + echo "Python: $PY_VERSION | Platform: $PLATFORM | Arch: $ARCH" + + case "$PLATFORM" in + linux) + case "$ARCH" in + x86_64|amd64) ARCH_TAG="x86_64" ;; + aarch64|arm64) ARCH_TAG="aarch64" ;; + *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; + esac + + # Detect musl libc (Alpine) vs glibc. + # ldd --version exits 1 on musl, so capture output instead of piping. + local ldd_output + ldd_output=$(ldd --version 2>&1 || true) + if echo "$ldd_output" | grep -qi musl || [ -f /etc/alpine-release ]; then + WHEEL_PLATFORM="musllinux_1_2_${ARCH_TAG}" + else + # auditwheel=skip: wheels are tagged linux_* not manylinux_2_34_* + WHEEL_PLATFORM="linux_${ARCH_TAG}" + fi + ;; + darwin) + WHEEL_PLATFORM="macosx_15_0_universal2" + ;; + *) + echo "Unsupported platform: $PLATFORM" + exit 1 + ;; + esac + + WHEEL_PATTERN="mssql_py_core-*-${PY_VERSION}-${PY_VERSION}-${WHEEL_PLATFORM}.whl" + echo "Wheel pattern: $WHEEL_PATTERN" +} + +download_nupkg() { + local feed_url="$1" + local output_dir="$2" + + rm -rf "$output_dir" + mkdir -p "$output_dir" + + echo "Resolving feed: $feed_url" + local feed_index + feed_index=$(curl -sS "$feed_url") + + PACKAGE_BASE_URL=$(echo "$feed_index" | python "$SCRIPT_DIR/resolve_nuget_feed.py") + if [ -z "$PACKAGE_BASE_URL" ]; then + echo "ERROR: Could not resolve PackageBaseAddress from feed" + exit 1 + fi + + local package_id="mssql-py-core-wheels" + local version_lower + version_lower=$(echo "$PACKAGE_VERSION" | tr '[:upper:]' '[:lower:]') + + NUPKG_URL="${PACKAGE_BASE_URL}${package_id}/${version_lower}/${package_id}.${version_lower}.nupkg" + NUPKG_PATH="$output_dir/${package_id}.${version_lower}.nupkg" + + echo "Downloading: $NUPKG_URL" + curl -sSL -o "$NUPKG_PATH" "$NUPKG_URL" + + local filesize + filesize=$(wc -c < "$NUPKG_PATH") + echo "Downloaded: $NUPKG_PATH ($filesize bytes)" + + if [ "$filesize" -eq 0 ]; then + echo "ERROR: Downloaded file is empty" + exit 1 + fi +} + +find_matching_wheel() { + local output_dir="$1" + local extract_dir="$output_dir/extracted" + + mkdir -p "$extract_dir" + if command -v unzip &>/dev/null; then + unzip -q "$NUPKG_PATH" -d "$extract_dir" + else + python -c "import zipfile; zipfile.ZipFile('$NUPKG_PATH').extractall('$extract_dir')" + fi + + local wheels_dir="$extract_dir/wheels" + if [ ! -d "$wheels_dir" ]; then + echo "ERROR: No 'wheels' directory found in NuGet package" + ls -la "$extract_dir" + exit 1 + fi + + MATCHING_WHEEL=$(find "$wheels_dir" -name "$WHEEL_PATTERN" | head -1) + if [ -z "$MATCHING_WHEEL" ]; then + echo "Available wheels:" + ls "$wheels_dir"/*.whl 2>/dev/null || echo " (none)" + # On musllinux (Alpine), no wheels may be available yet + if echo "$WHEEL_PLATFORM" | grep -q "musllinux"; then + echo "WARNING: No musllinux wheel found for: $WHEEL_PATTERN" + echo "mssql_py_core is not yet available for musllinux -- skipping." + rm -rf "$output_dir" + exit 0 + fi + echo "ERROR: No wheel found matching: $WHEEL_PATTERN" + exit 1 + fi + + echo "Found: $(basename "$MATCHING_WHEEL")" +} + +extract_and_verify() { + local target_dir="$REPO_ROOT" + local core_dir="$target_dir/mssql_py_core" + + if [ -d "$core_dir" ]; then + rm -rf "$core_dir" + echo "Cleaned previous mssql_py_core/" + fi + + python "$SCRIPT_DIR/extract_wheel.py" "$MATCHING_WHEEL" "$target_dir" + + echo "Verifying import..." + pushd "$target_dir" > /dev/null + python -c "import mssql_py_core; print(f'mssql_py_core loaded: {dir(mssql_py_core)}')" + popd > /dev/null +} + +# --- main --- + FEED_URL="${FEED_URL:-https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json}" OUTPUT_DIR="${TMPDIR:-/tmp}/mssql-py-core-wheels" -# Parse arguments while [[ $# -gt 0 ]]; do case "$1" in --feed-url) FEED_URL="$2"; shift 2 ;; @@ -32,207 +182,11 @@ done echo "=== Install mssql_py_core from NuGet wheel package ===" -# Determine repository root (two levels up from this script) -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" - -# Read version from pinned version file (required) -VERSION_FILE="$REPO_ROOT/eng/versions/mssql-py-core.version" -if [ ! -f "$VERSION_FILE" ]; then - echo "ERROR: Version file not found: $VERSION_FILE" - echo "This file must exist and contain the mssql-py-core-wheels NuGet package version." - exit 1 -fi -PACKAGE_VERSION=$(tr -d '[:space:]' < "$VERSION_FILE") -if [ -z "$PACKAGE_VERSION" ]; then - echo "ERROR: Version file is empty: $VERSION_FILE" - exit 1 -fi -echo "Using version from $VERSION_FILE: $PACKAGE_VERSION" - -# Determine platform info -PY_VERSION=$(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") -PLATFORM=$(python -c "import platform; print(platform.system().lower())") -ARCH=$(python -c "import platform; print(platform.machine().lower())") - -echo "Python: $PY_VERSION | Platform: $PLATFORM | Arch: $ARCH" - -# Map to wheel platform tags -# Detect musl (Alpine) vs glibc for Linux -case "$PLATFORM" in - linux) - # Detect musl libc (Alpine) vs glibc — multiple methods for robustness - # Note: ldd --version exits with code 1 on musl, which combined with - # pipefail causes the grep pipeline to fail. Use a variable instead. - IS_MUSL=false - LDD_OUTPUT=$(ldd --version 2>&1 || true) - if echo "$LDD_OUTPUT" | grep -qi musl; then - IS_MUSL=true - elif [ -f /etc/alpine-release ]; then - IS_MUSL=true - elif ls /lib/ld-musl-* >/dev/null 2>&1; then - IS_MUSL=true - fi - - if $IS_MUSL; then - # musllinux wheels keep the musllinux_1_2 platform tag - case "$ARCH" in - x86_64|amd64) WHEEL_PLATFORM="musllinux_1_2_x86_64" ;; - aarch64|arm64) WHEEL_PLATFORM="musllinux_1_2_aarch64" ;; - *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; - esac - else - # auditwheel=skip in pyproject.toml means manylinux wheels are - # tagged linux_* (not manylinux_2_34_*) because auditwheel repair - # — which renames the tag — is skipped. - case "$ARCH" in - x86_64|amd64) WHEEL_PLATFORM="linux_x86_64" ;; - aarch64|arm64) WHEEL_PLATFORM="linux_aarch64" ;; - *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; - esac - fi - ;; - darwin) - WHEEL_PLATFORM="macosx_15_0_universal2" - ;; - *) - echo "Unsupported platform: $PLATFORM" - exit 1 - ;; -esac - -WHEEL_PATTERN="mssql_py_core-*-${PY_VERSION}-${PY_VERSION}-${WHEEL_PLATFORM}.whl" -echo "Looking for wheel matching: $WHEEL_PATTERN" +read_version +detect_platform +download_nupkg "$FEED_URL" "$OUTPUT_DIR" +find_matching_wheel "$OUTPUT_DIR" +extract_and_verify -# Setup temp directory -rm -rf "$OUTPUT_DIR" -mkdir -p "$OUTPUT_DIR" - -# Resolve NuGet v3 feed -echo "Resolving feed: $FEED_URL" -FEED_INDEX=$(curl -sS "$FEED_URL") - -PACKAGE_BASE_URL=$(echo "$FEED_INDEX" | python -c " -import json, sys -data = json.load(sys.stdin) -for r in data['resources']: - if 'PackageBaseAddress' in r.get('@type', ''): - print(r['@id']) - break -") - -if [ -z "$PACKAGE_BASE_URL" ]; then - echo "Could not resolve PackageBaseAddress from feed" - exit 1 -fi -echo "Package base: $PACKAGE_BASE_URL" - -PACKAGE_ID="mssql-py-core-wheels" - -VERSION_LOWER=$(echo "$PACKAGE_VERSION" | tr '[:upper:]' '[:lower:]') -NUPKG_URL="${PACKAGE_BASE_URL}${PACKAGE_ID}/${VERSION_LOWER}/${PACKAGE_ID}.${VERSION_LOWER}.nupkg" -NUPKG_PATH="$OUTPUT_DIR/${PACKAGE_ID}.${VERSION_LOWER}.nupkg" - -echo "Downloading: $NUPKG_URL" -curl -sSL -o "$NUPKG_PATH" "$NUPKG_URL" -FILESIZE=$(stat -c%s "$NUPKG_PATH" 2>/dev/null || stat -f%z "$NUPKG_PATH" 2>/dev/null || echo "unknown") -echo "Downloaded: $NUPKG_PATH ($FILESIZE bytes)" - -if [ "$FILESIZE" = "0" ] || [ "$FILESIZE" = "unknown" ]; then - echo "ERROR: Downloaded file is empty or could not determine size" - exit 1 -fi - -# Extract NuGet (ZIP format) — use python if unzip is not available -EXTRACT_DIR="$OUTPUT_DIR/extracted" -mkdir -p "$EXTRACT_DIR" -if command -v unzip &>/dev/null; then - unzip -q "$NUPKG_PATH" -d "$EXTRACT_DIR" -else - python -c "import zipfile; zipfile.ZipFile('$NUPKG_PATH').extractall('$EXTRACT_DIR')" -fi - -# Find matching wheel -WHEELS_DIR="$EXTRACT_DIR/wheels" -if [ ! -d "$WHEELS_DIR" ]; then - echo "No 'wheels' directory found in NuGet package" - ls -la "$EXTRACT_DIR" - exit 1 -fi - -MATCHING_WHEEL=$(find "$WHEELS_DIR" -name "$WHEEL_PATTERN" | head -1) -if [ -z "$MATCHING_WHEEL" ]; then - echo "Available wheels:" - ls "$WHEELS_DIR"/*.whl 2>/dev/null || echo " (none)" - # On musllinux (Alpine), no wheels may be available yet — skip gracefully - if echo "$WHEEL_PLATFORM" | grep -q "musllinux"; then - echo "WARNING: No musllinux wheel found matching pattern: $WHEEL_PATTERN" - echo "mssql_py_core is not yet available for musllinux — skipping installation." - rm -rf "$OUTPUT_DIR" - exit 0 - fi - echo "ERROR: No wheel found matching pattern: $WHEEL_PATTERN" - exit 1 -fi - -echo "Found matching wheel: $(basename "$MATCHING_WHEEL")" - -# Extract mssql_py_core/ from the wheel into the repository root. -# The wheel is a ZIP file. We skip .dist-info/ metadata. -# mssql_py_core.libs/ won't exist because auditwheel=skip is set in pyproject.toml, -# but we skip it defensively in case an older wheel is used. -TARGET_DIR="$REPO_ROOT" -CORE_DIR="$TARGET_DIR/mssql_py_core" - -# Clean previous extraction -if [ -d "$CORE_DIR" ]; then - rm -rf "$CORE_DIR" - echo "Cleaned previous mssql_py_core/ directory" -fi - -echo "Extracting mssql_py_core from wheel into: $TARGET_DIR" - -python -c " -import zipfile, os, sys - -wheel_path = '$MATCHING_WHEEL' -target_dir = '$TARGET_DIR' -extracted = 0 - -with zipfile.ZipFile(wheel_path, 'r') as zf: - for entry in zf.namelist(): - # Skip dist-info metadata - if '.dist-info/' in entry: - continue - # Skip vendored shared libraries if present (auditwheel=skip means - # they won't be in the wheel; system OpenSSL is used at runtime) - if entry.startswith('mssql_py_core.libs/'): - continue - if entry.startswith('mssql_py_core/'): - out_path = os.path.join(target_dir, entry) - if entry.endswith('/'): - os.makedirs(out_path, exist_ok=True) - continue - os.makedirs(os.path.dirname(out_path), exist_ok=True) - with open(out_path, 'wb') as f: - f.write(zf.read(entry)) - extracted += 1 - print(f' Extracted: {entry}') - -if extracted == 0: - print('ERROR: No mssql_py_core files found in wheel', file=sys.stderr) - sys.exit(1) - -print(f'Extracted {extracted} file(s) into {target_dir}') -" - -# Verify import works (from repo root so mssql_py_core/ is on sys.path) -echo "Verifying mssql_py_core import..." -pushd "$REPO_ROOT" > /dev/null -python -c "import mssql_py_core; print(f'mssql_py_core loaded successfully: {dir(mssql_py_core)}')" -popd > /dev/null - -# Cleanup rm -rf "$OUTPUT_DIR" - echo "=== mssql_py_core extracted successfully ===" diff --git a/eng/scripts/resolve_nuget_feed.py b/eng/scripts/resolve_nuget_feed.py new file mode 100644 index 00000000..175fc9ba --- /dev/null +++ b/eng/scripts/resolve_nuget_feed.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +"""Resolve the PackageBaseAddress URL from a NuGet v3 feed index. + +Reads NuGet v3 service index JSON from stdin and prints the +PackageBaseAddress endpoint URL to stdout. + +Usage: + curl -sS "$FEED_URL" | python resolve_nuget_feed.py +""" +import json +import sys + + +def main() -> None: + data = json.load(sys.stdin) + for resource in data["resources"]: + if "PackageBaseAddress" in resource.get("@type", ""): + print(resource["@id"]) + return + print("ERROR: No PackageBaseAddress found in feed index", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() From a790860fa8f9ce6c5cfb5421dfa3b69fd633b5b6 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 17:34:53 -0800 Subject: [PATCH 23/35] Refactor install-mssql-py-core.ps1 into functions, reuse extract_wheel.py --- eng/scripts/install-mssql-py-core.ps1 | 270 +++++++++++--------------- 1 file changed, 108 insertions(+), 162 deletions(-) diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index 4fb271b7..6eeb85aa 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -4,17 +4,6 @@ feed and extracts the matching mssql_py_core binary into the repository root so that 'import mssql_py_core' works when running from the source tree. - The extracted files are placed at /mssql_py_core/ which contains: - - __init__.py - - mssql_py_core.. (native extension) - - This script is used identically for: - - Local development (dev runs it after build.bat/build.sh) - - PR validation pipelines - - Official build pipelines (before setup.py bdist_wheel) - - The package version is read from eng/versions/mssql-py-core.version (required). - .PARAMETER FeedUrl The NuGet v3 feed URL. This is a public feed — no authentication required. @@ -29,170 +18,127 @@ param( ) $ErrorActionPreference = 'Stop' +$ScriptDir = $PSScriptRoot +$RepoRoot = (Get-Item "$ScriptDir\..\..").FullName -Write-Host "=== Install mssql_py_core from NuGet wheel package ===" +function Read-PackageVersion { + $versionFile = Join-Path $RepoRoot "eng\versions\mssql-py-core.version" + if (-not (Test-Path $versionFile)) { + throw "Version file not found: $versionFile" + } + $script:PackageVersion = (Get-Content $versionFile -Raw).Trim() + if (-not $script:PackageVersion) { + throw "Version file is empty: $versionFile" + } + Write-Host "Version: $script:PackageVersion" +} -# Determine repository root (two levels up from this script) -$repoRoot = (Get-Item "$PSScriptRoot\..\..").FullName +function Get-PlatformInfo { + # Single python call to get version, platform, and arch + $info = & python -c "import sys, platform; v = sys.version_info; print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().lower()}')" + if ($LASTEXITCODE -ne 0) { throw "Failed to detect Python platform info" } -# Read version from pinned version file (required) -$versionFile = Join-Path $repoRoot "eng\versions\mssql-py-core.version" -if (-not (Test-Path $versionFile)) { - throw "Version file not found: $versionFile. This file must exist and contain the mssql-py-core-wheels NuGet package version." -} -$PackageVersion = (Get-Content $versionFile -Raw).Trim() -if (-not $PackageVersion) { - throw "Version file is empty: $versionFile" -} -Write-Host "Using version from $versionFile : $PackageVersion" - -# Determine platform info -$pyVersion = & python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')" -$platform = & python -c "import platform; print(platform.system().lower())" -$arch = & python -c "import platform; print(platform.machine().lower())" - -Write-Host "Python: $pyVersion | Platform: $platform | Arch: $arch" - -# Map to wheel filename platform tags -switch ($platform) { - 'windows' { - switch -Regex ($arch) { - 'amd64|x86_64' { $wheelPlatform = "win_amd64" } - 'arm64|aarch64' { $wheelPlatform = "win_arm64" } - default { throw "Unsupported Windows architecture: $arch" } - } - } - 'linux' { - # auditwheel=skip in pyproject.toml means glibc wheels are tagged - # linux_* (not manylinux_2_28_*) because auditwheel repair is skipped. - switch -Regex ($arch) { - 'x86_64|amd64' { $wheelPlatform = "linux_x86_64" } - 'aarch64|arm64' { $wheelPlatform = "linux_aarch64" } - default { throw "Unsupported Linux architecture: $arch" } - } + $parts = $info -split ' ' + $script:PyVersion = $parts[0] + $script:Platform = $parts[1] + $script:Arch = $parts[2] + + Write-Host "Python: $script:PyVersion | Platform: $script:Platform | Arch: $script:Arch" + + # Normalize arch tag + $archTag = switch -Regex ($script:Arch) { + 'amd64|x86_64' { 'x86_64' } + 'arm64|aarch64' { 'aarch64' } + default { throw "Unsupported architecture: $script:Arch" } } - 'darwin' { - $wheelPlatform = "macosx_15_0_universal2" + + $script:WheelPlatform = switch ($script:Platform) { + 'windows' { "win_$($archTag -replace 'x86_64','amd64')" } + 'linux' { "linux_$archTag" } + 'darwin' { 'macosx_15_0_universal2' } + default { throw "Unsupported platform: $script:Platform" } } - default { throw "Unsupported platform: $platform" } -} -$wheelPattern = "mssql_py_core-*-$pyVersion-$pyVersion-$wheelPlatform.whl" -Write-Host "Looking for wheel matching: $wheelPattern" - -# Create temp directory -if (Test-Path $OutputDir) { Remove-Item $OutputDir -Recurse -Force } -New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null - -# Download NuGet package -$nugetDir = "$OutputDir\nuget" -New-Item -ItemType Directory -Path $nugetDir -Force | Out-Null - -# Resolve NuGet v3 feed to find package base URL -Write-Host "Resolving feed: $FeedUrl" -$feedIndex = Invoke-RestMethod -Uri $FeedUrl -$packageBaseUrl = ($feedIndex.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress*' }).'@id' -if (-not $packageBaseUrl) { throw "Could not resolve PackageBaseAddress from feed" } -Write-Host "Package base: $packageBaseUrl" - -$packageId = "mssql-py-core-wheels" -$packageIdLower = $packageId.ToLower() - -$versionLower = $PackageVersion.ToLower() -$nupkgUrl = "$packageBaseUrl$packageIdLower/$versionLower/$packageIdLower.$versionLower.nupkg" -$nupkgPath = "$nugetDir\$packageIdLower.$versionLower.nupkg" - -Write-Host "Downloading: $nupkgUrl" -Invoke-WebRequest -Uri $nupkgUrl -OutFile $nupkgPath -Write-Host "Downloaded: $nupkgPath ($([math]::Round((Get-Item $nupkgPath).Length / 1MB, 2)) MB)" - -# Extract NuGet (it's a ZIP — rename so Expand-Archive accepts it) -$zipPath = "$nugetDir\$packageIdLower.$versionLower.zip" -Rename-Item -Path $nupkgPath -NewName (Split-Path $zipPath -Leaf) -$extractDir = "$nugetDir\extracted" -Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force - -# Find the matching wheel -$wheelsDir = "$extractDir\wheels" -if (-not (Test-Path $wheelsDir)) { - throw "No 'wheels' directory found in NuGet package. Contents: $(Get-ChildItem $extractDir -Recurse | Select-Object -ExpandProperty Name)" + $script:WheelPattern = "mssql_py_core-*-$script:PyVersion-$script:PyVersion-$script:WheelPlatform.whl" + Write-Host "Wheel pattern: $script:WheelPattern" } -$matchingWheel = Get-ChildItem $wheelsDir -Filter $wheelPattern | Select-Object -First 1 -if (-not $matchingWheel) { - Write-Host "Available wheels:" - Get-ChildItem $wheelsDir -Filter *.whl | ForEach-Object { Write-Host " $_" } - throw "No wheel found matching pattern: $wheelPattern" -} +function Get-NupkgFromFeed { + param([string]$FeedUrl, [string]$OutputDir) + + if (Test-Path $OutputDir) { Remove-Item $OutputDir -Recurse -Force } + New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + + Write-Host "Resolving feed: $FeedUrl" + $feedIndex = Invoke-RestMethod -Uri $FeedUrl + $packageBaseUrl = ($feedIndex.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress*' }).'@id' + if (-not $packageBaseUrl) { throw "Could not resolve PackageBaseAddress from feed" } + + $packageId = "mssql-py-core-wheels" + $versionLower = $script:PackageVersion.ToLower() + $nupkgUrl = "${packageBaseUrl}${packageId}/${versionLower}/${packageId}.${versionLower}.nupkg" + $script:NupkgPath = Join-Path $OutputDir "${packageId}.${versionLower}.nupkg" -Write-Host "Found matching wheel: $($matchingWheel.Name)" - -# Extract mssql_py_core/ from the wheel into the repository root. -# The wheel is a ZIP file containing mssql_py_core/__init__.py and -# mssql_py_core/mssql_py_core... -# We skip .dist-info/ metadata. -# mssql_py_core.libs/ won't exist because auditwheel=skip is set in pyproject.toml, -# but we skip it defensively in case an older wheel is used. -$targetDir = $repoRoot -$coreDir = Join-Path $targetDir "mssql_py_core" - -# Clean previous extraction -if (Test-Path $coreDir) { - Remove-Item $coreDir -Recurse -Force - Write-Host "Cleaned previous mssql_py_core/ directory" + Write-Host "Downloading: $nupkgUrl" + Invoke-WebRequest -Uri $nupkgUrl -OutFile $script:NupkgPath + $sizeMB = [math]::Round((Get-Item $script:NupkgPath).Length / 1MB, 2) + Write-Host "Downloaded: $script:NupkgPath ($sizeMB MB)" } -Write-Host "Extracting mssql_py_core from wheel into: $targetDir" - -# Use Python to extract (zipfile handles wheel format reliably) -$wheelPath = $matchingWheel.FullName -& python -c @" -import zipfile, os, sys - -wheel_path = r'$wheelPath' -target_dir = r'$targetDir' -extracted = 0 - -with zipfile.ZipFile(wheel_path, 'r') as zf: - for entry in zf.namelist(): - # Skip dist-info metadata - if '.dist-info/' in entry: - continue - # Skip vendored shared libraries if present (auditwheel=skip means - # they won't be in the wheel; system OpenSSL is used at runtime) - if entry.startswith('mssql_py_core.libs/'): - continue - if entry.startswith('mssql_py_core/'): - out_path = os.path.join(target_dir, entry) - if entry.endswith('/'): - os.makedirs(out_path, exist_ok=True) - continue - os.makedirs(os.path.dirname(out_path), exist_ok=True) - with open(out_path, 'wb') as f: - f.write(zf.read(entry)) - extracted += 1 - print(f' Extracted: {entry}') - -if extracted == 0: - print('ERROR: No mssql_py_core files found in wheel', file=sys.stderr) - sys.exit(1) - -print(f'Extracted {extracted} file(s) into {target_dir}') -"@ -if ($LASTEXITCODE -ne 0) { throw "Failed to extract mssql_py_core from wheel" } - -# Verify import works (from repo root so mssql_py_core/ is on sys.path) -Write-Host "Verifying mssql_py_core import..." -Push-Location $repoRoot -try { - & python -c "import mssql_py_core; print(f'mssql_py_core loaded successfully: {dir(mssql_py_core)}')" - if ($LASTEXITCODE -ne 0) { throw "Failed to import mssql_py_core" } +function Find-MatchingWheel { + param([string]$OutputDir) + + # nupkg is a ZIP — rename so Expand-Archive accepts it + $zipPath = $script:NupkgPath -replace '\.nupkg$', '.zip' + Rename-Item -Path $script:NupkgPath -NewName (Split-Path $zipPath -Leaf) + + $extractDir = Join-Path $OutputDir "extracted" + Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force + + $wheelsDir = Join-Path $extractDir "wheels" + if (-not (Test-Path $wheelsDir)) { + throw "No 'wheels' directory found in NuGet package" + } + + $script:MatchingWheel = Get-ChildItem $wheelsDir -Filter $script:WheelPattern | Select-Object -First 1 + if (-not $script:MatchingWheel) { + Write-Host "Available wheels:" + Get-ChildItem $wheelsDir -Filter *.whl | ForEach-Object { Write-Host " $_" } + throw "No wheel found matching: $script:WheelPattern" + } + + Write-Host "Found: $($script:MatchingWheel.Name)" } -finally { - Pop-Location + +function Install-AndVerify { + $coreDir = Join-Path $RepoRoot "mssql_py_core" + if (Test-Path $coreDir) { + Remove-Item $coreDir -Recurse -Force + Write-Host "Cleaned previous mssql_py_core/" + } + + & python "$ScriptDir\extract_wheel.py" $script:MatchingWheel.FullName $RepoRoot + if ($LASTEXITCODE -ne 0) { throw "Failed to extract mssql_py_core from wheel" } + + Write-Host "Verifying import..." + Push-Location $RepoRoot + try { + & python -c "import mssql_py_core; print(f'mssql_py_core loaded: {dir(mssql_py_core)}')" + if ($LASTEXITCODE -ne 0) { throw "Failed to import mssql_py_core" } + } finally { + Pop-Location + } } -# Cleanup temp files -Remove-Item $OutputDir -Recurse -Force -ErrorAction SilentlyContinue +# --- main --- +Write-Host "=== Install mssql_py_core from NuGet wheel package ===" + +Read-PackageVersion +Get-PlatformInfo +Get-NupkgFromFeed -FeedUrl $FeedUrl -OutputDir $OutputDir +Find-MatchingWheel -OutputDir $OutputDir +Install-AndVerify + +Remove-Item $OutputDir -Recurse -Force -ErrorAction SilentlyContinue Write-Host "=== mssql_py_core extracted successfully ===" From 257db0136e0569ab3ec24e21ddca0055e4dd46f5 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 18:03:03 -0800 Subject: [PATCH 24/35] Refactor mssql_py_core installation steps to use a template for consistency across platforms and containers --- eng/pipelines/pr-validation-pipeline.yml | 133 +++++++----------- eng/pipelines/steps/install-mssql-py-core.yml | 62 ++++++++ eng/scripts/install-mssql-py-core.sh | 5 +- eng/scripts/resolve_nuget_feed.py | 39 +++-- 4 files changed, 143 insertions(+), 96 deletions(-) create mode 100644 eng/pipelines/steps/install-mssql-py-core.yml diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index c2481e7b..a7637084 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -230,12 +230,9 @@ jobs: build.bat x64 displayName: 'Build .pyd file' - # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) - - task: PowerShell@2 - displayName: 'Install mssql_py_core from NuGet wheels' - inputs: - targetType: 'filePath' - filePath: 'eng/scripts/install-mssql-py-core.ps1' + - template: steps/install-mssql-py-core.yml + parameters: + platform: windows # Run tests for LocalDB - script: | @@ -504,11 +501,9 @@ jobs: ./build.sh displayName: 'Build pybind bindings (.so)' - # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) - - script: | - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - displayName: 'Install mssql_py_core from NuGet wheels' + - template: steps/install-mssql-py-core.yml + parameters: + platform: unix - script: | echo "Build successful, running tests now" @@ -682,14 +677,11 @@ jobs: " displayName: 'Build pybind bindings (.so) in $(distroName) container' - # Install mssql_py_core from NuGet wheel package inside container - - script: | - docker exec test-container-$(distroName) bash -c " - source /opt/venv/bin/activate - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - " - displayName: 'Install mssql_py_core in $(distroName) container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-$(distroName) + venvActivate: 'source /opt/venv/bin/activate' - script: | # Uninstall ODBC Driver before running tests @@ -1006,14 +998,11 @@ jobs: displayName: 'Build pybind bindings (.so) in $(distroName) ARM64 container' retryCountOnTaskFailure: 2 - # Install mssql_py_core from NuGet wheel package inside ARM64 container - - script: | - docker exec test-container-$(distroName)-$(archName) bash -c " - source /opt/venv/bin/activate - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - " - displayName: 'Install mssql_py_core in $(distroName) ARM64 container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-$(distroName)-$(archName) + venvActivate: 'source /opt/venv/bin/activate' - script: | # Uninstall ODBC Driver before running tests @@ -1223,14 +1212,11 @@ jobs: " displayName: 'Build pybind bindings (.so) in RHEL 9 container' - # Install mssql_py_core from NuGet wheel package inside RHEL 9 container - - script: | - docker exec test-container-rhel9 bash -c " - source myvenv/bin/activate - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - " - displayName: 'Install mssql_py_core in RHEL 9 container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-rhel9 + venvActivate: 'source myvenv/bin/activate' - script: | # Uninstall ODBC Driver before running tests @@ -1451,14 +1437,11 @@ jobs: displayName: 'Build pybind bindings (.so) in RHEL 9 ARM64 container' retryCountOnTaskFailure: 2 - # Install mssql_py_core from NuGet wheel package inside RHEL 9 ARM64 container - - script: | - docker exec test-container-rhel9-arm64 bash -c " - source myvenv/bin/activate - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - " - displayName: 'Install mssql_py_core in RHEL 9 ARM64 container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-rhel9-arm64 + venvActivate: 'source myvenv/bin/activate' - script: | # Uninstall ODBC Driver before running tests @@ -1687,14 +1670,11 @@ jobs: " displayName: 'Build pybind bindings (.so) in Alpine x86_64 container' - # Install mssql_py_core from NuGet wheel package inside Alpine container - - script: | - docker exec test-container-alpine bash -c " - source /workspace/venv/bin/activate - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - " - displayName: 'Install mssql_py_core in Alpine x86_64 container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-alpine + venvActivate: 'source /workspace/venv/bin/activate' - script: | # Uninstall ODBC Driver before running tests to use bundled libraries @@ -1941,14 +1921,11 @@ jobs: displayName: 'Build pybind bindings (.so) in Alpine ARM64 container' retryCountOnTaskFailure: 2 - # Install mssql_py_core from NuGet wheel package inside Alpine ARM64 container - - script: | - docker exec test-container-alpine-arm64 bash -c " - source /workspace/venv/bin/activate - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - " - displayName: 'Install mssql_py_core in Alpine ARM64 container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-alpine-arm64 + venvActivate: 'source /workspace/venv/bin/activate' - script: | # Uninstall ODBC Driver before running tests to use bundled libraries @@ -2072,12 +2049,9 @@ jobs: build.bat x64 displayName: 'Build .pyd file' - # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) - - task: PowerShell@2 - displayName: 'Install mssql_py_core from NuGet wheels' - inputs: - targetType: 'filePath' - filePath: 'eng/scripts/install-mssql-py-core.ps1' + - template: steps/install-mssql-py-core.yml + parameters: + platform: windows - script: | python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear @@ -2121,11 +2095,9 @@ jobs: ./build.sh displayName: 'Build pybind bindings (.so)' - # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) - - script: | - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - displayName: 'Install mssql_py_core from NuGet wheels' + - template: steps/install-mssql-py-core.yml + parameters: + platform: unix - script: | python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear @@ -2198,14 +2170,11 @@ jobs: " displayName: 'Build pybind bindings (.so) in Ubuntu container' - # Install mssql_py_core from NuGet wheel package inside Ubuntu container - - script: | - docker exec test-container-ubuntu-azuresql bash -c " - source /opt/venv/bin/activate - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - " - displayName: 'Install mssql_py_core in Ubuntu container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-ubuntu-azuresql + venvActivate: 'source /opt/venv/bin/activate' - script: | docker exec test-container-ubuntu-azuresql bash -c " @@ -2301,11 +2270,9 @@ jobs: ./build.sh codecov displayName: 'Build pybind bindings with coverage' - # Install mssql_py_core from NuGet wheel package (enables _bulkcopy tests) - - script: | - chmod +x eng/scripts/install-mssql-py-core.sh - ./eng/scripts/install-mssql-py-core.sh - displayName: 'Install mssql_py_core from NuGet wheels' + - template: steps/install-mssql-py-core.yml + parameters: + platform: unix - script: | # Generate unified coverage (Python + C++) diff --git a/eng/pipelines/steps/install-mssql-py-core.yml b/eng/pipelines/steps/install-mssql-py-core.yml new file mode 100644 index 00000000..08ee9a75 --- /dev/null +++ b/eng/pipelines/steps/install-mssql-py-core.yml @@ -0,0 +1,62 @@ +# Step template: Install mssql_py_core from NuGet wheel package +# +# Usage: +# # Windows (host) +# - template: steps/install-mssql-py-core.yml +# parameters: +# platform: windows +# +# # macOS / Linux (host) +# - template: steps/install-mssql-py-core.yml +# parameters: +# platform: unix +# +# # Inside a Docker container +# - template: steps/install-mssql-py-core.yml +# parameters: +# platform: container +# containerName: test-container-$(distroName) +# venvActivate: 'source /opt/venv/bin/activate' + +parameters: + - name: platform + type: string + values: [windows, unix, container] + + - name: containerName + type: string + default: '' + + - name: venvActivate + type: string + default: 'source /opt/venv/bin/activate' + + - name: displaySuffix + type: string + default: '' + +steps: +# Windows: run the PowerShell script directly on the host +- ${{ if eq(parameters.platform, 'windows') }}: + - task: PowerShell@2 + displayName: 'Install mssql_py_core from NuGet wheels${{ parameters.displaySuffix }}' + inputs: + targetType: 'filePath' + filePath: 'eng/scripts/install-mssql-py-core.ps1' + +# Unix host (macOS, Linux without container) +- ${{ if eq(parameters.platform, 'unix') }}: + - script: | + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + displayName: 'Install mssql_py_core from NuGet wheels${{ parameters.displaySuffix }}' + +# Inside a Docker container +- ${{ if eq(parameters.platform, 'container') }}: + - script: | + docker exec ${{ parameters.containerName }} bash -c " + ${{ parameters.venvActivate }} + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + " + displayName: 'Install mssql_py_core in ${{ parameters.containerName }}${{ parameters.displaySuffix }}' diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index c3ac508b..9545d999 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -86,10 +86,7 @@ download_nupkg() { mkdir -p "$output_dir" echo "Resolving feed: $feed_url" - local feed_index - feed_index=$(curl -sS "$feed_url") - - PACKAGE_BASE_URL=$(echo "$feed_index" | python "$SCRIPT_DIR/resolve_nuget_feed.py") + PACKAGE_BASE_URL=$(python "$SCRIPT_DIR/resolve_nuget_feed.py" "$feed_url") if [ -z "$PACKAGE_BASE_URL" ]; then echo "ERROR: Could not resolve PackageBaseAddress from feed" exit 1 diff --git a/eng/scripts/resolve_nuget_feed.py b/eng/scripts/resolve_nuget_feed.py index 175fc9ba..eea9d786 100644 --- a/eng/scripts/resolve_nuget_feed.py +++ b/eng/scripts/resolve_nuget_feed.py @@ -1,24 +1,45 @@ #!/usr/bin/env python """Resolve the PackageBaseAddress URL from a NuGet v3 feed index. -Reads NuGet v3 service index JSON from stdin and prints the -PackageBaseAddress endpoint URL to stdout. +Fetches the NuGet v3 service index from the given feed URL and prints +the PackageBaseAddress endpoint URL to stdout. Usage: - curl -sS "$FEED_URL" | python resolve_nuget_feed.py + python resolve_nuget_feed.py """ import json import sys +import urllib.request -def main() -> None: - data = json.load(sys.stdin) +def resolve(feed_url: str) -> str: + """Fetch the feed index and return the PackageBaseAddress URL. + + NuGet v3 feeds expose a service index (index.json) listing available + API resources. The PackageBaseAddress resource provides a flat + container URL for downloading .nupkg files by convention: + {base}/{id}/{version}/{id}.{version}.nupkg + + We need this base URL because we download the mssql-py-core-wheels + nupkg directly via HTTP rather than using the NuGet CLI. + """ + with urllib.request.urlopen(feed_url) as resp: + data = json.loads(resp.read()) for resource in data["resources"]: if "PackageBaseAddress" in resource.get("@type", ""): - print(resource["@id"]) - return - print("ERROR: No PackageBaseAddress found in feed index", file=sys.stderr) - sys.exit(1) + return resource["@id"] + raise ValueError("No PackageBaseAddress found in feed index") + + +def main() -> None: + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(2) + try: + print(resolve(sys.argv[1])) + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) if __name__ == "__main__": From 7fd968e72d9a16589ab862250840b06ac3ff62f0 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 18:20:54 -0800 Subject: [PATCH 25/35] Enhance documentation for NuGet feed resolution in scripts to clarify PackageBaseAddress extraction --- eng/scripts/install-mssql-py-core.ps1 | 16 ++++++++++------ eng/scripts/install-mssql-py-core.sh | 1 + eng/scripts/resolve_nuget_feed.py | 21 +++++++++++++++++++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 index 6eeb85aa..6b5029c5 100644 --- a/eng/scripts/install-mssql-py-core.ps1 +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -40,8 +40,8 @@ function Get-PlatformInfo { $parts = $info -split ' ' $script:PyVersion = $parts[0] - $script:Platform = $parts[1] - $script:Arch = $parts[2] + $script:Platform = $parts[1] + $script:Arch = $parts[2] Write-Host "Python: $script:PyVersion | Platform: $script:Platform | Arch: $script:Arch" @@ -54,9 +54,9 @@ function Get-PlatformInfo { $script:WheelPlatform = switch ($script:Platform) { 'windows' { "win_$($archTag -replace 'x86_64','amd64')" } - 'linux' { "linux_$archTag" } - 'darwin' { 'macosx_15_0_universal2' } - default { throw "Unsupported platform: $script:Platform" } + 'linux' { "linux_$archTag" } + 'darwin' { 'macosx_15_0_universal2' } + default { throw "Unsupported platform: $script:Platform" } } $script:WheelPattern = "mssql_py_core-*-$script:PyVersion-$script:PyVersion-$script:WheelPlatform.whl" @@ -70,12 +70,15 @@ function Get-NupkgFromFeed { New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null Write-Host "Resolving feed: $FeedUrl" + # Fetch the NuGet v3 service index and extract the PackageBaseAddress URL. + # See resolve_nuget_feed.py for the JSON schema and detailed explanation. $feedIndex = Invoke-RestMethod -Uri $FeedUrl $packageBaseUrl = ($feedIndex.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress*' }).'@id' if (-not $packageBaseUrl) { throw "Could not resolve PackageBaseAddress from feed" } $packageId = "mssql-py-core-wheels" $versionLower = $script:PackageVersion.ToLower() + # e.g. https://pkgs.dev.azure.com/.../nuget/v3/flat2/mssql-py-core-wheels/0.1.0-dev.20260222.140833/mssql-py-core-wheels.0.1.0-dev.20260222.140833.nupkg $nupkgUrl = "${packageBaseUrl}${packageId}/${versionLower}/${packageId}.${versionLower}.nupkg" $script:NupkgPath = Join-Path $OutputDir "${packageId}.${versionLower}.nupkg" @@ -125,7 +128,8 @@ function Install-AndVerify { try { & python -c "import mssql_py_core; print(f'mssql_py_core loaded: {dir(mssql_py_core)}')" if ($LASTEXITCODE -ne 0) { throw "Failed to import mssql_py_core" } - } finally { + } + finally { Pop-Location } } diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index 9545d999..e4ec97b5 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -96,6 +96,7 @@ download_nupkg() { local version_lower version_lower=$(echo "$PACKAGE_VERSION" | tr '[:upper:]' '[:lower:]') + # e.g. https://pkgs.dev.azure.com/.../nuget/v3/flat2/mssql-py-core-wheels/0.1.0-dev.20260222.140833/mssql-py-core-wheels.0.1.0-dev.20260222.140833.nupkg NUPKG_URL="${PACKAGE_BASE_URL}${package_id}/${version_lower}/${package_id}.${version_lower}.nupkg" NUPKG_PATH="$output_dir/${package_id}.${version_lower}.nupkg" diff --git a/eng/scripts/resolve_nuget_feed.py b/eng/scripts/resolve_nuget_feed.py index eea9d786..d4bcfb1d 100644 --- a/eng/scripts/resolve_nuget_feed.py +++ b/eng/scripts/resolve_nuget_feed.py @@ -16,8 +16,25 @@ def resolve(feed_url: str) -> str: """Fetch the feed index and return the PackageBaseAddress URL. NuGet v3 feeds expose a service index (index.json) listing available - API resources. The PackageBaseAddress resource provides a flat - container URL for downloading .nupkg files by convention: + API resources. The JSON looks like: + + { + "version": "3.0.0", + "resources": [ + { + "@id": "https://pkgs.dev.azure.com/.../nuget/v3/flat2/", + "@type": "PackageBaseAddress/3.0.0" + }, + { + "@id": "https://pkgs.dev.azure.com/.../query", + "@type": "SearchQueryService" + } + ... + ] + } + + The PackageBaseAddress resource provides a flat container URL for + downloading .nupkg files by convention: {base}/{id}/{version}/{id}.{version}.nupkg We need this base URL because we download the mssql-py-core-wheels From 3f34603a821b135290fc8f9465d3a9792c8b1464 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 18:53:47 -0800 Subject: [PATCH 26/35] Update platform tags for manylinux compatibility in get_platform_info function --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d30839da..58d0a468 100644 --- a/setup.py +++ b/setup.py @@ -53,9 +53,9 @@ def get_platform_info(): is_musl = libc_name == "" or "musl" in libc_name.lower() if target_arch == "x86_64": - return "x86_64", "musllinux_1_2_x86_64" if is_musl else "manylinux_2_34_x86_64" + return "x86_64", "musllinux_1_2_x86_64" if is_musl else "manylinux_2_28_x86_64" elif target_arch in ["aarch64", "arm64"]: - return "aarch64", "musllinux_1_2_aarch64" if is_musl else "manylinux_2_34_aarch64" + return "aarch64", "musllinux_1_2_aarch64" if is_musl else "manylinux_2_28_aarch64" else: raise OSError( f"Unsupported architecture '{target_arch}' for Linux; expected 'x86_64' or 'aarch64'." From faf381e2a7aff4a92a0d848656354c51f2ba6a22 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 19:24:58 -0800 Subject: [PATCH 27/35] Update Docker image tag for manylinux compatibility in build script --- OneBranchPipelines/stages/build-linux-single-stage.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index 397ea532..a372d310 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -104,12 +104,13 @@ stages: - script: | # Determine image based on LINUX_TAG and ARCH - # manylinux_2_34 = AlmaLinux 9 (glibc 2.34, OpenSSL 3.x) - # Required because mssql_py_core is linked against libssl.so.3 + # manylinux_2_28 = AlmaLinux 8 (glibc 2.28) + # Note: mssql_py_core (built in mssql-rs) requires OpenSSL 3 / glibc 2.34, + # but it is pre-built and downloaded from NuGet — not compiled here. if [[ "$(LINUX_TAG)" == "musllinux" ]]; then IMAGE="quay.io/pypa/musllinux_1_2_$(ARCH)" else - IMAGE="quay.io/pypa/manylinux_2_34_$(ARCH)" + IMAGE="quay.io/pypa/manylinux_2_28_$(ARCH)" fi docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ From ca28c4818a0c01a3f96c9ac48ef3d84d02837e2c Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 20:53:25 -0800 Subject: [PATCH 28/35] Refactor artifact consolidation and version validation in pipelines --- .../jobs/consolidate-artifacts-job.yml | 23 +++------ eng/pipelines/official-release-pipeline.yml | 21 ++++++++ eng/scripts/validate-release-versions.ps1 | 49 +++++++++++++++++++ setup.py | 8 --- 4 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 eng/scripts/validate-release-versions.ps1 diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 095dbdb9..478803aa 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -26,23 +26,9 @@ jobs: value: '$(Build.ArtifactStagingDirectory)' steps: - - checkout: self # Need source for repackaging script and version file + - checkout: self fetchDepth: 1 - # Official builds must not use dev/nightly mssql-py-core versions - - ${{ if eq(parameters.oneBranchType, 'Official') }}: - - bash: | - set -e - VERSION=$(cat $(Build.SourcesDirectory)/eng/versions/mssql-py-core.version | tr -d '[:space:]') - echo "mssql-py-core version: $VERSION" - if echo "$VERSION" | grep -qiE '(dev|nightly|alpha|beta|rc|preview)'; then - echo "##[error]Official builds cannot use pre-release mssql-py-core version: $VERSION" - echo "##[error]Update eng/versions/mssql-py-core.version to a stable release version." - exit 1 - fi - echo "Version '$VERSION' is acceptable for Official builds." - displayName: 'Validate mssql-py-core version for Official build' - # Download ALL artifacts from current build # Matrix jobs publish as: Windows_, macOS_, Linux_ # This downloads all of them automatically (27 total artifacts) @@ -127,6 +113,13 @@ jobs: displayName: 'Consolidate Windows symbols (optional)' continueOnError: true + # Include mssql-py-core version file for traceability + - bash: | + set -e + cp $(Build.SourcesDirectory)/eng/versions/mssql-py-core.version $(ob_outputDirectory)/dist/ + echo "mssql-py-core version: $(cat $(ob_outputDirectory)/dist/mssql-py-core.version)" + displayName: 'Include mssql-py-core version' + # Verify consolidation - bash: | echo "==========================================" diff --git a/eng/pipelines/official-release-pipeline.yml b/eng/pipelines/official-release-pipeline.yml index 6a24ed38..4a6ac3e3 100644 --- a/eng/pipelines/official-release-pipeline.yml +++ b/eng/pipelines/official-release-pipeline.yml @@ -21,6 +21,27 @@ jobs: targetPath: '$(Build.SourcesDirectory)\dist' displayName: 'Download release wheel files artifact from latest successful run on main branch' + # Download the mssql-py-core version file from the same build + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2162 + buildVersionToDownload: 'latest' + branchName: '$(Build.SourceBranch)' + artifactName: 'mssql-python-wheels-dist' + itemPattern: 'mssql-py-core.version' + targetPath: '$(Build.SourcesDirectory)\versions' + displayName: 'Download mssql-py-core version file' + + # Validate that mssql-py-core version is a stable release (no dev/alpha/beta/rc) + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\eng\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\versions\mssql-py-core.version"' + # Show content of the downloaded artifact - script: | echo "Contents of the dist directory:" diff --git a/eng/scripts/validate-release-versions.ps1 b/eng/scripts/validate-release-versions.ps1 new file mode 100644 index 00000000..a9cca199 --- /dev/null +++ b/eng/scripts/validate-release-versions.ps1 @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Validates that a version file contains a stable (non-prerelease) version. + +.DESCRIPTION + Reads a single .version file and rejects any version containing + dev, alpha, beta, or rc tags. Intended to gate official releases. + +.PARAMETER VersionFile + Path to a .version file. Defaults to eng/versions/mssql-py-core.version + relative to the repository root. + +.EXAMPLE + # Run from repo root (uses default path): + .\eng\scripts\validate-release-versions.ps1 + + # Explicit path: + .\eng\scripts\validate-release-versions.ps1 -VersionFile C:\work\mssql-python\eng\versions\mssql-py-core.version +#> + +param( + [string]$VersionFile +) + +$ErrorActionPreference = 'Stop' + +if (-not $VersionFile) { + $repoRoot = (Resolve-Path "$PSScriptRoot\..\..").Path + $VersionFile = Join-Path $repoRoot 'eng\versions\mssql-py-core.version' +} + +if (-not (Test-Path $VersionFile)) { + Write-Error "Version file not found: $VersionFile" + exit 1 +} + +$version = (Get-Content $VersionFile -Raw).Trim() +$name = [System.IO.Path]::GetFileNameWithoutExtension($VersionFile) + +if ($version -match '(dev|alpha|beta|rc)') { + Write-Host "FAIL: $name version '$version' is a pre-release ($($Matches[1]))" -ForegroundColor Red + Write-Error "$name version is pre-release. Official releases require stable versions." + exit 1 +} + +Write-Host "OK: $name version '$version'" -ForegroundColor Green diff --git a/setup.py b/setup.py index 58d0a468..f2da6673 100644 --- a/setup.py +++ b/setup.py @@ -9,17 +9,12 @@ PROJECT_ROOT = Path(__file__).resolve().parent -# --------------------------------------------------------------------------- # Custom distribution to force platform-specific wheel -# --------------------------------------------------------------------------- class BinaryDistribution(Distribution): def has_ext_modules(self): return True -# --------------------------------------------------------------------------- -# Platform / Python tag helpers -# --------------------------------------------------------------------------- def get_platform_info(): """Get platform-specific architecture and platform tag information.""" if sys.platform.startswith("win"): @@ -105,9 +100,6 @@ def validate_mssql_py_core(): print("mssql_py_core validation: OK") -# --------------------------------------------------------------------------- -# Custom bdist_wheel – sets platform tag -# --------------------------------------------------------------------------- class CustomBdistWheel(bdist_wheel): def finalize_options(self): # Call the original finalize_options first to initialize self.bdist_dir From 6a81cf5e386c6cea39bb7458319a66f523fb5985 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 20:56:23 -0800 Subject: [PATCH 29/35] Add version file download and validation for mssql-py-core in release pipeline --- eng/pipelines/dummy-release-pipeline.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index 9fcf985c..b19ed009 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -21,6 +21,27 @@ jobs: targetPath: '$(Build.SourcesDirectory)\dist' displayName: 'Download release wheel files artifact from latest successful run on main branch' + # Download the mssql-py-core version file from the same build + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2162 + buildVersionToDownload: 'latest' + branchName: '$(Build.SourceBranch)' + artifactName: 'mssql-python-wheels-dist' + itemPattern: 'mssql-py-core.version' + targetPath: '$(Build.SourcesDirectory)\versions' + displayName: 'Download mssql-py-core version file' + + # Validate that mssql-py-core version is a stable release (no dev/alpha/beta/rc) + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\eng\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\versions\mssql-py-core.version"' + # Show content of the downloaded artifact - script: | echo "Contents of the dist directory:" From ff1ac61a5c361a7f147e6223d88f106bf535c4c7 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 21:57:22 -0800 Subject: [PATCH 30/35] Add glibc version check for mssql_py_core import verification --- eng/scripts/install-mssql-py-core.sh | 42 +++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index e4ec97b5..7d358e33 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -149,6 +149,34 @@ find_matching_wheel() { echo "Found: $(basename "$MATCHING_WHEEL")" } +# Returns 0 (true) if the runtime glibc is new enough to load the .so. +# The mssql_py_core native extension is built on manylinux_2_34 (glibc 2.34). +# Build containers running manylinux_2_28 have glibc 2.28 — too old to dlopen it. +# On musl (Alpine) or macOS we always attempt the import. +can_verify_import() { + case "$PLATFORM" in + linux) + # musl doesn't use glibc versioning — always try + if echo "$WHEEL_PLATFORM" | grep -q musl; then + return 0 + fi + local glibc_version + glibc_version=$(ldd --version 2>&1 | head -1 | grep -oP '[0-9]+\.[0-9]+$' || echo "0.0") + local major minor + major=$(echo "$glibc_version" | cut -d. -f1) + minor=$(echo "$glibc_version" | cut -d. -f2) + # Require glibc >= 2.34 + if [ "$major" -gt 2 ] 2>/dev/null || { [ "$major" -eq 2 ] && [ "$minor" -ge 34 ]; } 2>/dev/null; then + return 0 + fi + return 1 + ;; + *) + return 0 + ;; + esac +} + extract_and_verify() { local target_dir="$REPO_ROOT" local core_dir="$target_dir/mssql_py_core" @@ -160,10 +188,16 @@ extract_and_verify() { python "$SCRIPT_DIR/extract_wheel.py" "$MATCHING_WHEEL" "$target_dir" - echo "Verifying import..." - pushd "$target_dir" > /dev/null - python -c "import mssql_py_core; print(f'mssql_py_core loaded: {dir(mssql_py_core)}')" - popd > /dev/null + # Skip import verification when glibc is older than what the .so requires + # (e.g. manylinux_2_28 build containers with glibc 2.28, but .so needs 2.34). + if can_verify_import; then + echo "Verifying import..." + pushd "$target_dir" > /dev/null + python -c "import mssql_py_core; print(f'mssql_py_core loaded: {dir(mssql_py_core)}')" + popd > /dev/null + else + echo "Skipping import verification (glibc too old for runtime load)" + fi } # --- main --- From a11f322438dc5c307ebb9810611529bdf6e0be10 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 22:40:42 -0800 Subject: [PATCH 31/35] Add runtime check for mssql_py_core loading in bulkcopy tests --- tests/test_019_bulkcopy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_019_bulkcopy.py b/tests/test_019_bulkcopy.py index fa5e2fda..cabcb8c8 100644 --- a/tests/test_019_bulkcopy.py +++ b/tests/test_019_bulkcopy.py @@ -5,6 +5,10 @@ import pytest +# Skip the entire module when mssql_py_core can't be loaded at runtime +# (e.g. manylinux_2_28 build containers where glibc is too old for the .so). +mssql_py_core = pytest.importorskip("mssql_py_core", reason="mssql_py_core not loadable (glibc too old?)") + def test_connection_and_cursor(cursor): """Test that connection and cursor work correctly.""" From 295bd57010d0c61ce65ab47186ee346136b45ae1 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 22:59:05 -0800 Subject: [PATCH 32/35] Add version validation for mssql-py-core in release pipelines --- OneBranchPipelines/dummy-release-pipeline.yml | 8 +++++ .../official-release-pipeline.yml | 8 +++++ .../scripts/validate-release-versions.ps1 | 0 eng/pipelines/build-whl-pipeline.yml | 30 ++++--------------- eng/pipelines/dummy-release-pipeline.yml | 21 ------------- eng/pipelines/official-release-pipeline.yml | 21 ------------- 6 files changed, 21 insertions(+), 67 deletions(-) rename {eng => OneBranchPipelines}/scripts/validate-release-versions.ps1 (100%) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 9c8637f6..e65e065a 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -174,6 +174,14 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" + # Step 3.5: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: '[TEST] Verify Wheel Integrity' diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 822a727a..a3656ec8 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -177,6 +177,14 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" + # Step 3.5: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: 'Verify Wheel Integrity' diff --git a/eng/scripts/validate-release-versions.ps1 b/OneBranchPipelines/scripts/validate-release-versions.ps1 similarity index 100% rename from eng/scripts/validate-release-versions.ps1 rename to OneBranchPipelines/scripts/validate-release-versions.ps1 diff --git a/eng/pipelines/build-whl-pipeline.yml b/eng/pipelines/build-whl-pipeline.yml index cfda6dba..a6540c8a 100644 --- a/eng/pipelines/build-whl-pipeline.yml +++ b/eng/pipelines/build-whl-pipeline.yml @@ -168,13 +168,6 @@ jobs: TargetFolder: '$(Build.ArtifactStagingDirectory)\all-pdbs' displayName: 'Place PDB file into artifacts directory' - # Extract mssql_py_core from NuGet into repo root before building the wheel - - task: PowerShell@2 - displayName: 'Install mssql_py_core from NuGet' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\eng\scripts\install-mssql-py-core.ps1' - # Build wheel package for the current architecture - script: | python -m pip install --upgrade pip @@ -189,7 +182,7 @@ jobs: SourceFolder: '$(Build.SourcesDirectory)\dist' Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)\dist' - displayName: 'Collect wheel package' + displayName: 'Collect wheel package' # Publish the collected .pyd file(s) as build artifacts - task: PublishBuildArtifacts@1 @@ -349,11 +342,6 @@ jobs: env: DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - # Extract mssql_py_core from NuGet into repo root before building the wheel - - script: | - bash $(Build.SourcesDirectory)/eng/scripts/install-mssql-py-core.sh - displayName: 'Install mssql_py_core from NuGet' - # Build wheel package for universal2 - script: | python -m pip install --upgrade pip @@ -368,7 +356,7 @@ jobs: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)/dist' displayName: 'Collect wheel package' - + # Publish the collected .so file(s) as build artifacts - task: PublishBuildArtifacts@1 condition: succeededOrFailed() @@ -456,10 +444,10 @@ jobs: if command -v dnf >/dev/null 2>&1; then dnf -y update || true # Toolchain + CMake + unixODBC headers + Kerberos + keyutils + ccache - dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true + dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true elif command -v yum >/dev/null 2>&1; then yum -y update || true - yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true + yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true else echo "No dnf/yum found in manylinux image" >&2 fi @@ -475,7 +463,7 @@ jobs: set -euxo pipefail apk update || true # Toolchain + CMake + unixODBC headers + Kerberos + keyutils + ccache - apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl || true + apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache || true # Quick visibility for logs echo "---- tool versions ----" @@ -502,16 +490,12 @@ jobs: PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; test -x \$PY || { echo 'Python \$PY missing'; exit 0; } # skip if not present ln -sf \$PY /usr/local/bin/python; - ln -sf \$PY /usr/local/bin/python3; python -m pip install -U pip setuptools wheel pybind11; echo 'python:' \$(python -V); which python; # 👉 run from the directory that has CMakeLists.txt cd /workspace/mssql_python/pybind; bash build.sh; - # Extract mssql_py_core from NuGet for this Python version - bash /workspace/eng/scripts/install-mssql-py-core.sh; - # back to repo root to build the wheel cd /workspace; python setup.py bdist_wheel; @@ -526,16 +510,12 @@ jobs: PY=/opt/python/${PYBIN}-${PYBIN}/bin/python; test -x \$PY || { echo 'Python \$PY missing'; exit 0; } # skip if not present ln -sf \$PY /usr/local/bin/python; - ln -sf \$PY /usr/local/bin/python3; python -m pip install -U pip setuptools wheel pybind11; echo 'python:' \$(python -V); which python; # 👉 run from the directory that has CMakeLists.txt cd /workspace/mssql_python/pybind; bash build.sh; - # Extract mssql_py_core from NuGet for this Python version - bash /workspace/eng/scripts/install-mssql-py-core.sh; - # back to repo root to build the wheel cd /workspace; python setup.py bdist_wheel; diff --git a/eng/pipelines/dummy-release-pipeline.yml b/eng/pipelines/dummy-release-pipeline.yml index b19ed009..9fcf985c 100644 --- a/eng/pipelines/dummy-release-pipeline.yml +++ b/eng/pipelines/dummy-release-pipeline.yml @@ -21,27 +21,6 @@ jobs: targetPath: '$(Build.SourcesDirectory)\dist' displayName: 'Download release wheel files artifact from latest successful run on main branch' - # Download the mssql-py-core version file from the same build - - task: DownloadPipelineArtifact@2 - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: 2162 - buildVersionToDownload: 'latest' - branchName: '$(Build.SourceBranch)' - artifactName: 'mssql-python-wheels-dist' - itemPattern: 'mssql-py-core.version' - targetPath: '$(Build.SourcesDirectory)\versions' - displayName: 'Download mssql-py-core version file' - - # Validate that mssql-py-core version is a stable release (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: 'Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\eng\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\versions\mssql-py-core.version"' - # Show content of the downloaded artifact - script: | echo "Contents of the dist directory:" diff --git a/eng/pipelines/official-release-pipeline.yml b/eng/pipelines/official-release-pipeline.yml index 4a6ac3e3..6a24ed38 100644 --- a/eng/pipelines/official-release-pipeline.yml +++ b/eng/pipelines/official-release-pipeline.yml @@ -21,27 +21,6 @@ jobs: targetPath: '$(Build.SourcesDirectory)\dist' displayName: 'Download release wheel files artifact from latest successful run on main branch' - # Download the mssql-py-core version file from the same build - - task: DownloadPipelineArtifact@2 - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: 2162 - buildVersionToDownload: 'latest' - branchName: '$(Build.SourceBranch)' - artifactName: 'mssql-python-wheels-dist' - itemPattern: 'mssql-py-core.version' - targetPath: '$(Build.SourcesDirectory)\versions' - displayName: 'Download mssql-py-core version file' - - # Validate that mssql-py-core version is a stable release (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: 'Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\eng\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\versions\mssql-py-core.version"' - # Show content of the downloaded artifact - script: | echo "Contents of the dist directory:" From 71045810a70a41942fb6f53529e727bdea7d38fa Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Sun, 22 Feb 2026 23:09:17 -0800 Subject: [PATCH 33/35] Refactor comments for clarity and improve formatting in bulkcopy tests --- OneBranchPipelines/build-release-package-pipeline.yml | 2 +- tests/test_019_bulkcopy.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index eff7fd9d..2c0c64ce 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -401,7 +401,7 @@ extends: # - musllinux: musl-based (Alpine Linux) # Architectures: x86_64 (AMD/Intel), aarch64 (ARM64) # Each stage: - # 1. Starts PyPA Docker container (manylinux_2_34 or musllinux_1_2) + # 1. Starts PyPA Docker container (manylinux_2_28 or musllinux_1_2) # 2. Starts SQL Server Docker container # 3. For each Python version (cp310-cp314): # a. Builds .so native extension diff --git a/tests/test_019_bulkcopy.py b/tests/test_019_bulkcopy.py index cabcb8c8..a50e5d6c 100644 --- a/tests/test_019_bulkcopy.py +++ b/tests/test_019_bulkcopy.py @@ -7,7 +7,9 @@ # Skip the entire module when mssql_py_core can't be loaded at runtime # (e.g. manylinux_2_28 build containers where glibc is too old for the .so). -mssql_py_core = pytest.importorskip("mssql_py_core", reason="mssql_py_core not loadable (glibc too old?)") +mssql_py_core = pytest.importorskip( + "mssql_py_core", reason="mssql_py_core not loadable (glibc too old?)" +) def test_connection_and_cursor(cursor): From 588d5ba90db5f7440f67bbe1beff57fd62607b30 Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Mon, 23 Feb 2026 05:13:21 -0800 Subject: [PATCH 34/35] Enhance security and error handling in extraction scripts - Implement path traversal protection in extract_wheel.py - Update install-mssql-py-core.sh to use consistent Python command - Enforce HTTPS URL validation in resolve_nuget_feed.py --- eng/scripts/extract_wheel.py | 9 ++++++--- eng/scripts/install-mssql-py-core.sh | 17 +++++------------ eng/scripts/resolve_nuget_feed.py | 8 +++++++- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/eng/scripts/extract_wheel.py b/eng/scripts/extract_wheel.py index be3808f0..70d8ef50 100644 --- a/eng/scripts/extract_wheel.py +++ b/eng/scripts/extract_wheel.py @@ -25,12 +25,15 @@ def extract(wheel_path: str, target_dir: str) -> int: continue out_path = os.path.join(target_dir, entry) + real_out = os.path.realpath(out_path) + if not real_out.startswith(os.path.realpath(target_dir) + os.sep): + raise ValueError(f"Path traversal blocked: {entry}") if entry.endswith("/"): - os.makedirs(out_path, exist_ok=True) + os.makedirs(real_out, exist_ok=True) continue - os.makedirs(os.path.dirname(out_path), exist_ok=True) - with open(out_path, "wb") as f: + os.makedirs(os.path.dirname(real_out), exist_ok=True) + with open(real_out, "wb") as f: f.write(zf.read(entry)) extracted += 1 print(f" Extracted: {entry}") diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh index 7d358e33..a18282be 100644 --- a/eng/scripts/install-mssql-py-core.sh +++ b/eng/scripts/install-mssql-py-core.sh @@ -38,7 +38,7 @@ read_version() { } detect_platform() { - read -r PY_VERSION PLATFORM ARCH <<< "$(python -c " + read -r PY_VERSION PLATFORM ARCH <<< "$("$PYTHON" -c " import sys, platform v = sys.version_info print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().lower()}')" @@ -86,7 +86,7 @@ download_nupkg() { mkdir -p "$output_dir" echo "Resolving feed: $feed_url" - PACKAGE_BASE_URL=$(python "$SCRIPT_DIR/resolve_nuget_feed.py" "$feed_url") + PACKAGE_BASE_URL=$("$PYTHON" "$SCRIPT_DIR/resolve_nuget_feed.py" "$feed_url") if [ -z "$PACKAGE_BASE_URL" ]; then echo "ERROR: Could not resolve PackageBaseAddress from feed" exit 1 @@ -121,7 +121,7 @@ find_matching_wheel() { if command -v unzip &>/dev/null; then unzip -q "$NUPKG_PATH" -d "$extract_dir" else - python -c "import zipfile; zipfile.ZipFile('$NUPKG_PATH').extractall('$extract_dir')" + "$PYTHON" -c "import zipfile, sys; zipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])" "$NUPKG_PATH" "$extract_dir" fi local wheels_dir="$extract_dir/wheels" @@ -135,13 +135,6 @@ find_matching_wheel() { if [ -z "$MATCHING_WHEEL" ]; then echo "Available wheels:" ls "$wheels_dir"/*.whl 2>/dev/null || echo " (none)" - # On musllinux (Alpine), no wheels may be available yet - if echo "$WHEEL_PLATFORM" | grep -q "musllinux"; then - echo "WARNING: No musllinux wheel found for: $WHEEL_PATTERN" - echo "mssql_py_core is not yet available for musllinux -- skipping." - rm -rf "$output_dir" - exit 0 - fi echo "ERROR: No wheel found matching: $WHEEL_PATTERN" exit 1 fi @@ -186,14 +179,14 @@ extract_and_verify() { echo "Cleaned previous mssql_py_core/" fi - python "$SCRIPT_DIR/extract_wheel.py" "$MATCHING_WHEEL" "$target_dir" + "$PYTHON" "$SCRIPT_DIR/extract_wheel.py" "$MATCHING_WHEEL" "$target_dir" # Skip import verification when glibc is older than what the .so requires # (e.g. manylinux_2_28 build containers with glibc 2.28, but .so needs 2.34). if can_verify_import; then echo "Verifying import..." pushd "$target_dir" > /dev/null - python -c "import mssql_py_core; print(f'mssql_py_core loaded: {dir(mssql_py_core)}')" + "$PYTHON" -c "import mssql_py_core; print(f'mssql_py_core loaded: {dir(mssql_py_core)}')" popd > /dev/null else echo "Skipping import verification (glibc too old for runtime load)" diff --git a/eng/scripts/resolve_nuget_feed.py b/eng/scripts/resolve_nuget_feed.py index d4bcfb1d..ea6c075e 100644 --- a/eng/scripts/resolve_nuget_feed.py +++ b/eng/scripts/resolve_nuget_feed.py @@ -9,8 +9,11 @@ """ import json import sys +import urllib.parse import urllib.request +_REQUEST_TIMEOUT_SECS = 120 + def resolve(feed_url: str) -> str: """Fetch the feed index and return the PackageBaseAddress URL. @@ -40,7 +43,10 @@ def resolve(feed_url: str) -> str: We need this base URL because we download the mssql-py-core-wheels nupkg directly via HTTP rather than using the NuGet CLI. """ - with urllib.request.urlopen(feed_url) as resp: + parsed = urllib.parse.urlparse(feed_url) + if parsed.scheme != "https": + raise ValueError(f"Only https:// URLs are allowed, got {parsed.scheme}://") + with urllib.request.urlopen(feed_url, timeout=_REQUEST_TIMEOUT_SECS) as resp: data = json.loads(resp.read()) for resource in data["resources"]: if "PackageBaseAddress" in resource.get("@type", ""): From 3115edc1dca59a55942e03d65b2fce334614bebb Mon Sep 17 00:00:00 2001 From: Saurabh Singh <1623701+saurabh500@users.noreply.github.com> Date: Mon, 23 Feb 2026 05:18:50 -0800 Subject: [PATCH 35/35] Refactor mssql_py_core validation to occur within CustomBdistWheel.run() for improved handling during package installation --- setup.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index f2da6673..61db7a08 100644 --- a/setup.py +++ b/setup.py @@ -110,6 +110,10 @@ def finalize_options(self): self.plat_name = platform_tag print(f"Setting wheel platform tag to: {self.plat_name} (arch: {arch})") + def run(self): + validate_mssql_py_core() + bdist_wheel.run(self) + # --------------------------------------------------------------------------- # Package discovery @@ -122,10 +126,10 @@ def finalize_options(self): arch, platform_tag = get_platform_info() print(f"Detected architecture: {arch} (platform tag: {platform_tag})") -# Validate that mssql_py_core has been pre-extracted into /mssql_py_core/. -# Extraction is done by eng/scripts/install-mssql-py-core before running setup.py. -validate_mssql_py_core() -packages.append("mssql_py_core") +# mssql_py_core is validated inside CustomBdistWheel.run() so that editable +# installs (pip install -e .) and other setup.py commands are not blocked. +if (PROJECT_ROOT / "mssql_py_core").is_dir(): + packages.append("mssql_py_core") # Add platform-specific packages if sys.platform.startswith("win"):