diff --git a/.gitignore b/.gitignore index 3069e19d4..3f9bd64e1 100644 --- a/.gitignore +++ b/.gitignore @@ -61,5 +61,8 @@ build/ *venv*/ **/*venv*/ +# Extracted mssql_py_core (from eng/scripts/install-mssql-py-core) +mssql_py_core/ + # learning files learnings/ diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 9c8637f65..e65e065a5 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -174,6 +174,14 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" + # Step 3.5: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: '[TEST] Verify Wheel Integrity' diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 0ef960fc3..478803aa3 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -26,7 +26,8 @@ jobs: value: '$(Build.ArtifactStagingDirectory)' steps: - - checkout: none # No source code needed for consolidation + - checkout: self + fetchDepth: 1 # Download ALL artifacts from current build # Matrix jobs publish as: Windows_, macOS_, Linux_ @@ -112,6 +113,13 @@ jobs: displayName: 'Consolidate Windows symbols (optional)' continueOnError: true + # Include mssql-py-core version file for traceability + - bash: | + set -e + cp $(Build.SourcesDirectory)/eng/versions/mssql-py-core.version $(ob_outputDirectory)/dist/ + echo "mssql-py-core version: $(cat $(ob_outputDirectory)/dist/mssql-py-core.version)" + displayName: 'Include mssql-py-core version' + # Verify consolidation - bash: | echo "==========================================" diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 822a727a0..a3656ec8b 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -177,6 +177,14 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" + # Step 3.5: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: 'Verify Wheel Integrity' diff --git a/OneBranchPipelines/scripts/validate-release-versions.ps1 b/OneBranchPipelines/scripts/validate-release-versions.ps1 new file mode 100644 index 000000000..a9cca199d --- /dev/null +++ b/OneBranchPipelines/scripts/validate-release-versions.ps1 @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Validates that a version file contains a stable (non-prerelease) version. + +.DESCRIPTION + Reads a single .version file and rejects any version containing + dev, alpha, beta, or rc tags. Intended to gate official releases. + +.PARAMETER VersionFile + Path to a .version file. Defaults to eng/versions/mssql-py-core.version + relative to the repository root. + +.EXAMPLE + # Run from repo root (uses default path): + .\eng\scripts\validate-release-versions.ps1 + + # Explicit path: + .\eng\scripts\validate-release-versions.ps1 -VersionFile C:\work\mssql-python\eng\versions\mssql-py-core.version +#> + +param( + [string]$VersionFile +) + +$ErrorActionPreference = 'Stop' + +if (-not $VersionFile) { + $repoRoot = (Resolve-Path "$PSScriptRoot\..\..").Path + $VersionFile = Join-Path $repoRoot 'eng\versions\mssql-py-core.version' +} + +if (-not (Test-Path $VersionFile)) { + Write-Error "Version file not found: $VersionFile" + exit 1 +} + +$version = (Get-Content $VersionFile -Raw).Trim() +$name = [System.IO.Path]::GetFileNameWithoutExtension($VersionFile) + +if ($version -match '(dev|alpha|beta|rc)') { + Write-Host "FAIL: $name version '$version' is a pre-release ($($Matches[1]))" -ForegroundColor Red + Write-Error "$name version is pre-release. Official releases require stable versions." + exit 1 +} + +Write-Host "OK: $name version '$version'" -ForegroundColor Green diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index 99dbbd461..a372d310d 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -104,6 +104,9 @@ stages: - script: | # Determine image based on LINUX_TAG and ARCH + # manylinux_2_28 = AlmaLinux 8 (glibc 2.28) + # Note: mssql_py_core (built in mssql-rs) requires OpenSSL 3 / glibc 2.34, + # but it is pre-built and downloaded from NuGet — not compiled here. if [[ "$(LINUX_TAG)" == "musllinux" ]]; then IMAGE="quay.io/pypa/musllinux_1_2_$(ARCH)" else @@ -126,10 +129,10 @@ stages: set -euxo pipefail if command -v dnf >/dev/null 2>&1; then dnf -y update || true - dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true + dnf -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true elif command -v yum >/dev/null 2>&1; then yum -y update || true - yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache || true + yum -y install gcc gcc-c++ make cmake unixODBC-devel krb5-libs keyutils-libs ccache curl || true fi gcc --version || true cmake --version || true @@ -138,7 +141,7 @@ stages: docker exec build-$(LINUX_TAG)-$(ARCH) sh -lc ' set -euxo pipefail apk update || true - apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache || true + apk add --no-cache bash build-base cmake unixodbc-dev krb5-libs keyutils-libs ccache curl || true gcc --version || true cmake --version || true ' @@ -208,6 +211,9 @@ stages: cd /workspace/mssql_python/pybind; bash build.sh; + # Step 3.5: Extract mssql_py_core from NuGet for this Python version + bash /workspace/eng/scripts/install-mssql-py-core.sh; + # Step 4: Build wheel echo "Building wheel package..."; cd /workspace; @@ -273,6 +279,9 @@ stages: cd /workspace/mssql_python/pybind; bash build.sh; + # Step 3.5: Extract mssql_py_core from NuGet for this Python version + bash /workspace/eng/scripts/install-mssql-py-core.sh; + # Step 4: Build wheel echo "Building wheel package..."; cd /workspace; diff --git a/OneBranchPipelines/stages/build-macos-single-stage.yml b/OneBranchPipelines/stages/build-macos-single-stage.yml index 71ccaf607..352d6863b 100644 --- a/OneBranchPipelines/stages/build-macos-single-stage.yml +++ b/OneBranchPipelines/stages/build-macos-single-stage.yml @@ -169,6 +169,15 @@ stages: env: DB_PASSWORD: $(DB_PASSWORD) + # ========================= + # MSSQL_PY_CORE INSTALLATION + # ========================= + # Extract mssql_py_core from NuGet into repo root BEFORE testing + # Required for bulkcopy tests which use mssql_py_core native bindings + - script: | + bash $(Build.SourcesDirectory)/eng/scripts/install-mssql-py-core.sh + displayName: 'Install mssql_py_core from NuGet' + # ========================= # TESTING # ========================= @@ -186,6 +195,7 @@ stages: # ========================= # WHEEL BUILD # ========================= + # Build wheel package from setup.py # Wheel filename: mssql_python-X.Y.Z-cp3XX-cp3XX-macosx_XX_X_universal2.whl # bdist_wheel = build binary wheel distribution (contains pre-compiled .so) diff --git a/OneBranchPipelines/stages/build-windows-single-stage.yml b/OneBranchPipelines/stages/build-windows-single-stage.yml index 3523267ca..972fe4949 100644 --- a/OneBranchPipelines/stages/build-windows-single-stage.yml +++ b/OneBranchPipelines/stages/build-windows-single-stage.yml @@ -238,6 +238,17 @@ stages: displayName: 'Build PYD for $(targetArch)' continueOnError: false + # ========================= + # MSSQL_PY_CORE INSTALLATION + # ========================= + # Extract mssql_py_core from NuGet into repo root BEFORE testing + # Required for bulkcopy tests which use mssql_py_core native bindings + - task: PowerShell@2 + displayName: 'Install mssql_py_core from NuGet' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\eng\scripts\install-mssql-py-core.ps1' + # ========================= # TESTING # ========================= diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index b3df96ff8..a76370840 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -230,6 +230,10 @@ jobs: build.bat x64 displayName: 'Build .pyd file' + - template: steps/install-mssql-py-core.yml + parameters: + platform: windows + # Run tests for LocalDB - script: | python -m pytest -v --junitxml=test-results-localdb.xml --cov=. --cov-report=xml:coverage-localdb.xml --capture=tee-sys --cache-clear @@ -497,6 +501,10 @@ jobs: ./build.sh displayName: 'Build pybind bindings (.so)' + - template: steps/install-mssql-py-core.yml + parameters: + platform: unix + - script: | echo "Build successful, running tests now" python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear @@ -669,6 +677,12 @@ jobs: " displayName: 'Build pybind bindings (.so) in $(distroName) container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-$(distroName) + venvActivate: 'source /opt/venv/bin/activate' + - script: | # Uninstall ODBC Driver before running tests docker exec test-container-$(distroName) bash -c " @@ -984,6 +998,12 @@ jobs: displayName: 'Build pybind bindings (.so) in $(distroName) ARM64 container' retryCountOnTaskFailure: 2 + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-$(distroName)-$(archName) + venvActivate: 'source /opt/venv/bin/activate' + - script: | # Uninstall ODBC Driver before running tests docker exec test-container-$(distroName)-$(archName) bash -c " @@ -1192,6 +1212,12 @@ jobs: " displayName: 'Build pybind bindings (.so) in RHEL 9 container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-rhel9 + venvActivate: 'source myvenv/bin/activate' + - script: | # Uninstall ODBC Driver before running tests docker exec test-container-rhel9 bash -c " @@ -1411,6 +1437,12 @@ jobs: displayName: 'Build pybind bindings (.so) in RHEL 9 ARM64 container' retryCountOnTaskFailure: 2 + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-rhel9-arm64 + venvActivate: 'source myvenv/bin/activate' + - script: | # Uninstall ODBC Driver before running tests docker exec test-container-rhel9-arm64 bash -c " @@ -1638,6 +1670,12 @@ jobs: " displayName: 'Build pybind bindings (.so) in Alpine x86_64 container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-alpine + venvActivate: 'source /workspace/venv/bin/activate' + - script: | # Uninstall ODBC Driver before running tests to use bundled libraries docker exec test-container-alpine bash -c " @@ -1883,6 +1921,12 @@ jobs: displayName: 'Build pybind bindings (.so) in Alpine ARM64 container' retryCountOnTaskFailure: 2 + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-alpine-arm64 + venvActivate: 'source /workspace/venv/bin/activate' + - script: | # Uninstall ODBC Driver before running tests to use bundled libraries docker exec test-container-alpine-arm64 bash -c " @@ -2005,6 +2049,10 @@ jobs: build.bat x64 displayName: 'Build .pyd file' + - template: steps/install-mssql-py-core.yml + parameters: + platform: windows + - script: | python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear displayName: 'Run tests on Azure SQL Database' @@ -2047,6 +2095,10 @@ jobs: ./build.sh displayName: 'Build pybind bindings (.so)' + - template: steps/install-mssql-py-core.yml + parameters: + platform: unix + - script: | python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear displayName: 'Run tests on Azure SQL Database' @@ -2118,6 +2170,12 @@ jobs: " displayName: 'Build pybind bindings (.so) in Ubuntu container' + - template: steps/install-mssql-py-core.yml + parameters: + platform: container + containerName: test-container-ubuntu-azuresql + venvActivate: 'source /opt/venv/bin/activate' + - script: | docker exec test-container-ubuntu-azuresql bash -c " export DEBIAN_FRONTEND=noninteractive @@ -2212,6 +2270,10 @@ jobs: ./build.sh codecov displayName: 'Build pybind bindings with coverage' + - template: steps/install-mssql-py-core.yml + parameters: + platform: unix + - script: | # Generate unified coverage (Python + C++) chmod +x ./generate_codecov.sh diff --git a/eng/pipelines/steps/install-mssql-py-core.yml b/eng/pipelines/steps/install-mssql-py-core.yml new file mode 100644 index 000000000..08ee9a750 --- /dev/null +++ b/eng/pipelines/steps/install-mssql-py-core.yml @@ -0,0 +1,62 @@ +# Step template: Install mssql_py_core from NuGet wheel package +# +# Usage: +# # Windows (host) +# - template: steps/install-mssql-py-core.yml +# parameters: +# platform: windows +# +# # macOS / Linux (host) +# - template: steps/install-mssql-py-core.yml +# parameters: +# platform: unix +# +# # Inside a Docker container +# - template: steps/install-mssql-py-core.yml +# parameters: +# platform: container +# containerName: test-container-$(distroName) +# venvActivate: 'source /opt/venv/bin/activate' + +parameters: + - name: platform + type: string + values: [windows, unix, container] + + - name: containerName + type: string + default: '' + + - name: venvActivate + type: string + default: 'source /opt/venv/bin/activate' + + - name: displaySuffix + type: string + default: '' + +steps: +# Windows: run the PowerShell script directly on the host +- ${{ if eq(parameters.platform, 'windows') }}: + - task: PowerShell@2 + displayName: 'Install mssql_py_core from NuGet wheels${{ parameters.displaySuffix }}' + inputs: + targetType: 'filePath' + filePath: 'eng/scripts/install-mssql-py-core.ps1' + +# Unix host (macOS, Linux without container) +- ${{ if eq(parameters.platform, 'unix') }}: + - script: | + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + displayName: 'Install mssql_py_core from NuGet wheels${{ parameters.displaySuffix }}' + +# Inside a Docker container +- ${{ if eq(parameters.platform, 'container') }}: + - script: | + docker exec ${{ parameters.containerName }} bash -c " + ${{ parameters.venvActivate }} + chmod +x eng/scripts/install-mssql-py-core.sh + ./eng/scripts/install-mssql-py-core.sh + " + displayName: 'Install mssql_py_core in ${{ parameters.containerName }}${{ parameters.displaySuffix }}' diff --git a/eng/scripts/extract_wheel.py b/eng/scripts/extract_wheel.py new file mode 100644 index 000000000..70d8ef504 --- /dev/null +++ b/eng/scripts/extract_wheel.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +"""Extract mssql_py_core package files from a wheel into a target directory. + +Wheels are ZIP files. This script extracts only mssql_py_core/ entries, +skipping .dist-info metadata and vendored .libs directories. + +Usage: + python extract_wheel.py +""" +import os +import sys +import zipfile + + +def extract(wheel_path: str, target_dir: str) -> int: + """Extract mssql_py_core/ entries from a wheel, return count of files extracted.""" + extracted = 0 + with zipfile.ZipFile(wheel_path, "r") as zf: + for entry in zf.namelist(): + if ".dist-info/" in entry: + continue + if entry.startswith("mssql_py_core.libs/"): + continue + if not entry.startswith("mssql_py_core/"): + continue + + out_path = os.path.join(target_dir, entry) + real_out = os.path.realpath(out_path) + if not real_out.startswith(os.path.realpath(target_dir) + os.sep): + raise ValueError(f"Path traversal blocked: {entry}") + if entry.endswith("/"): + os.makedirs(real_out, exist_ok=True) + continue + + os.makedirs(os.path.dirname(real_out), exist_ok=True) + with open(real_out, "wb") as f: + f.write(zf.read(entry)) + extracted += 1 + print(f" Extracted: {entry}") + + return extracted + + +def main() -> None: + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(2) + + wheel_path, target_dir = sys.argv[1], sys.argv[2] + count = extract(wheel_path, target_dir) + + if count == 0: + print("ERROR: No mssql_py_core files found in wheel", file=sys.stderr) + sys.exit(1) + + print(f"Extracted {count} file(s) into {target_dir}") + + +if __name__ == "__main__": + main() diff --git a/eng/scripts/install-mssql-py-core.ps1 b/eng/scripts/install-mssql-py-core.ps1 new file mode 100644 index 000000000..6b5029c57 --- /dev/null +++ b/eng/scripts/install-mssql-py-core.ps1 @@ -0,0 +1,148 @@ +<# +.SYNOPSIS + Downloads the mssql-py-core-wheels NuGet package from a public Azure Artifacts + feed and extracts the matching mssql_py_core binary into the repository root + so that 'import mssql_py_core' works when running from the source tree. + +.PARAMETER FeedUrl + The NuGet v3 feed URL. This is a public feed — no authentication required. + +.PARAMETER OutputDir + Temporary directory for downloaded artifacts. Cleaned up after extraction. + Defaults to $env:TEMP\mssql-py-core-wheels. +#> + +param( + [string]$FeedUrl = "https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json", + [string]$OutputDir = "$env:TEMP\mssql-py-core-wheels" +) + +$ErrorActionPreference = 'Stop' +$ScriptDir = $PSScriptRoot +$RepoRoot = (Get-Item "$ScriptDir\..\..").FullName + +function Read-PackageVersion { + $versionFile = Join-Path $RepoRoot "eng\versions\mssql-py-core.version" + if (-not (Test-Path $versionFile)) { + throw "Version file not found: $versionFile" + } + $script:PackageVersion = (Get-Content $versionFile -Raw).Trim() + if (-not $script:PackageVersion) { + throw "Version file is empty: $versionFile" + } + Write-Host "Version: $script:PackageVersion" +} + +function Get-PlatformInfo { + # Single python call to get version, platform, and arch + $info = & python -c "import sys, platform; v = sys.version_info; print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().lower()}')" + if ($LASTEXITCODE -ne 0) { throw "Failed to detect Python platform info" } + + $parts = $info -split ' ' + $script:PyVersion = $parts[0] + $script:Platform = $parts[1] + $script:Arch = $parts[2] + + Write-Host "Python: $script:PyVersion | Platform: $script:Platform | Arch: $script:Arch" + + # Normalize arch tag + $archTag = switch -Regex ($script:Arch) { + 'amd64|x86_64' { 'x86_64' } + 'arm64|aarch64' { 'aarch64' } + default { throw "Unsupported architecture: $script:Arch" } + } + + $script:WheelPlatform = switch ($script:Platform) { + 'windows' { "win_$($archTag -replace 'x86_64','amd64')" } + 'linux' { "linux_$archTag" } + 'darwin' { 'macosx_15_0_universal2' } + default { throw "Unsupported platform: $script:Platform" } + } + + $script:WheelPattern = "mssql_py_core-*-$script:PyVersion-$script:PyVersion-$script:WheelPlatform.whl" + Write-Host "Wheel pattern: $script:WheelPattern" +} + +function Get-NupkgFromFeed { + param([string]$FeedUrl, [string]$OutputDir) + + if (Test-Path $OutputDir) { Remove-Item $OutputDir -Recurse -Force } + New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + + Write-Host "Resolving feed: $FeedUrl" + # Fetch the NuGet v3 service index and extract the PackageBaseAddress URL. + # See resolve_nuget_feed.py for the JSON schema and detailed explanation. + $feedIndex = Invoke-RestMethod -Uri $FeedUrl + $packageBaseUrl = ($feedIndex.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress*' }).'@id' + if (-not $packageBaseUrl) { throw "Could not resolve PackageBaseAddress from feed" } + + $packageId = "mssql-py-core-wheels" + $versionLower = $script:PackageVersion.ToLower() + # e.g. https://pkgs.dev.azure.com/.../nuget/v3/flat2/mssql-py-core-wheels/0.1.0-dev.20260222.140833/mssql-py-core-wheels.0.1.0-dev.20260222.140833.nupkg + $nupkgUrl = "${packageBaseUrl}${packageId}/${versionLower}/${packageId}.${versionLower}.nupkg" + $script:NupkgPath = Join-Path $OutputDir "${packageId}.${versionLower}.nupkg" + + Write-Host "Downloading: $nupkgUrl" + Invoke-WebRequest -Uri $nupkgUrl -OutFile $script:NupkgPath + $sizeMB = [math]::Round((Get-Item $script:NupkgPath).Length / 1MB, 2) + Write-Host "Downloaded: $script:NupkgPath ($sizeMB MB)" +} + +function Find-MatchingWheel { + param([string]$OutputDir) + + # nupkg is a ZIP — rename so Expand-Archive accepts it + $zipPath = $script:NupkgPath -replace '\.nupkg$', '.zip' + Rename-Item -Path $script:NupkgPath -NewName (Split-Path $zipPath -Leaf) + + $extractDir = Join-Path $OutputDir "extracted" + Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force + + $wheelsDir = Join-Path $extractDir "wheels" + if (-not (Test-Path $wheelsDir)) { + throw "No 'wheels' directory found in NuGet package" + } + + $script:MatchingWheel = Get-ChildItem $wheelsDir -Filter $script:WheelPattern | Select-Object -First 1 + if (-not $script:MatchingWheel) { + Write-Host "Available wheels:" + Get-ChildItem $wheelsDir -Filter *.whl | ForEach-Object { Write-Host " $_" } + throw "No wheel found matching: $script:WheelPattern" + } + + Write-Host "Found: $($script:MatchingWheel.Name)" +} + +function Install-AndVerify { + $coreDir = Join-Path $RepoRoot "mssql_py_core" + if (Test-Path $coreDir) { + Remove-Item $coreDir -Recurse -Force + Write-Host "Cleaned previous mssql_py_core/" + } + + & python "$ScriptDir\extract_wheel.py" $script:MatchingWheel.FullName $RepoRoot + if ($LASTEXITCODE -ne 0) { throw "Failed to extract mssql_py_core from wheel" } + + Write-Host "Verifying import..." + Push-Location $RepoRoot + try { + & python -c "import mssql_py_core; print(f'mssql_py_core loaded: {dir(mssql_py_core)}')" + if ($LASTEXITCODE -ne 0) { throw "Failed to import mssql_py_core" } + } + finally { + Pop-Location + } +} + +# --- main --- + +Write-Host "=== Install mssql_py_core from NuGet wheel package ===" + +Read-PackageVersion +Get-PlatformInfo +Get-NupkgFromFeed -FeedUrl $FeedUrl -OutputDir $OutputDir +Find-MatchingWheel -OutputDir $OutputDir +Install-AndVerify + +Remove-Item $OutputDir -Recurse -Force -ErrorAction SilentlyContinue +Write-Host "=== mssql_py_core extracted successfully ===" diff --git a/eng/scripts/install-mssql-py-core.sh b/eng/scripts/install-mssql-py-core.sh new file mode 100644 index 000000000..a18282be5 --- /dev/null +++ b/eng/scripts/install-mssql-py-core.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash +# Downloads the mssql-py-core-wheels NuGet package from a public Azure Artifacts +# feed and extracts the matching mssql_py_core binary into the repository root +# so that 'import mssql_py_core' works when running from the source tree. +# +# The extracted files are placed at /mssql_py_core/ which contains: +# - __init__.py +# - mssql_py_core..so (native extension) +# +# This script is used identically for: +# - Local development (dev runs it after build.sh) +# - PR validation pipelines +# - Official build pipelines (before setup.py bdist_wheel) +# +# The package version is read from eng/versions/mssql-py-core.version (required). +# +# Usage: +# ./install-mssql-py-core.sh [--feed-url URL] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +PYTHON="${PYTHON:-$(command -v python || command -v python3)}" + +read_version() { + local version_file="$REPO_ROOT/eng/versions/mssql-py-core.version" + if [ ! -f "$version_file" ]; then + echo "ERROR: Version file not found: $version_file" + exit 1 + fi + PACKAGE_VERSION=$(tr -d '[:space:]' < "$version_file") + if [ -z "$PACKAGE_VERSION" ]; then + echo "ERROR: Version file is empty: $version_file" + exit 1 + fi + echo "Version: $PACKAGE_VERSION" +} + +detect_platform() { + read -r PY_VERSION PLATFORM ARCH <<< "$("$PYTHON" -c " +import sys, platform +v = sys.version_info +print(f'cp{v.major}{v.minor} {platform.system().lower()} {platform.machine().lower()}')" + )" + + echo "Python: $PY_VERSION | Platform: $PLATFORM | Arch: $ARCH" + + case "$PLATFORM" in + linux) + case "$ARCH" in + x86_64|amd64) ARCH_TAG="x86_64" ;; + aarch64|arm64) ARCH_TAG="aarch64" ;; + *) echo "Unsupported Linux architecture: $ARCH"; exit 1 ;; + esac + + # Detect musl libc (Alpine) vs glibc. + # ldd --version exits 1 on musl, so capture output instead of piping. + local ldd_output + ldd_output=$(ldd --version 2>&1 || true) + if echo "$ldd_output" | grep -qi musl || [ -f /etc/alpine-release ]; then + WHEEL_PLATFORM="musllinux_1_2_${ARCH_TAG}" + else + # auditwheel=skip: wheels are tagged linux_* not manylinux_2_34_* + WHEEL_PLATFORM="linux_${ARCH_TAG}" + fi + ;; + darwin) + WHEEL_PLATFORM="macosx_15_0_universal2" + ;; + *) + echo "Unsupported platform: $PLATFORM" + exit 1 + ;; + esac + + WHEEL_PATTERN="mssql_py_core-*-${PY_VERSION}-${PY_VERSION}-${WHEEL_PLATFORM}.whl" + echo "Wheel pattern: $WHEEL_PATTERN" +} + +download_nupkg() { + local feed_url="$1" + local output_dir="$2" + + rm -rf "$output_dir" + mkdir -p "$output_dir" + + echo "Resolving feed: $feed_url" + PACKAGE_BASE_URL=$("$PYTHON" "$SCRIPT_DIR/resolve_nuget_feed.py" "$feed_url") + if [ -z "$PACKAGE_BASE_URL" ]; then + echo "ERROR: Could not resolve PackageBaseAddress from feed" + exit 1 + fi + + local package_id="mssql-py-core-wheels" + local version_lower + version_lower=$(echo "$PACKAGE_VERSION" | tr '[:upper:]' '[:lower:]') + + # e.g. https://pkgs.dev.azure.com/.../nuget/v3/flat2/mssql-py-core-wheels/0.1.0-dev.20260222.140833/mssql-py-core-wheels.0.1.0-dev.20260222.140833.nupkg + NUPKG_URL="${PACKAGE_BASE_URL}${package_id}/${version_lower}/${package_id}.${version_lower}.nupkg" + NUPKG_PATH="$output_dir/${package_id}.${version_lower}.nupkg" + + echo "Downloading: $NUPKG_URL" + curl -sSL -o "$NUPKG_PATH" "$NUPKG_URL" + + local filesize + filesize=$(wc -c < "$NUPKG_PATH") + echo "Downloaded: $NUPKG_PATH ($filesize bytes)" + + if [ "$filesize" -eq 0 ]; then + echo "ERROR: Downloaded file is empty" + exit 1 + fi +} + +find_matching_wheel() { + local output_dir="$1" + local extract_dir="$output_dir/extracted" + + mkdir -p "$extract_dir" + if command -v unzip &>/dev/null; then + unzip -q "$NUPKG_PATH" -d "$extract_dir" + else + "$PYTHON" -c "import zipfile, sys; zipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])" "$NUPKG_PATH" "$extract_dir" + fi + + local wheels_dir="$extract_dir/wheels" + if [ ! -d "$wheels_dir" ]; then + echo "ERROR: No 'wheels' directory found in NuGet package" + ls -la "$extract_dir" + exit 1 + fi + + MATCHING_WHEEL=$(find "$wheels_dir" -name "$WHEEL_PATTERN" | head -1) + if [ -z "$MATCHING_WHEEL" ]; then + echo "Available wheels:" + ls "$wheels_dir"/*.whl 2>/dev/null || echo " (none)" + echo "ERROR: No wheel found matching: $WHEEL_PATTERN" + exit 1 + fi + + echo "Found: $(basename "$MATCHING_WHEEL")" +} + +# Returns 0 (true) if the runtime glibc is new enough to load the .so. +# The mssql_py_core native extension is built on manylinux_2_34 (glibc 2.34). +# Build containers running manylinux_2_28 have glibc 2.28 — too old to dlopen it. +# On musl (Alpine) or macOS we always attempt the import. +can_verify_import() { + case "$PLATFORM" in + linux) + # musl doesn't use glibc versioning — always try + if echo "$WHEEL_PLATFORM" | grep -q musl; then + return 0 + fi + local glibc_version + glibc_version=$(ldd --version 2>&1 | head -1 | grep -oP '[0-9]+\.[0-9]+$' || echo "0.0") + local major minor + major=$(echo "$glibc_version" | cut -d. -f1) + minor=$(echo "$glibc_version" | cut -d. -f2) + # Require glibc >= 2.34 + if [ "$major" -gt 2 ] 2>/dev/null || { [ "$major" -eq 2 ] && [ "$minor" -ge 34 ]; } 2>/dev/null; then + return 0 + fi + return 1 + ;; + *) + return 0 + ;; + esac +} + +extract_and_verify() { + local target_dir="$REPO_ROOT" + local core_dir="$target_dir/mssql_py_core" + + if [ -d "$core_dir" ]; then + rm -rf "$core_dir" + echo "Cleaned previous mssql_py_core/" + fi + + "$PYTHON" "$SCRIPT_DIR/extract_wheel.py" "$MATCHING_WHEEL" "$target_dir" + + # Skip import verification when glibc is older than what the .so requires + # (e.g. manylinux_2_28 build containers with glibc 2.28, but .so needs 2.34). + if can_verify_import; then + echo "Verifying import..." + pushd "$target_dir" > /dev/null + "$PYTHON" -c "import mssql_py_core; print(f'mssql_py_core loaded: {dir(mssql_py_core)}')" + popd > /dev/null + else + echo "Skipping import verification (glibc too old for runtime load)" + fi +} + +# --- main --- + +FEED_URL="${FEED_URL:-https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json}" +OUTPUT_DIR="${TMPDIR:-/tmp}/mssql-py-core-wheels" + +while [[ $# -gt 0 ]]; do + case "$1" in + --feed-url) FEED_URL="$2"; shift 2 ;; + *) echo "Unknown argument: $1"; exit 1 ;; + esac +done + +echo "=== Install mssql_py_core from NuGet wheel package ===" + +read_version +detect_platform +download_nupkg "$FEED_URL" "$OUTPUT_DIR" +find_matching_wheel "$OUTPUT_DIR" +extract_and_verify + +rm -rf "$OUTPUT_DIR" +echo "=== mssql_py_core extracted successfully ===" diff --git a/eng/scripts/resolve_nuget_feed.py b/eng/scripts/resolve_nuget_feed.py new file mode 100644 index 000000000..ea6c075e8 --- /dev/null +++ b/eng/scripts/resolve_nuget_feed.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +"""Resolve the PackageBaseAddress URL from a NuGet v3 feed index. + +Fetches the NuGet v3 service index from the given feed URL and prints +the PackageBaseAddress endpoint URL to stdout. + +Usage: + python resolve_nuget_feed.py +""" +import json +import sys +import urllib.parse +import urllib.request + +_REQUEST_TIMEOUT_SECS = 120 + + +def resolve(feed_url: str) -> str: + """Fetch the feed index and return the PackageBaseAddress URL. + + NuGet v3 feeds expose a service index (index.json) listing available + API resources. The JSON looks like: + + { + "version": "3.0.0", + "resources": [ + { + "@id": "https://pkgs.dev.azure.com/.../nuget/v3/flat2/", + "@type": "PackageBaseAddress/3.0.0" + }, + { + "@id": "https://pkgs.dev.azure.com/.../query", + "@type": "SearchQueryService" + } + ... + ] + } + + The PackageBaseAddress resource provides a flat container URL for + downloading .nupkg files by convention: + {base}/{id}/{version}/{id}.{version}.nupkg + + We need this base URL because we download the mssql-py-core-wheels + nupkg directly via HTTP rather than using the NuGet CLI. + """ + parsed = urllib.parse.urlparse(feed_url) + if parsed.scheme != "https": + raise ValueError(f"Only https:// URLs are allowed, got {parsed.scheme}://") + with urllib.request.urlopen(feed_url, timeout=_REQUEST_TIMEOUT_SECS) as resp: + data = json.loads(resp.read()) + for resource in data["resources"]: + if "PackageBaseAddress" in resource.get("@type", ""): + return resource["@id"] + raise ValueError("No PackageBaseAddress found in feed index") + + +def main() -> None: + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(2) + try: + print(resolve(sys.argv[1])) + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/eng/versions/mssql-py-core.version b/eng/versions/mssql-py-core.version new file mode 100644 index 000000000..56099240e --- /dev/null +++ b/eng/versions/mssql-py-core.version @@ -0,0 +1 @@ +0.1.0-dev.20260222.140833 diff --git a/setup.py b/setup.py index 352e2b201..61db7a088 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,13 @@ import os import sys +from pathlib import Path + from setuptools import setup, find_packages from setuptools.dist import Distribution from wheel.bdist_wheel import bdist_wheel +PROJECT_ROOT = Path(__file__).resolve().parent + # Custom distribution to force platform-specific wheel class BinaryDistribution(Distribution): @@ -53,7 +57,49 @@ def get_platform_info(): ) -# Custom bdist_wheel command to override platform tag +# --------------------------------------------------------------------------- +# mssql_py_core validation +# --------------------------------------------------------------------------- +def validate_mssql_py_core(): + """Validate that mssql_py_core has been extracted into the project root. + + Expects ``/mssql_py_core/`` to contain: + - ``__init__.py`` + - At least one native extension (``.pyd`` on Windows, ``.so`` on Linux/macOS) + + The extraction is performed by ``eng/scripts/install-mssql-py-core.ps1`` + (Windows) or ``eng/scripts/install-mssql-py-core.sh`` (Linux/macOS) + and must be run before ``setup.py bdist_wheel``. + + Raises SystemExit if mssql_py_core is missing or invalid. + """ + core_dir = PROJECT_ROOT / "mssql_py_core" + + if not core_dir.is_dir(): + sys.exit( + "ERROR: mssql_py_core/ directory not found in project root. " + "Run eng/scripts/install-mssql-py-core to extract it before building." + ) + + # Check for __init__.py + if not (core_dir / "__init__.py").is_file(): + sys.exit("ERROR: mssql_py_core/__init__.py not found.") + + # Check for native extension (.pyd on Windows, .so on Linux/macOS) + ext = ".pyd" if sys.platform.startswith("win") else ".so" + native_files = list(core_dir.glob(f"mssql_py_core*{ext}")) + if not native_files: + sys.exit( + f"ERROR: No mssql_py_core native extension ({ext}) found " + f"in mssql_py_core/. Run eng/scripts/install-mssql-py-core to extract it." + ) + + for f in native_files: + print(f" Found mssql_py_core native extension: {f.name}") + + print("mssql_py_core validation: OK") + + class CustomBdistWheel(bdist_wheel): def finalize_options(self): # Call the original finalize_options first to initialize self.bdist_dir @@ -64,6 +110,14 @@ def finalize_options(self): self.plat_name = platform_tag print(f"Setting wheel platform tag to: {self.plat_name} (arch: {arch})") + def run(self): + validate_mssql_py_core() + bdist_wheel.run(self) + + +# --------------------------------------------------------------------------- +# Package discovery +# --------------------------------------------------------------------------- # Find all packages in the current directory packages = find_packages() @@ -72,6 +126,11 @@ def finalize_options(self): arch, platform_tag = get_platform_info() print(f"Detected architecture: {arch} (platform tag: {platform_tag})") +# mssql_py_core is validated inside CustomBdistWheel.run() so that editable +# installs (pip install -e .) and other setup.py commands are not blocked. +if (PROJECT_ROOT / "mssql_py_core").is_dir(): + packages.append("mssql_py_core") + # Add platform-specific packages if sys.platform.startswith("win"): packages.extend( @@ -94,6 +153,24 @@ def finalize_options(self): ] ) +# --------------------------------------------------------------------------- +# package_data – binaries to include in the wheel +# --------------------------------------------------------------------------- +package_data = { + "mssql_python": [ + "py.typed", + "ddbc_bindings.cp*.pyd", + "ddbc_bindings.cp*.so", + "libs/*", + "libs/**/*", + "*.dll", + ], + "mssql_py_core": [ + "mssql_py_core.cp*.pyd", + "mssql_py_core.cp*.so", + ], +} + setup( name="mssql-python", version="1.3.0", @@ -104,17 +181,7 @@ def finalize_options(self): author_email="mssql-python@microsoft.com", url="https://github.com/microsoft/mssql-python", packages=packages, - package_data={ - # Include PYD and DLL files inside mssql_python, exclude YML files - "mssql_python": [ - "py.typed", # Marker file for PEP 561 typing support - "ddbc_bindings.cp*.pyd", # Include all PYD files - "ddbc_bindings.cp*.so", # Include all SO files - "libs/*", - "libs/**/*", - "*.dll", - ] - }, + package_data=package_data, include_package_data=True, # Requires >= Python 3.10 python_requires=">=3.10", diff --git a/tests/test_019_bulkcopy.py b/tests/test_019_bulkcopy.py new file mode 100644 index 000000000..a50e5d6c6 --- /dev/null +++ b/tests/test_019_bulkcopy.py @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Basic integration tests for bulkcopy via the mssql_python driver.""" + +import pytest + +# Skip the entire module when mssql_py_core can't be loaded at runtime +# (e.g. manylinux_2_28 build containers where glibc is too old for the .so). +mssql_py_core = pytest.importorskip( + "mssql_py_core", reason="mssql_py_core not loadable (glibc too old?)" +) + + +def test_connection_and_cursor(cursor): + """Test that connection and cursor work correctly.""" + cursor.execute("SELECT 1 AS connected") + result = cursor.fetchone() + assert result[0] == 1 + + +def test_insert_and_fetch(cursor): + """Test basic insert and fetch operations.""" + table_name = "mssql_python_test_basic" + + # Create table + cursor.execute(f"IF OBJECT_ID('{table_name}', 'U') IS NOT NULL DROP TABLE {table_name}") + cursor.execute(f"CREATE TABLE {table_name} (id INT, name NVARCHAR(50))") + cursor.connection.commit() + + # Insert data + cursor.execute(f"INSERT INTO {table_name} (id, name) VALUES (?, ?)", (1, "Alice")) + cursor.execute(f"INSERT INTO {table_name} (id, name) VALUES (?, ?)", (2, "Bob")) + + # Fetch and verify + cursor.execute(f"SELECT id, name FROM {table_name} ORDER BY id") + rows = cursor.fetchall() + + assert len(rows) == 2 + assert rows[0][0] == 1 and rows[0][1] == "Alice" + assert rows[1][0] == 2 and rows[1][1] == "Bob" + + # Cleanup + cursor.execute(f"DROP TABLE {table_name}") + + +def test_bulkcopy_basic(cursor): + """Test basic bulkcopy operation via mssql_python driver with auto-mapping. + + Uses automatic column mapping (columns mapped by ordinal position). + """ + table_name = "mssql_python_bulkcopy_test" + + # Create table + cursor.execute(f"IF OBJECT_ID('{table_name}', 'U') IS NOT NULL DROP TABLE {table_name}") + cursor.execute(f"CREATE TABLE {table_name} (id INT, name VARCHAR(50), value FLOAT)") + cursor.connection.commit() + + # Prepare test data - columns match table order (id, name, value) + data = [ + (1, "Alice", 100.5), + (2, "Bob", 200.75), + (3, "Charlie", 300.25), + ] + + # Perform bulkcopy with auto-mapping (no column_mappings specified) + # Using explicit timeout parameter instead of kwargs + result = cursor._bulkcopy(table_name, data, timeout=60) + + # Verify result + assert result is not None + assert result["rows_copied"] == 3 + + # Verify data was inserted correctly + cursor.execute(f"SELECT id, name, value FROM {table_name} ORDER BY id") + rows = cursor.fetchall() + + assert len(rows) == 3 + assert rows[0][0] == 1 and rows[0][1] == "Alice" and abs(rows[0][2] - 100.5) < 0.01 + assert rows[1][0] == 2 and rows[1][1] == "Bob" and abs(rows[1][2] - 200.75) < 0.01 + assert rows[2][0] == 3 and rows[2][1] == "Charlie" and abs(rows[2][2] - 300.25) < 0.01 + + # Cleanup + cursor.execute(f"DROP TABLE {table_name}")