conflicts resolved

.builders/deps/build_dependencies.txt

@@ -1,6 +1,6 @@
 hatchling==1.21.1; python_version > '3.0'
 hatchling==0.25.1; python_version < '3.0'
-setuptools==66.1.1; python_version > '3.0'
+setuptools==75.6.0; python_version > '3.0'
 setuptools==40.9.0; python_version < '3.0'
 wheel==0.38.4; python_version > '3.0'
 wheel==0.37.1; python_version < '3.0'

.builders/images/windows-x86_64/Dockerfile

@@ -35,6 +35,15 @@ RUN curl -SL --output PowerShell-%POWERSHELL_VERSION%-win-x64.msi https://github
 COPY helpers.ps1 C:\helpers.ps1
 SHELL ["pwsh", "-Command", ". C:\\helpers.ps1;"]
 
+# Enable long paths
+# https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=powershell#registry-setting-to-enable-long-paths
+RUN New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" `
+ -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force
+# Reduce the chance of hitting path limits (the MSVC compiler cl.exe doesn't seem to respect that optioin)
+# This variable is honored by pip
+ENV TMP="C:\tmp" `
+    TEMP="C:\tmp"
+
 # Install 7-Zip ZS
 ENV 7ZIP_VERSION="22.01" `
     7ZIP_ZS_VERSION="1.5.5-R3"
@@ -95,10 +104,36 @@ RUN Get-RemoteFile `
     Remove-Item $Env:IBM_MQ_VERSION-IBM-MQC-Redist-Win64.zip; `
     setx /M MQ_FILE_PATH 'C:\ibm_mq'
 
+# Perl
+ENV PERL_VERSION="5.40.0.1"
+RUN Get-RemoteFile `
+      -Uri https://github.com/StrawberryPerl/Perl-Dist-Strawberry/releases/download/SP_54001_64bit_UCRT/strawberry-perl-$Env:PERL_VERSION-64bit-portable.zip `
+      -Path "strawberry-perl-$Env:PERL_VERSION-64bit.zip" `
+    -Hash '754f3e2a8e473dc68d1540c7802fb166a025f35ef18960c4564a31f8b5933907' && `
+    7z x "strawberry-perl-$Env:PERL_VERSION-64bit.zip" -o"C:\perl" && `
+    Add-ToPath -Append "C:\perl\perl\bin" && `
+    Remove-Item "strawberry-perl-$Env:PERL_VERSION-64bit.zip"
+
+ENV OPENSSL_VERSION="3.3.2"
+
 # Set up runner
 COPY runner_dependencies.txt C:\runner_dependencies.txt
 RUN python -m pip install --no-warn-script-location -r C:\runner_dependencies.txt
 
+COPY build_script.ps1 C:\build_script.ps1
+COPY update_librdkafka_manifest.py C:\update_librdkafka_manifest.py
+ENV DD_BUILD_COMMAND="pwsh C:\build_script.ps1"
+
+# Python packages that we want to build regardless of whether prebuilt versions exist on PyPI
+ENV PIP_NO_BINARY="confluent_kafka"
+# Where to find native dependencies when building extensions and for wheel repairing
+RUN New-Item -Path "C:\include" -ItemType Directory
+RUN New-Item -Path "C:\lib" -ItemType Directory
+RUN New-Item -Path "C:\bin" -ItemType Directory
+ENV INCLUDE="C:\include"
+ENV LIB="C:\lib"
+RUN Add-ToPath -Append "C:\bin"
+
 # Restore the default Windows shell for correct batch processing.
 SHELL ["cmd", "/S", "/C"]
 

.builders/images/windows-x86_64/build_script.ps1

@@ -0,0 +1,57 @@
+$ErrorActionPreference = 'Stop'
+$PSNativeCommandUseErrorActionPreference = $true
+
+. C:\helpers.ps1
+
+# The librdkafka version needs to stay in sync with the confluent-kafka version,
+# thus we extract the version from the requirements file
+$kafka_version = Get-Content 'C:\mnt\requirements.in' | perl -nE 'say/^\D*(\d+\.\d+\.\d+)\D*$/ if /confluent-kafka==/'
+Write-Host "Will build librdkafka $kafka_version"
+
+# Download and unpack the source
+Get-RemoteFile `
+  -Uri "https://github.com/confluentinc/librdkafka/archive/refs/tags/v${kafka_version}.tar.gz" `
+  -Path "librdkafka-${kafka_version}.tar.gz" `
+  -Hash '0ddf205ad8d36af0bc72a2fec20639ea02e1d583e353163bf7f4683d949e901b'
+7z x "librdkafka-${kafka_version}.tar.gz" -o"C:\"
+7z x "C:\librdkafka-${kafka_version}.tar" -o"C:\librdkafka"
+Remove-Item "librdkafka-${kafka_version}.tar.gz"
+
+# Build librdkafka
+# Based on this job from upstream:
+# https://github.com/confluentinc/librdkafka/blob/cb8c19c43011b66c4b08b25e5150455a247e1ff3/.semaphore/semaphore.yml#L265
+# Install vcpkg
+Set-Location "C:\"
+$triplet = "x64-windows"
+$librdkafka_dir = "C:\librdkafka\librdkafka-${kafka_version}"
+
+& "${librdkafka_dir}\win32\setup-vcpkg.ps1"
+# Get deps
+Set-Location "$librdkafka_dir"
+# Patch the the vcpkg manifest to to override the OpenSSL version
+python C:\update_librdkafka_manifest.py vcpkg.json --set-version openssl:${Env:OPENSSL_VERSION}
+
+C:\vcpkg\vcpkg integrate install
+C:\vcpkg\vcpkg --feature-flags=versions install --triplet $triplet
+# Build
+& .\win32\msbuild.ps1 -platform x64
+
+# Copy outputs to where they can be found
+# This is partially inspired by
+# https://github.com/confluentinc/librdkafka/blob/cb8c19c43011b66c4b08b25e5150455a247e1ff3/win32/package-zip.ps1
+$toolset = "v142"
+$platform = "x64"
+$config = "Release"
+$srcdir = "win32\outdir\${toolset}\${platform}\$config"
+$bindir = "C:\bin"
+$libdir = "C:\lib"
+$includedir = "C:\include"
+
+Copy-Item "${srcdir}\librdkafka.dll","${srcdir}\librdkafkacpp.dll",
+"${srcdir}\libcrypto-3-x64.dll","${srcdir}\libssl-3-x64.dll",
+"${srcdir}\zlib1.dll","${srcdir}\zstd.dll","${srcdir}\libcurl.dll" -Destination $bindir
+Copy-Item "${srcdir}\librdkafka.lib","${srcdir}\librdkafkacpp.lib" -Destination $libdir
+
+New-Item -Path $includedir\librdkafka -ItemType Directory
+Copy-Item -Path ".\src\*" -Filter *.h -Destination $includedir\librdkafka
+

.builders/images/windows-x86_64/update_librdkafka_manifest.py

@@ -0,0 +1,25 @@
+import json
+from argparse import ArgumentParser
+
+
+def main(manifest_file, versions):
+    with open(manifest_file) as f:
+        manifest = json.load(f)
+
+    for dep, version in versions.items():
+        manifest.setdefault("overrides", []).append({
+            "name": dep,
+            "version": version,
+        })
+
+    with open(manifest_file, 'w') as f:
+        json.dump(manifest, f)
+
+
+if __name__ == '__main__':
+    ap = ArgumentParser()
+    ap.add_argument("file")
+    ap.add_argument("--set-version", action="append", required=True)
+
+    args = ap.parse_args()
+    main(args.file, dict(spec.split(':') for spec in args.set_version))

.builders/scripts/repair_wheels.py

@@ -6,10 +6,12 @@
 import shutil
 import sys
 import time
+from fnmatch import fnmatch
 from functools import cache
 from hashlib import sha256
 from pathlib import Path
 from typing import Iterator, NamedTuple
+from zipfile import ZipFile
 
 import urllib3
 from utils import extract_metadata, normalize_project_name
@@ -64,6 +66,14 @@ def wheel_was_built(wheel: Path) -> bool:
     return file_hash != wheel_hashes[wheel.name]
 
 
+def find_patterns_in_wheel(wheel: Path, patterns: list[str]) -> list[str]:
+    """Returns all found files inside `wheel` that match the given glob-style pattern"""
+    with ZipFile(wheel) as zf:
+        names = zf.namelist()
+
+    return [name for name in names for pat in patterns if fnmatch(name, pat)]
+
+
 class WheelName(NamedTuple):
     """Helper class to manipulate wheel names."""
     # Note: this implementation ignores build tags (it drops them on parsing)
@@ -98,6 +108,12 @@ def repair_linux(source_dir: str, built_dir: str, external_dir: str) -> None:
         'libmqic_r.so',
     })
 
+    external_invalid_file_patterns = [
+        # We don't accept OpenSSL in external wheels
+        '*.libs/libssl*.so.3',
+        '*.libs/libcrypto*.so.3',
+    ]
+
     # Hardcoded policy to the minimum we need to currently support
     policies = WheelPolicies()
     policy = policies.get_policy_by_name(os.environ['MANYLINUX_POLICY'])
@@ -109,8 +125,19 @@ def repair_linux(source_dir: str, built_dir: str, external_dir: str) -> None:
 
     for wheel in iter_wheels(source_dir):
         print(f'--> {wheel.name}')
+
         if not wheel_was_built(wheel):
             print('Using existing wheel')
+
+            unacceptable_files = find_patterns_in_wheel(wheel, external_invalid_file_patterns)
+            if unacceptable_files:
+                print(
+                    f"Found copies of unacceptable files in external wheel '{wheel.name}'",
+                    f'(matching {external_invalid_file_patterns}): ',
+                    unacceptable_files,
+                )
+                sys.exit(1)
+
             shutil.move(wheel, external_dir)
             continue
 
@@ -140,11 +167,27 @@ def repair_windows(source_dir: str, built_dir: str, external_dir: str) -> None:
 
     exclusions = ['mqic.dll']
 
+    external_invalid_file_patterns = [
+        # We don't accept OpenSSL in external wheels
+        '*.libs/libssl-3*.dll',
+        '*.libs/libcrypto-3*.dll',
+    ]
+
     for wheel in iter_wheels(source_dir):
         print(f'--> {wheel.name}')
 
         if not wheel_was_built(wheel):
             print('Using existing wheel')
+
+            unacceptable_files = find_patterns_in_wheel(wheel, external_invalid_file_patterns)
+            if unacceptable_files:
+                print(
+                    f"Found copies of unacceptable files in external wheel '{wheel.name}'",
+                    f'(matching {external_invalid_file_patterns}): ',
+                    unacceptable_files,
+                )
+                sys.exit(1)
+
             shutil.move(wheel, external_dir)
             continue
 
@@ -206,6 +249,7 @@ def copy_filt_func(libname):
         print(f'--> {wheel.name}')
         if not wheel_was_built(wheel):
             print('Using existing wheel')
+
             shutil.move(wheel, external_dir)
             continue
 

.deps/image_digests.json

@@ -1,5 +1,5 @@
 {
-  "linux-aarch64": "sha256:0c67a49a4d4ec217dd0f841ee139eaf061616f6e61c6bc758617d4c50c7a8aa2",
-  "linux-x86_64": "sha256:5e421218e377e4c1d0769b148e569f4ff4a8c60fbd2be8411db9158a644a0b0a",
-  "windows-x86_64": "sha256:feefe940fe3f382bf4833bc29a9d614d6f6bb3592258a905a261167184b20eab"
+  "linux-aarch64": "sha256:e09d2c24c54286a660b8fab396f8a69dc58907551dda5623cb69212e29565019",
+  "linux-x86_64": "sha256:e5b6ea5078b388bfd3f6e99d3573bbd4d4491020613de23f6fc23d535f14387f",
+  "windows-x86_64": "sha256:d6fbd8983205cb9af2d17ee47110e985572e8d47617a1cb5a0cbbbfeb2e30292"
 }

.deps/metadata.json

@@ -1,3 +1,3 @@
 {
-  "sha256": "5983d33689b7f9b74eb94d7ff13b7eac3c0b2f8cd5084e8ded76c28ca5476a20"
+  "sha256": "4d5260e77bea1467dc82ebe382d0cbc42d3f9b62c64c36898f975393c42da56d"
 }