feat: use python3.11+ for cibuildwheel driver (#1912)

* chore: use SPEC 0 schedule for cibuildwheel

* remove support for {python} and {pip} in commands

* Remove specific Python versions from the update-dependencies job

The requirements pinning is done by uv now, so we don't need to
run the versions of Python to do the pinning anymore.

---------

Co-authored-by: Joe Rickerby <[email protected]>

Author: mayeut

.cirrus.yml

@@ -7,7 +7,6 @@ run_tests: &RUN_TESTS
   run_cibuildwheel_tests_script:
     - python ./bin/run_tests.py
 
-
 linux_x86_task:
   timeout_in: 120m
   compute_engine_instance:
@@ -16,10 +15,15 @@ linux_x86_task:
     platform: linux
     cpu: 8
     memory: 8G
-
+  env:
+    VENV_ROOT: ${HOME}/venv-cibuildwheel
+    PATH: ${VENV_ROOT}/bin:${PATH}
   install_pre_requirements_script:
     - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
-    - apt install -y python3-venv python-is-python3
+    - add-apt-repository -y ppa:deadsnakes/ppa
+    - apt-get update
+    - apt-get install -y python3.12-venv
+    - python3.12 -m venv ${VENV_ROOT}
   <<: *RUN_TESTS
 
 linux_aarch64_task:
@@ -30,10 +34,15 @@ linux_aarch64_task:
     platform: linux
     cpu: 4
     memory: 4G
-
+  env:
+    VENV_ROOT: ${HOME}/venv-cibuildwheel
+    PATH: ${VENV_ROOT}/bin:${PATH}
   install_pre_requirements_script:
     - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
-    - apt install -y python3-venv python-is-python3
+    - add-apt-repository -y ppa:deadsnakes/ppa
+    - apt-get update
+    - apt-get install -y python3.12-venv
+    - python3.12 -m venv ${VENV_ROOT}
   <<: *RUN_TESTS
 
 windows_x86_task:
@@ -46,30 +55,32 @@ windows_x86_task:
     memory: 8G
 
   install_pre_requirements_script:
-    - choco install -y --no-progress python3 --version 3.10.6
+    - choco install -y --no-progress python3 --version 3.12.4
     - refreshenv
     - echo PATH=%PATH% >> "%CIRRUS_ENV%"
   <<: *RUN_TESTS
 
 macos_arm64_task:
   macos_instance:
     image: ghcr.io/cirruslabs/macos-runner:sonoma
-
   env:
-    PATH: /opt/homebrew/opt/[email protected]/libexec/bin:$PATH
+    VENV_ROOT: ${HOME}/venv-cibuildwheel
+    PATH: ${VENV_ROOT}/bin:${PATH}
   install_pre_requirements_script:
-    - brew install [email protected]
+    - brew install [email protected]
+    - python3.12 -m venv ${VENV_ROOT}
   <<: *RUN_TESTS
 
 macos_arm64_cp38_task:
   macos_instance:
     image: ghcr.io/cirruslabs/macos-runner:sonoma
-
   env:
-    PATH: /opt/homebrew/opt/[email protected]/libexec/bin:$PATH
+    VENV_ROOT: ${HOME}/venv-cibuildwheel
+    PATH: ${VENV_ROOT}/bin:${PATH}
     PYTEST_ADDOPTS: --run-cp38-universal2 -k 'test_cp38_arm64_testing_universal2_installer or test_arch_auto'
   install_pre_requirements_script:
-    - brew install [email protected]
+    - brew install [email protected]
+    - python3.12 -m venv ${VENV_ROOT}
     - curl -fsSLO https://www.python.org/ftp/python/3.8.10/python-3.8.10-macos11.pkg
     - sudo installer -pkg python-3.8.10-macos11.pkg -target /
     - rm python-3.8.10-macos11.pkg

.github/workflows/test.yml

@@ -41,7 +41,7 @@ jobs:
         python_version: ['3.13']
         include:
         - os: ubuntu-latest
-          python_version: '3.8'
+          python_version: '3.11'
     timeout-minutes: 180
     steps:
     - uses: actions/checkout@v4

.github/workflows/update-dependencies.yml

@@ -33,8 +33,6 @@ jobs:
     - uses: actions/checkout@v4
 
     - uses: wntrblm/[email protected]
-      with:
-        python-versions: "3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13"
 
     - name: "Run update: dependencies"
       run: nox --force-color -s update_constraints

.pre-commit-config.yaml

@@ -24,9 +24,9 @@ repos:
   rev: v1.14.1
   hooks:
   - id: mypy
-    name: mypy 3.8 on cibuildwheel/
+    name: mypy 3.11 on cibuildwheel/
     exclude: ^cibuildwheel/resources/.*py|bin/generate_schema.py$
-    args: ["--python-version=3.8"]
+    args: ["--python-version=3.11"]
     additional_dependencies: &mypy-dependencies
       - bracex
       - dependency-groups>=1.2
@@ -47,9 +47,9 @@ repos:
       - uv
       - validate-pyproject
   - id: mypy
-    name: mypy 3.12
+    name: mypy 3.13
     exclude: ^cibuildwheel/resources/.*py$
-    args: ["--python-version=3.12"]
+    args: ["--python-version=3.13"]
     additional_dependencies: *mypy-dependencies
 
 - repo: https://github.com/shellcheck-py/shellcheck-py

.travis.yml

@@ -8,13 +8,13 @@ branches:
 
 jobs:
   include:
-    - name: Linux | x86_64 + i686 | Python 3.9
-      python: 3.9
+    - name: Linux | x86_64 + i686 | Python 3.11
+      python: 3.11
       services: docker
       env: PYTHON=python
 
-    - name: Linux | arm64 | Python 3.9
-      python: 3.9
+    - name: Linux | arm64 | Python 3.11
+      python: 3.11
       services: docker
       arch: arm64-graviton2
       group: edge
@@ -29,8 +29,8 @@ jobs:
           packages:
             - docker-ce docker-ce-cli containerd.io
 
-    - name: Linux | ppc64le | Python 3.9
-      python: 3.9
+    - name: Linux | ppc64le | Python 3.11
+      python: 3.11
       services: docker
       arch: ppc64le
       allow_failure: True
@@ -40,16 +40,16 @@ jobs:
         # c.f. https://travis-ci.community/t/running-out-of-disk-space-quota-when-using-docker-on-ppc64le/11634
         - PYTEST_ADDOPTS='-k "not test_manylinuxXXXX_only"'
 
-    - name: Windows | x86_64 | Python 3.9
+    - name: Windows | x86_64 | Python 3.11
       os: windows
       language: shell
       before_install:
-        - choco upgrade python3 -y --version 3.9.13 --limit-output --params "/InstallDir:C:\\Python39"
+        - choco upgrade python3 -y --version 3.11.9 --limit-output --params "/InstallDir:C:\\Python311"
       env:
-        - PYTHON=C:\\Python39\\python
+        - PYTHON=C:\\Python311\\python
 
-    - name: Linux | s390x | Python 3.9
-      python: 3.9
+    - name: Linux | s390x | Python 3.11
+      python: 3.11
       services: docker
       arch: s390x
       allow_failure: True

CI.md

@@ -1,11 +1,11 @@
 This is a summary of the host Python versions and platforms covered by the different CI platforms:
 
-|         | 3.8                              | 3.9       | 3.10      | 3.11    | 3.12                                             |
-|---------|----------------------------------|-----------|-----------|---------|--------------------------------------------------|
-| Linux   | Azure Pipelines / GitHub Actions | Travis CI | Cirrus CI |         | AppVeyor¹ / CircleCI¹ / GitHub Actions / GitLab¹ |
-| macOS   | Azure Pipelines                  |           | Cirrus CI | GitLab¹ | AppVeyor¹ /CircleCI¹ / GitHub Actions            |
-| Windows | Azure Pipelines                  | Travis CI | Cirrus CI |         | AppVeyor¹ / GitHub Actions / GitLab¹             |
+|         | 3.11                                         | 3.12                                        | 3.13           |
+|---------|----------------------------------------------|---------------------------------------------|----------------|
+| Linux   | Azure Pipelines / GitHub Actions / Travis CI | AppVeyor¹ / CircleCI¹ / Cirrus CI / GitLab¹ | GitHub Actions |
+| macOS   | Azure Pipelines / GitLab¹                    | AppVeyor¹ / CircleCI¹ / Cirrus CI / GitLab¹ | GitHub Actions |
+| Windows | Azure Pipelines / Travis CI                  | AppVeyor¹ / Cirrus CI / GitLab¹             | GitHub Actions |
 
 > ¹ Runs a reduced set of tests to reduce CI load
 
-Non-x86 architectures are covered on Travis CI using Python 3.9.
+Non-x86 architectures are covered on Travis CI using Python 3.11.

azure-pipelines.yml

@@ -5,37 +5,37 @@ pr:
       - .pre-commit-config.yaml
 
 jobs:
-- job: linux_38
+- job: linux_311
   timeoutInMinutes: 120
   pool: {vmImage: 'Ubuntu-20.04'}
   steps:
     - task: UsePythonVersion@0
       inputs:
-        versionSpec: '3.8'
+        versionSpec: '3.11'
     - bash: |
         docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
         python -m pip install dependency-groups
         python -m dependency_groups test | xargs python -m pip install -e.
         python ./bin/run_tests.py
 
-- job: macos_38
+- job: macos_311
   pool: {vmImage: 'macOS-13'}
   steps:
     - task: UsePythonVersion@0
       inputs:
-        versionSpec: '3.8'
+        versionSpec: '3.11'
     - bash: |
         python -m pip install dependency-groups
         python -m dependency_groups test | xargs python -m pip install -e.
         python ./bin/run_tests.py --num-processes 2
 
-- job: windows_38
+- job: windows_311
   pool: {vmImage: 'windows-2019'}
   timeoutInMinutes: 180
   steps:
     - task: UsePythonVersion@0
       inputs:
-        versionSpec: '3.8'
+        versionSpec: '3.11'
     - bash: |
         python -m pip install dependency-groups
         python -m dependency_groups test | xargs python -m pip install -e.

bin/bump_version.py

@@ -11,17 +11,13 @@
 import os
 import subprocess
 import sys
+import tomllib
 import urllib.parse
 from pathlib import Path
 
 import click
 from packaging.version import InvalidVersion, Version
 
-if sys.version_info < (3, 11):
-    import tomli as tomllib
-else:
-    import tomllib
-
 config = [
     # file path, version find/replace format
     ("pyproject.toml", 'version = "{}"'),

bin/update_nodejs.py

@@ -4,6 +4,7 @@
 
 import difflib
 import logging
+import tomllib
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Final
@@ -16,8 +17,6 @@
 from rich.logging import RichHandler
 from rich.syntax import Syntax
 
-from cibuildwheel._compat import tomllib
-
 log = logging.getLogger("cibw")
 
 # Looking up the dir instead of using utils.resources_dir

bin/update_pythons.py

@@ -5,9 +5,10 @@
 import copy
 import difflib
 import logging
+import tomllib
 from collections.abc import Mapping, MutableMapping
 from pathlib import Path
-from typing import Any, Final, Literal, TypedDict, Union
+from typing import Any, Final, Literal, TypedDict
 
 import click
 import requests
@@ -17,7 +18,6 @@
 from rich.logging import RichHandler
 from rich.syntax import Syntax
 
-from cibuildwheel._compat import tomllib
 from cibuildwheel.extra import dump_python_configurations
 
 log = logging.getLogger("cibw")
@@ -50,7 +50,7 @@ class ConfigMacOS(TypedDict):
     url: str
 
 
-AnyConfig = Union[ConfigWinCP, ConfigWinPP, ConfigMacOS]
+AnyConfig = ConfigWinCP | ConfigWinPP | ConfigMacOS
 
 
 # The following set of "Versions" classes allow the initial call to the APIs to

bin/update_virtualenv.py

@@ -5,6 +5,7 @@
 import difflib
 import logging
 import subprocess
+import tomllib
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Final
@@ -15,8 +16,6 @@
 from rich.logging import RichHandler
 from rich.syntax import Syntax
 
-from cibuildwheel._compat import tomllib
-
 log = logging.getLogger("cibw")
 
 # Looking up the dir instead of using utils.resources_dir

cibuildwheel/__main__.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import contextlib
 import dataclasses
 import os
 import shutil
@@ -12,7 +13,7 @@
 from collections.abc import Iterable, Sequence, Set
 from pathlib import Path
 from tempfile import mkdtemp
-from typing import Protocol
+from typing import Protocol, assert_never
 
 import cibuildwheel
 import cibuildwheel.linux
@@ -21,7 +22,6 @@
 import cibuildwheel.util
 import cibuildwheel.windows
 from cibuildwheel import errors
-from cibuildwheel._compat.typing import assert_never
 from cibuildwheel.architecture import Architecture, allowed_architectures_check
 from cibuildwheel.logger import log
 from cibuildwheel.options import CommandLineArguments, Options, compute_options
@@ -31,7 +31,6 @@
     BuildSelector,
     CIProvider,
     Unbuffered,
-    chdir,
     detect_ci_provider,
     fix_ansi_codes_for_github_actions,
     strtobool,
@@ -200,7 +199,7 @@ def main_inner(global_options: GlobalOptions) -> None:
         # This is now the new package dir
         args.package_dir = project_dir.resolve()
 
-        with chdir(project_dir):
+        with contextlib.chdir(project_dir):
             build_in_directory(args)
     finally:
         # avoid https://github.com/python/cpython/issues/86962 by performing
@@ -418,10 +417,10 @@ def detect_warnings(*, options: Options, identifiers: Iterable[str]) -> list[str
         if any(o and ("{python}" in o or "{pip}" in o) for o in option_values):
             # Reminder: in an f-string, double braces means literal single brace
             msg = (
-                f"{option_name}: '{{python}}' and '{{pip}}' are no longer needed, "
-                "and will be removed in cibuildwheel 3. Simply use 'python' or 'pip' instead."
+                f"{option_name}: '{{python}}' and '{{pip}}' are no longer supported "
+                "and have been removed in cibuildwheel 3. Simply use 'python' or 'pip' instead."
             )
-            warnings.append(msg)
+            raise errors.ConfigurationError(msg)
 
     return warnings