gh-143120: pixi builds for free-threading and TSAN

Tools/pixi-packages/README.md

@@ -12,15 +12,17 @@ python = { git = "https://github.com/python/cpython", subdirectory = "Tools/pixi
 ```
 
 This is particularly useful when developers need to build CPython from source
-(for example, for an ASan-instrumented build), as it does not require any manual
+(for example, for an ASan or TSan-instrumented build), as it does not require any manual
 clone or build steps. Instead, Pixi will automatically handle both the build
 and installation of the package.
 
 Each package definition is contained in a subdirectory, but they share the build script
 `build.sh` in this directory. Currently defined package variants:
 
 - `default`
-- `asan`: ASan-instrumented build with `PYTHON_ASAN=1`
+- `free-threading`
+- `asan`: ASan-instrumented build
+- `tsan-free-threading`: TSan-instrumented free-threading build
 
 ## Maintenance
 
@@ -30,7 +32,22 @@ Each package definition is contained in a subdirectory, but they share the build
 
 ## Opportunities for future improvement
 
-- More package variants (such as TSan, UBSan)
+- More package variants (such as UBSan)
 - Support for Windows
 - Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/4599
 - A workaround can be removed from the build script once https://github.com/prefix-dev/rattler-build/issues/2012 is resolved
+
+## Troubleshooting
+
+TSan builds may crash on Linux with
+```
+FATAL: ThreadSanitizer: unexpected memory mapping 0x7977bd072000-0x7977bd500000
+```
+To fix it, try reducing `mmap_rnd_bits`:
+
+```bash
+$ sudo sysctl vm.mmap_rnd_bits
+vm.mmap_rnd_bits = 32  # too high for TSan
+$ sudo sysctl vm.mmap_rnd_bits=28  # reduce it
+vm.mmap_rnd_bits = 28
+```

Tools/pixi-packages/build.sh

@@ -1,11 +1,19 @@
 #!/bin/bash
 
-if [[ "${PYTHON_VARIANT}" == "asan" ]]; then
+if [[ "${PYTHON_VARIANT}" == "free-threading" ]]; then
+    echo "BUILD TYPE: FREE-THREADING"
+    BUILD_DIR="../build_free_threading"
+    CONFIGURE_EXTRA="--disable-gil"
+elif [[ "${PYTHON_VARIANT}" == "asan" ]]; then
     echo "BUILD TYPE: ASAN"
     BUILD_DIR="../build_asan"
     CONFIGURE_EXTRA="--with-address-sanitizer"
-    export PYTHON_ASAN="1"
     export ASAN_OPTIONS="strict_init_order=true"
+elif [[ "${PYTHON_VARIANT}" == "tsan-free-threading" ]]; then
+    echo "BUILD TYPE: TSAN FREE-THREADING"
+    BUILD_DIR="../build_tsan_free_threading"
+    CONFIGURE_EXTRA="--disable-gil --with-thread-sanitizer"
+    export TSAN_OPTIONS="suppressions=${SRC_DIR}/Tools/tsan/suppressions_free_threading.txt"
 else
     echo "BUILD TYPE: DEFAULT"
     BUILD_DIR="../build"
@@ -33,5 +41,6 @@ ln -sf "${PREFIX}/bin/python3" "${PREFIX}/bin/python"
 
 # https://github.com/prefix-dev/rattler-build/issues/2012
 if [[ ${OSTYPE} == "darwin"* ]]; then
-    cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/libclang_rt.asan_osx_dynamic.dylib"
+    cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/"
+    cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.tsan_osx_dynamic.dylib" "${PREFIX}/lib/"
 fi

Tools/pixi-packages/free-threading/pixi.toml

@@ -0,0 +1,8 @@
+[workspace]
+channels = ["https://prefix.dev/conda-forge"]
+platforms = ["osx-arm64", "linux-64"]
+preview = ["pixi-build"]
+
+[package.build.backend]
+name = "pixi-build-rattler-build"
+version = "*"

Tools/pixi-packages/free-threading/recipe.yaml

@@ -0,0 +1,63 @@
+context:
+  # Keep up to date
+  version: "3.15"
+
+package:
+  name: python
+  version: ${{ version }}
+
+source:
+  - path: ../../..
+
+build:
+  files:
+    exclude:
+      - "*.o"
+  script:
+    file: ../build.sh
+    env:
+      PYTHON_VARIANT: "free-threading"
+
+# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+requirements:
+  build:
+    - ${{ compiler('c') }}
+    - ${{ compiler('cxx') }}
+    - make
+    - pkg-config
+    # configure script looks for llvm-ar for lto
+    - if: osx
+      then:
+        - llvm-tools
+    - if: linux
+      then:
+        - ld_impl_${{ target_platform }}
+        - binutils_impl_${{ target_platform }}
+    - clang-19
+    - llvm-tools-19
+
+  host:
+    - bzip2
+    - sqlite
+    - liblzma-devel
+    - zlib
+    - zstd
+    - openssl
+    - readline
+    - tk
+    # These two are just to get the headers needed for tk.h, but is unused
+    - xorg-libx11
+    - xorg-xorgproto
+    - ncurses
+    - libffi
+    - if: linux
+      then:
+        - ld_impl_${{ target_platform }}
+        - libuuid
+    - libmpdec-devel
+    - expat
+
+about:
+  homepage: https://www.python.org/
+  license: Python-2.0
+  license_file: LICENSE

Tools/pixi-packages/tsan-free-threading/pixi.toml

@@ -0,0 +1,8 @@
+[workspace]
+channels = ["https://prefix.dev/conda-forge"]
+platforms = ["osx-arm64", "linux-64"]
+preview = ["pixi-build"]
+
+[package.build.backend]
+name = "pixi-build-rattler-build"
+version = "*"

Tools/pixi-packages/tsan-free-threading/recipe.yaml

@@ -0,0 +1,63 @@
+context:
+  # Keep up to date
+  version: "3.15"
+
+package:
+  name: python
+  version: ${{ version }}
+
+source:
+  - path: ../../..
+
+build:
+  files:
+    exclude:
+      - "*.o"
+  script:
+    file: ../build.sh
+    env:
+      PYTHON_VARIANT: "tsan-free-threading"
+
+# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+requirements:
+  build:
+    - ${{ compiler('c') }}
+    - ${{ compiler('cxx') }}
+    - make
+    - pkg-config
+    # configure script looks for llvm-ar for lto
+    - if: osx
+      then:
+        - llvm-tools
+    - if: linux
+      then:
+        - ld_impl_${{ target_platform }}
+        - binutils_impl_${{ target_platform }}
+    - clang-19
+    - llvm-tools-19
+
+  host:
+    - bzip2
+    - sqlite
+    - liblzma-devel
+    - zlib
+    - zstd
+    - openssl
+    - readline
+    - tk
+    # These two are just to get the headers needed for tk.h, but is unused
+    - xorg-libx11
+    - xorg-xorgproto
+    - ncurses
+    - libffi
+    - if: linux
+      then:
+        - ld_impl_${{ target_platform }}
+        - libuuid
+    - libmpdec-devel
+    - expat
+
+about:
+  homepage: https://www.python.org/
+  license: Python-2.0
+  license_file: LICENSE