diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 4c53162..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,162 +0,0 @@ - -skip_branch_with_pr: true - -environment: - python_stack: python 3.12 - PYTHON_VERSION: 3.12.6 - PYTHON_VERSION_SHORT: 3.12 - GITHUB_TOKEN: - secure: 9SKIwc3VSfYJ5IChvNR74rlTF9BMbAfhCGu1/TmYJBMtC6lkY+UDDkZNK7rC9xnQFUxMrNgoo9kNcNAbKbU8XAcrSwkP2H4mX04FI7P+YbxfiWC8nVHhGNxR4LzO+GO0 - - matrix: - - job_name: Build Python for iOS and macOS - APPVEYOR_BUILD_WORKER_IMAGE: macos-sonoma - - - job_name: Build Python for Android - APPVEYOR_BUILD_WORKER_IMAGE: ubuntu-gce-c - NDK_VERSION: r27 - - - job_name: Build Python for Linux - APPVEYOR_BUILD_WORKER_IMAGE: ubuntu-gce-c - PYTHON_DIST_RELEASE: 20240909 - - - job_name: Build Python for Windows - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 - -matrix: - fast_finish: true - -stack: $python_stack - -install: -- python --version - -for: - # ====================================== - # Build Python for iOS and macOS - # ====================================== - - - matrix: - only: - - job_name: Build Python for iOS and macOS - - build_script: - - cd darwin - - # Build Python for iOS and macOS - - git clone --branch=$PYTHON_VERSION_SHORT https://github.com/beeware/Python-Apple-support.git - - mkdir -p dist - - sh: | - pushd Python-Apple-support - make iOS || exit 1 - tar -czf ../dist/python-ios-mobile-forge-$PYTHON_VERSION_SHORT.tar.gz install support -C . - make macOS || exit 1 - popd - - # Package for Dart - - ./package-ios-for-dart.sh Python-Apple-support $PYTHON_VERSION_SHORT - - ./package-macos-for-dart.sh Python-Apple-support $PYTHON_VERSION_SHORT - - # Push all archives to artifacts - - find dist -maxdepth 1 -type f -iname python-*.tar.gz -exec appveyor PushArtifact -DeploymentName python-darwin {} \; - - test: off - - deploy: - provider: GitHub - auth_token: $(GITHUB_TOKEN) - release: v$(PYTHON_VERSION_SHORT) - artifact: python-darwin - - # ====================================== - # Build Python for Android - # ====================================== - - - matrix: - only: - - job_name: Build Python for Android - - build_script: - - cd android - - # Build all Python ABIs - - ./build-all.sh $PYTHON_VERSION - - # Package support package for use with mobile-forge - - mkdir -p dist - - tar -czf dist/python-android-mobile-forge-$PYTHON_VERSION_SHORT.tar.gz install support - - # Package individual ABIs for use with serious_python Flutter package - - ./package-for-dart.sh install $PYTHON_VERSION arm64-v8a - - ./package-for-dart.sh install $PYTHON_VERSION armeabi-v7a - - ./package-for-dart.sh install $PYTHON_VERSION x86_64 - - # Push all archives to artifacts - - find dist -maxdepth 1 -type f -iname python-android-*.tar.gz -exec appveyor PushArtifact -DeploymentName python-android {} \; - - test: off - - deploy: - provider: GitHub - auth_token: $(GITHUB_TOKEN) - release: v$(PYTHON_VERSION_SHORT) - artifact: python-android - - # ====================================== - # Build Python for Linux - # ====================================== - - - matrix: - only: - - job_name: Build Python for Linux - - build_script: - - cd linux - - ./package-for-linux.sh x86_64 "_v3" - - ./package-for-linux.sh aarch64 "" - - # Push all archives to artifacts - - ls - - find . -maxdepth 1 -type f -iname "python-linux-dart-*.tar.gz" -exec appveyor PushArtifact -DeploymentName python-linux {} \; - - test: off - - deploy: - provider: GitHub - auth_token: $(GITHUB_TOKEN) - release: v$(PYTHON_VERSION_SHORT) - artifact: python-linux - - # ====================================== - # Build Python for Windows - # ====================================== - - - matrix: - only: - - job_name: Build Python for Windows - - install: - - C:\Python312\python --version - - build_script: - - cd windows - - curl -OL https://www.python.org/ftp/python/3.12.5/python-3.12.5-amd64.exe - - start /wait python-3.12.5-amd64.exe /uninstall /quiet - - - curl -OL https://www.python.org/ftp/python/%PYTHON_VERSION%/python-%PYTHON_VERSION%-amd64.exe - - start /wait python-%PYTHON_VERSION%-amd64.exe /quiet - - dir C:\python312-dist - - C:\python312-dist\python -m compileall -b C:\python312-dist\Lib - - 7z a -xr@exclude.txt python-windows-for-dart-%PYTHON_VERSION_SHORT%.zip C:\python312-dist\* - - test: off - - artifacts: - - path: windows\python-windows-for-dart-*.zip - name: python-windows - - deploy: - provider: GitHub - auth_token: $(GITHUB_TOKEN) - release: v$(PYTHON_VERSION_SHORT) - artifact: python-windows \ No newline at end of file diff --git a/.github/workflows/build-python-version.yml b/.github/workflows/build-python-version.yml new file mode 100644 index 0000000..3bb6f6c --- /dev/null +++ b/.github/workflows/build-python-version.yml @@ -0,0 +1,194 @@ +name: Build Python Version + +on: + workflow_call: + inputs: + python_version: + description: Full Python version (e.g. 3.13.12) + required: true + type: string + workflow_dispatch: + inputs: + python_version: + description: Full Python version (e.g. 3.13.12) + required: true + type: choice + options: + - 3.12.12 + - 3.13.12 + - 3.14.3 + +env: + PYTHON_VERSION: ${{ inputs.python_version || github.event.inputs.python_version }} + PYTHON_DIST_RELEASE: 20260203 # https://github.com/astral-sh/python-build-standalone/releases + +jobs: + build-darwin: + name: Build Python for iOS and macOS + runs-on: macos-15 + permissions: + contents: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Derive short Python version + shell: bash + run: | + echo "PYTHON_VERSION_SHORT=$(echo "$PYTHON_VERSION" | cut -d. -f1,2)" >> "$GITHUB_ENV" + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION_SHORT }} + + - name: Show Python version + run: python --version + + - name: Build Python for iOS and macOS + working-directory: darwin + shell: bash + run: | + git clone --branch="$PYTHON_VERSION_SHORT" https://github.com/beeware/Python-Apple-support.git + mkdir -p dist + + pushd Python-Apple-support + make iOS + tar -czf ../dist/python-ios-mobile-forge-$PYTHON_VERSION_SHORT.tar.gz install support -C . + make macOS + popd + + bash ./package-ios-for-dart.sh Python-Apple-support "$PYTHON_VERSION_SHORT" + bash ./package-macos-for-dart.sh Python-Apple-support "$PYTHON_VERSION_SHORT" + + - name: Upload Darwin build artifacts + uses: actions/upload-artifact@v4 + with: + name: python-darwin-${{ env.PYTHON_VERSION_SHORT }} + path: darwin/dist/python-*.tar.gz + if-no-files-found: error + + build-android: + name: Build Python for Android + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - name: Derive short Python version + shell: bash + run: | + echo "PYTHON_VERSION_SHORT=$(echo "$PYTHON_VERSION" | cut -d. -f1,2)" >> "$GITHUB_ENV" + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + - run: python --version + - working-directory: android + shell: bash + run: | + bash ./build-all.sh "$PYTHON_VERSION" + mkdir -p dist + tar -czf dist/python-android-mobile-forge-$PYTHON_VERSION_SHORT.tar.gz install support + bash ./package-for-dart.sh install "$PYTHON_VERSION" arm64-v8a + bash ./package-for-dart.sh install "$PYTHON_VERSION" x86_64 + read version_major version_minor < <(echo "$PYTHON_VERSION" | sed -E 's/^([0-9]+)\.([0-9]+).*/\1 \2/') + version_int=$((version_major * 100 + version_minor)) + if [ $version_int -lt 313 ]; then + bash ./package-for-dart.sh install "$PYTHON_VERSION" armeabi-v7a + fi + - uses: actions/upload-artifact@v4 + with: + name: python-android-${{ env.PYTHON_VERSION_SHORT }} + path: android/dist/python-android-*.tar.gz + if-no-files-found: error + + build-linux: + name: Build Python for Linux + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - name: Derive short Python version + shell: bash + run: | + echo "PYTHON_VERSION_SHORT=$(echo "$PYTHON_VERSION" | cut -d. -f1,2)" >> "$GITHUB_ENV" + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + - run: python --version + - working-directory: linux + shell: bash + run: | + bash ./package-for-linux.sh x86_64 "_v2" + bash ./package-for-linux.sh aarch64 "" + - uses: actions/upload-artifact@v4 + with: + name: python-linux-${{ env.PYTHON_VERSION_SHORT }} + path: linux/python-linux-dart-*.tar.gz + if-no-files-found: error + + build-windows: + name: Build Python for Windows + runs-on: windows-2022 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - name: Derive short Python version + shell: pwsh + run: | + $parts = "${{ env.PYTHON_VERSION }}".Split(".") + "PYTHON_VERSION_SHORT=$($parts[0]).$($parts[1])" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION_SHORT }} + - name: Show Python version + shell: pwsh + run: | + python --version + python -c "import sys; print(sys.executable)" + - name: Build CPython from sources and package for Dart + shell: pwsh + run: | + .\windows\package-for-dart.ps1 ` + -PythonVersion "${{ env.PYTHON_VERSION }}" ` + -PythonVersionShort "${{ env.PYTHON_VERSION_SHORT }}" + - uses: actions/upload-artifact@v4 + with: + name: python-windows-${{ env.PYTHON_VERSION_SHORT }} + path: windows/python-windows-for-dart-*.zip + if-no-files-found: error + + publish-release: + name: Publish Release Assets + runs-on: ubuntu-latest + needs: + - build-darwin + - build-android + - build-linux + - build-windows + permissions: + contents: write + steps: + - name: Derive short Python version + shell: bash + run: | + echo "PYTHON_VERSION_SHORT=$(echo "$PYTHON_VERSION" | cut -d. -f1,2)" >> "$GITHUB_ENV" + - name: Download all build artifacts + uses: actions/download-artifact@v4 + with: + pattern: python-*-${{ env.PYTHON_VERSION_SHORT }} + path: release-artifacts + merge-multiple: true + + - name: Publish all artifacts to release + uses: softprops/action-gh-release@v2 + with: + tag_name: v${{ env.PYTHON_VERSION_SHORT }} + files: release-artifacts/* + fail_on_unmatched_files: true + generate_release_notes: false + draft: false + prerelease: false diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml new file mode 100644 index 0000000..a2320a1 --- /dev/null +++ b/.github/workflows/build-python.yml @@ -0,0 +1,22 @@ +name: Build Python Packages + +on: + push: + branches: + - '**' + workflow_dispatch: + +jobs: + build-matrix: + name: Build Python ${{ matrix.python_version }} + strategy: + fail-fast: false + matrix: + python_version: + - 3.12.12 + - 3.13.12 + - 3.14.3 + uses: ./.github/workflows/build-python-version.yml + with: + python_version: ${{ matrix.python_version }} + secrets: inherit diff --git a/android/README.md b/android/README.md index 37aa5ef..d933a65 100644 --- a/android/README.md +++ b/android/README.md @@ -1,26 +1,30 @@ # Python for Android -Scripts and CI jobs for building Python 3 for Android. +Scripts and CI jobs for building Python for Android. * Can be run on both Linux and macOS. -* Build Python 3.12 - specific or the last minor version. -* Installs NDK r26d or use pre-installed one with path configured by `NDK_HOME` variable. -* Creates Python installation with a structure suitable for https://github.com/flet-dev/mobile-forge +* Python 3.12 uses the legacy patched cross-build flow. +* Python 3.13+ uses CPython's official `Android/android.py` build flow. +* Creates Python installation with a structure suitable for https://github.com/flet-dev/mobile-forge. ## Usage -To build the latest minor version of Python 3.12 for selected Android API: +To build Python for a specific ABI: ``` -./build.sh 3.12 arm64-v8a +./build.sh 3.13.12 arm64-v8a ``` To build all ABIs: ``` -./build-all.sh 3.12 +./build-all.sh 3.13.12 ``` +ABI support: +* Python 3.12: `arm64-v8a`, `armeabi-v7a`, `x86_64`, `x86` +* Python 3.13+: `arm64-v8a`, `x86_64` + ## Credits Build process depends on: @@ -30,4 +34,4 @@ Based on the work from: * https://github.com/chaquo/chaquopy/tree/master/target * https://github.com/beeware/Python-Android-support * https://github.com/beeware/cpython-android-source-deps -* https://github.com/GRRedWings/python3-android \ No newline at end of file +* https://github.com/GRRedWings/python3-android diff --git a/android/abi-to-host.sh b/android/abi-to-host.sh new file mode 100644 index 0000000..6e717aa --- /dev/null +++ b/android/abi-to-host.sh @@ -0,0 +1,18 @@ +case ${abi:?} in + armeabi-v7a) + HOST=arm-linux-androideabi + ;; + arm64-v8a) + HOST=aarch64-linux-android + ;; + x86) + HOST=i686-linux-android + ;; + x86_64) + HOST=x86_64-linux-android + ;; + *) + echo "Unknown ABI: '$abi'" + exit 1 + ;; +esac diff --git a/android/android-env.sh b/android/android-env.sh index 19f60bd..dbfdb2c 100644 --- a/android/android-env.sh +++ b/android/android-env.sh @@ -1,8 +1,29 @@ -fail() { +# This script must be sourced with the following variables already set: +: ${HOST:?} # GNU target triplet + +# You may also override the following: +: ${api_level:=24} # Minimum Android API level the build will run on +: ${PREFIX:-} # Path in which to find required libraries + +NDK_VERSION=r27d + +# Print all messages on stderr so they're visible when running within build-wheel. +log() { echo "$1" >&2 +} + +fail() { + log "$1" exit 1 } +# When moving to a new version of the NDK, carefully review the following: +# +# * https://developer.android.com/ndk/downloads/revision_history +# +# * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md +# where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: +# https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md if [[ -z "${NDK_HOME-}" ]]; then NDK_HOME=$HOME/ndk/$NDK_VERSION echo "NDK_HOME environment variable is not set." @@ -52,10 +73,10 @@ else echo "NDK home: $NDK_HOME" fi -if [ $host_triplet = "arm-linux-androideabi" ]; then +if [ $HOST = "arm-linux-androideabi" ]; then clang_triplet=armv7a-linux-androideabi else - clang_triplet=$host_triplet + clang_triplet=$HOST fi # These variables are based on BuildSystemMaintainers.md above, and @@ -63,7 +84,7 @@ fi toolchain=$(echo $NDK_HOME/toolchains/llvm/prebuilt/*) export AR="$toolchain/bin/llvm-ar" export AS="$toolchain/bin/llvm-as" -export CC="$toolchain/bin/${clang_triplet}$api_level-clang" +export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" export CXX="${CC}++" export LD="$toolchain/bin/ld" export NM="$toolchain/bin/llvm-nm" @@ -78,28 +99,38 @@ for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; fi done -# Use -idirafter so that package-specified -I directories take priority. For example, -# grpcio provides its own BoringSSL headers which must be used rather than our OpenSSL. -export CFLAGS="-idirafter ${prefix:?}/include" -export LDFLAGS="-L${prefix:?}/lib -Wl,--build-id=sha1 -Wl,--no-rosegment" +export CFLAGS="-D__BIONIC_NO_PAGE_SIZE_MACRO" +export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,-z,max-page-size=16384" + +# Unlike Linux, Android does not implicitly use a dlopened library to resolve +# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used +# (https://github.com/android/ndk/issues/1244). So any library that fails to +# build with this flag, would also fail to load at runtime. +LDFLAGS="$LDFLAGS -Wl,--no-undefined" + +# Many packages get away with omitting -lm on Linux, but Android is stricter. +LDFLAGS="$LDFLAGS -lm" -# Many packages get away with omitting this on standard Linux, but Android is stricter. -LDFLAGS+=" -lm" +# -mstackrealign is included where necessary in the clang launcher scripts which are +# pointed to by $CC, so we don't need to include it here. +if [ $HOST = "arm-linux-androideabi" ]; then + CFLAGS="$CFLAGS -march=armv7-a -mthumb" +fi + +if [ -n "${PREFIX:-}" ]; then + abs_prefix=$(realpath $PREFIX) + CFLAGS="$CFLAGS -I$abs_prefix/include" + LDFLAGS="$LDFLAGS -L$abs_prefix/lib" -case $abi in - armeabi-v7a) - CFLAGS+=" -march=armv7-a -mthumb" - ;; - x86) - # -mstackrealign is unnecessary because it's included in the clang launcher script - # which is pointed to by $CC. - ;; -esac + export PKG_CONFIG="pkg-config --define-prefix" + export PKG_CONFIG_LIBDIR="$abs_prefix/lib/pkgconfig" +fi -export PKG_CONFIG="pkg-config --define-prefix" -export PKG_CONFIG_LIBDIR="$prefix/lib/pkgconfig" +# When compiling C++, some build systems will combine CFLAGS and CXXFLAGS, and some will +# use CXXFLAGS alone. +export CXXFLAGS=$CFLAGS -# conda-build variable name +# Use the same variable name as conda-build if [ $(uname) = "Darwin" ]; then export CPU_COUNT=$(sysctl -n hw.ncpu) else diff --git a/android/build-all.sh b/android/build-all.sh index 3297b77..e536dd6 100755 --- a/android/build-all.sh +++ b/android/build-all.sh @@ -1,9 +1,18 @@ -#!/bin/bash -set -eu +#!/usr/bin/env bash +set -euo pipefail python_version=${1:?} -abis="arm64-v8a armeabi-v7a x86_64 x86" +read version_major version_minor < <( + echo "$python_version" | sed -E 's/^([0-9]+)\.([0-9]+).*/\1 \2/' +) +version_int=$((version_major * 100 + version_minor)) + +if [ $version_int -ge 313 ]; then + abis="arm64-v8a x86_64" +else + abis="arm64-v8a armeabi-v7a x86_64 x86" +fi for abi in $abis; do - ./build.sh $python_version $abi -done \ No newline at end of file + bash ./build.sh $python_version $abi +done diff --git a/android/build.sh b/android/build.sh index 3205faa..e61db63 100755 --- a/android/build.sh +++ b/android/build.sh @@ -1,237 +1,186 @@ #!/bin/bash -set -eu +set -eu -o pipefail -python_version=${1:?} +script_dir=$(dirname $(realpath $0)) +version=${1:?} abi=${2:?} -NDK_VERSION=r27 -api_level=24 - -bzip2_version=1.0.8-1 -xz_version=5.4.6-0 -libffi_version=3.4.4-2 -openssl_version=3.0.15-0 -sqlite_version=3.45.2-0 - -os=android -build=custom - -project_dir=$(dirname $(realpath $0)) -downloads=$project_dir/downloads - -# build short Python version -read python_version_major python_version_minor < <(echo $python_version | sed -E 's/^([0-9]+)\.([0-9]+).*/\1 \2/') -if [[ $python_version =~ ^[0-9]+\.[0-9]+$ ]]; then - python_version=$(curl --silent "https://www.python.org/ftp/python/" | sed -nr "s/^.*\"($python_version_major\.$python_version_minor\.[0-9]+)\/\".*$/\1/p" | sort -rV | head -n 1) - echo "Python version: $python_version" -fi -python_version_short=$python_version_major.$python_version_minor -python_version_int=$(($python_version_major * 100 + $python_version_minor)) - -curl_flags="--disable --fail --location --create-dirs --progress-bar" +read version_major version_minor version_micro < <( + echo $version | sed -E 's/^([0-9]+)\.([0-9]+)\.([0-9]+).*/\1 \2 \3/' +) +version_short=$version_major.$version_minor +version_no_pre=$version_major.$version_minor.$version_micro +version_int=$(($version_major * 100 + $version_minor)) + +PREFIX="$script_dir/install/android/$abi/python-${version}" +mkdir -p "$PREFIX" +PREFIX=$(realpath "$PREFIX") + +downloads=$script_dir/downloads mkdir -p $downloads -case $abi in - armeabi-v7a) - host_triplet=arm-linux-androideabi - ;; - arm64-v8a) - host_triplet=aarch64-linux-android - ;; - x86) - host_triplet=i686-linux-android - ;; - x86_64) - host_triplet=x86_64-linux-android - ;; - *) - fail "Unknown ABI: '$abi'" - ;; -esac - -# create VERSIONS support file -support_versions=$project_dir/support/$python_version_short/$os/VERSIONS -mkdir -p $(dirname $support_versions) -echo ">>> Create VERSIONS file for $os" -echo "Python version: $python_version " > $support_versions -echo "Build: $build" >> $support_versions -echo "Min $os version: $api_level" >> $support_versions -echo "---------------------" >> $support_versions -echo "libFFI: $libffi_version" >> $support_versions -echo "BZip2: $bzip2_version" >> $support_versions -echo "OpenSSL: $openssl_version" >> $support_versions -echo "XZ: $xz_version" >> $support_versions - -# BZip2 -# =============== -bzip2_install=$project_dir/install/$os/$abi/bzip2-$bzip2_version -bzip2_lib=$bzip2_install/lib/libbz2.a -bzip2_filename=bzip2-$bzip2_version-$host_triplet.tar.gz - -echo ">>> Download BZip2 for $abi" -curl $curl_flags -o $downloads/$bzip2_filename \ - https://github.com/beeware/cpython-android-source-deps/releases/download/bzip2-$bzip2_version/$bzip2_filename - -echo ">>> Install BZip2 for $abi" -mkdir -p $bzip2_install -tar zxvf $downloads/$bzip2_filename -C $bzip2_install -touch $bzip2_lib - -# XZ (LZMA) -# ================= -xz_install=$project_dir/install/$os/$abi/xz-$xz_version -xz_lib=$xz_install/lib/liblzma.a -xz_filename=xz-$xz_version-$host_triplet.tar.gz - -echo ">>> Download XZ for $abi" -curl $curl_flags -o $downloads/$xz_filename \ - https://github.com/beeware/cpython-android-source-deps/releases/download/xz-$xz_version/$xz_filename - -echo ">>> Install XZ for $abi" -mkdir -p $xz_install -tar zxvf $downloads/$xz_filename -C $xz_install -touch $xz_lib - -# LibFFI -# ================= -libffi_install=$project_dir/install/$os/$abi/libffi-$libffi_version -libffi_lib=$libffi_install/lib/libffi.a -libffi_filename=libffi-$libffi_version-$host_triplet.tar.gz - -echo ">>> Download LibFFI for $abi" -curl $curl_flags -o $downloads/$libffi_filename \ - https://github.com/beeware/cpython-android-source-deps/releases/download/libffi-$libffi_version/$libffi_filename - -echo ">>> Install LibFFI for $abi" -mkdir -p $libffi_install -tar zxvf $downloads/$libffi_filename -C $libffi_install -touch $libffi_lib - -# OpenSSL -# ================= -openssl_install=$project_dir/install/$os/$abi/openssl-$openssl_version -openssl_lib=$openssl_install/lib/libssl.a -openssl_filename=openssl-$openssl_version-$host_triplet.tar.gz - -echo ">>> Download OpenSSL for $abi" -curl $curl_flags -o $downloads/$openssl_filename \ - https://github.com/beeware/cpython-android-source-deps/releases/download/openssl-$openssl_version/$openssl_filename - -echo ">>> Install OpenSSL for $abi" -mkdir -p $openssl_install -tar zxvf $downloads/$openssl_filename -C $openssl_install -touch $openssl_lib - -# SQLite -# ================= -sqlite_install=$project_dir/install/$os/$abi/sqlite-$sqlite_version -sqlite_lib=$sqlite_install/lib/libsqlite3.la -sqlite_filename=sqlite-$sqlite_version-$host_triplet.tar.gz - -echo ">>> Download SQLite for $abi" -curl $curl_flags -o $downloads/$sqlite_filename \ - https://github.com/beeware/cpython-android-source-deps/releases/download/sqlite-$sqlite_version/$sqlite_filename - -echo ">>> Install SQLite for $abi" -mkdir -p $sqlite_install -tar zxvf $downloads/$sqlite_filename -C $sqlite_install -touch $sqlite_lib - -# Python -# =============== - -build_dir=$project_dir/build/$os/$abi -python_build_dir=$project_dir/build/$os/$abi/python-$python_version -python_install=$project_dir/install/$os/$abi/python-$python_version -python_lib=$sqlite_install/lib/libpython$python_version_short.a -python_filename=Python-$python_version.tgz - -echo ">>> Download Python for $abi" -curl $curl_flags -o $downloads/$python_filename \ - https://www.python.org/ftp/python/$python_version/$python_filename - -echo ">>> Unpack Python for $abi" -rm -rf $build_dir -mkdir -p $build_dir -tar zxvf $downloads/$python_filename -C $build_dir -mv $build_dir/Python-$python_version $python_build_dir -touch $python_build_dir/configure +cd $script_dir +. abi-to-host.sh +: ${api_level:=24} -echo ">>> Configuring Python build environment for $abi" +if [ $version_int -le 312 ]; then + . android-env.sh +fi -# configure build environment -prefix=$python_build_dir -. $project_dir/android-env.sh +# Download and unpack Python source code. +version_dir=$script_dir/build/$version +mkdir -p $version_dir +cd $version_dir +src_filename=Python-$version.tgz +wget -c https://www.python.org/ftp/python/$version_no_pre/$src_filename -cd $python_build_dir +build_dir=$version_dir/$abi +rm -rf $build_dir +tar -xf "$src_filename" +mv "Python-$version" "$build_dir" +cd "$build_dir" -# apply patches -echo ">>> Patching Python for $abi" -patches="dynload_shlib lfs soname" -if [ $python_version_int -le 311 ]; then +# Apply patches. +patches="" +if [ $version_int -le 311 ]; then patches+=" sysroot_paths" fi -if [ $python_version_int -ge 311 ]; then +if [ $version_int -eq 311 ]; then patches+=" python_for_build_deps" fi -if [ $python_version_int -ge 312 ]; then +if [ $version_int -le 312 ]; then + patches+=" soname" +fi +if [ $version_int -eq 312 ]; then patches+=" bldlibrary grp" fi for name in $patches; do - patch -p1 -i $project_dir/patches/$name.patch + patch_file="$script_dir/patches/$name.patch" + echo "$patch_file" + patch -p1 -i "$patch_file" done -# Add sysroot paths, otherwise Python 3.8's setup.py will think libz is unavailable. -CFLAGS+=" -I$toolchain/sysroot/usr/include" -LDFLAGS+=" -L$toolchain/sysroot/usr/lib/$host_triplet/$api_level" - -# The configure script omits -fPIC on Android, because it was unnecessary on older versions of -# the NDK (https://bugs.python.org/issue26851). But it's definitely necessary on the current -# version, otherwise we get linker errors like "Parser/myreadline.o: relocation R_386_GOTOFF -# against preemptible symbol PyOS_InputHook cannot be used when making a shared object". -export CCSHARED="-fPIC" - -# Override some tests. -cat > config.site <>> Configuring Python for $abi" -./configure \ - LIBLZMA_CFLAGS="-I$xz_install/include" \ - LIBLZMA_LIBS="-L$xz_install/lib -llzma" \ - BZIP2_CFLAGS="-I$bzip2_install/include" \ - BZIP2_LIBS="-L$bzip2_install/lib -lbz2" \ - LIBFFI_CFLAGS="-I$libffi_install/include" \ - LIBFFI_LIBS="-L$libffi_install/lib -lffi" \ - --host=$host_triplet \ - --build=$(./config.guess) \ - --with-build-python=yes \ - --prefix="$python_install" \ - --enable-ipv6 \ - --with-openssl="$openssl_install" \ - --enable-shared \ - --without-ensurepip \ - 2>&1 | tee -a ../python-$python_version.config.log - -echo ">>> Building Python for $abi" -make all \ - 2>&1 | tee -a ../python-$python_version.build.log - -echo ">>> Installing Python for $abi" -make install \ - 2>&1 | tee -a ../python-$python_version.install.log - -echo ">>> Copying Python dependencies $abi" -cp {$openssl_install,$sqlite_install}/lib/*_python.so $python_install/lib - -echo ">>> Stripping dynamic libraries for $abi" -find $python_install -type f -iname "*.so" -exec $STRIP --strip-unneeded {} \; - -echo ">>> Replacing host platform" -sed -i -e "s/_PYTHON_HOST_PLATFORM=.*/_PYTHON_HOST_PLATFORM=android-$api_level-$abi/" $python_install/lib/python$python_version_short/config-$python_version_short/Makefile - -# the end! \ No newline at end of file +# Remove any existing installation in the prefix. +rm -rf $PREFIX/{include,lib}/python$version_short +rm -rf $PREFIX/lib/libpython$version_short* + +# create VERSIONS support file +support_versions=$script_dir/support/$version_short/android/VERSIONS +mkdir -p $(dirname $support_versions) +echo ">>> Create VERSIONS file for android" +echo "Python version: $version" > $support_versions +echo "Build: custom" >> $support_versions +echo "Min Android version: $api_level" >> $support_versions +echo "---------------------" >> $support_versions + +if [ $version_int -le 312 ]; then + # Download and unpack libraries needed to compile Python. For a given Python + # version, we must maintain binary compatibility with existing wheels. + libs="bzip2-1.0.8-2 libffi-3.4.4-3 sqlite-3.45.3-3 xz-5.4.6-1" + if [ $version_int -le 308 ]; then + libs+=" openssl-1.1.1w-3" + else + libs+=" openssl-3.0.15-4" + fi + + url_prefix="https://github.com/beeware/cpython-android-source-deps/releases/download" + for name_ver in $libs; do + IFS=- read lib_name lib_ver <<< "$name_ver" + url="$url_prefix/$name_ver/$name_ver-$HOST.tar.gz" + echo "$url" + + lib_dir="$script_dir/install/android/$abi/${lib_name}-${lib_ver}" + mkdir -p $lib_dir + lib_file=$downloads/${lib_name}-${lib_ver}-${abi}.tar.gz + curl -Lf "$url" -o $lib_file + tar -xf $lib_file -C $lib_dir + cp -R $lib_dir/* $PREFIX + echo "${lib_name}: $lib_ver" >> $support_versions + done + + # Add sysroot paths, otherwise Python 3.8's setup.py will think libz is unavailable. + CFLAGS+=" -I$toolchain/sysroot/usr/include" + LDFLAGS+=" -L$toolchain/sysroot/usr/lib/$HOST/$api_level" + + # The configure script omits -fPIC on Android, because it was unnecessary on older versions of + # the NDK (https://bugs.python.org/issue26851). But it's definitely necessary on the current + # version, otherwise we get linker errors like "Parser/myreadline.o: relocation R_386_GOTOFF + # against preemptible symbol PyOS_InputHook cannot be used when making a shared object". + export CCSHARED="-fPIC" + + # Override some tests. + cd "$build_dir" + cat > config.site <<-EOF + # Things that can't be autodetected when cross-compiling. + ac_cv_aligned_required=no # Default of "yes" changes hash function to FNV, which breaks Numba. + ac_cv_file__dev_ptmx=no + ac_cv_file__dev_ptc=no + EOF + export CONFIG_SITE=$(pwd)/config.site + + configure_args="--host=$HOST --build=$(./config.guess) \ + --enable-shared --without-ensurepip --with-openssl=$PREFIX" + + # This prevents the "getaddrinfo bug" test, which can't be run when cross-compiling. + configure_args+=" --enable-ipv6" + + # Some of the patches involve missing Makefile dependencies, which allowed extension + # modules to be built before libpython3.x.so in parallel builds. In case this happens + # again, make sure there's no libpython3.x.a, otherwise the modules may end up silently + # linking with that instead. + if [ $version_int -ge 310 ]; then + configure_args+=" --without-static-libpython" + fi + + if [ $version_int -ge 311 ]; then + configure_args+=" --with-build-python=yes" + fi + + ./configure $configure_args + + make -j $CPU_COUNT + make install prefix=$PREFIX + + echo ">>> Replacing host platform" + sed -i -e "s/_PYTHON_HOST_PLATFORM=.*/_PYTHON_HOST_PLATFORM=android-$api_level-$abi/" $PREFIX/lib/python$version_short/config-$version_short/Makefile +else + case "$abi" in + arm64-v8a|x86_64) + ;; + *) + echo "Python $version_short official Android build supports only: arm64-v8a, x86_64" + exit 1 + ;; + esac + + # CPython's Android tooling expects ANDROID_HOME and ANDROID_API_LEVEL. + export ANDROID_API_LEVEL="$api_level" + if [ -z "${ANDROID_HOME:-}" ]; then + if [ -d "$HOME/Library/Android/sdk" ]; then + export ANDROID_HOME="$HOME/Library/Android/sdk" + elif [ -d "$HOME/Android/Sdk" ]; then + export ANDROID_HOME="$HOME/Android/Sdk" + else + export ANDROID_HOME="$script_dir/android-sdk" + mkdir -p "$ANDROID_HOME" + fi + fi + + # Reuse already-installed NDK by exposing it at the location expected by + # CPython's Android/android-env.sh. + if [ -z "${NDK_HOME:-}" ] && [ -d "$HOME/ndk/r27d" ]; then + export NDK_HOME="$HOME/ndk/r27d" + fi + cpython_ndk_version=$(sed -n 's/^ndk_version=//p' Android/android-env.sh | head -n1) + if [ -n "${NDK_HOME:-}" ] && [ -d "$NDK_HOME" ] && [ -n "${cpython_ndk_version:-}" ]; then + mkdir -p "$ANDROID_HOME/ndk" + if [ ! -e "$ANDROID_HOME/ndk/$cpython_ndk_version" ]; then + ln -s "$NDK_HOME" "$ANDROID_HOME/ndk/$cpython_ndk_version" + fi + fi + + Android/android.py configure-build + Android/android.py make-build + Android/android.py configure-host "$HOST" + Android/android.py make-host "$HOST" + cp -a "cross-build/$HOST/prefix/"* "$PREFIX" +fi diff --git a/android/package-for-dart.sh b/android/package-for-dart.sh index 759bf86..70d840e 100755 --- a/android/package-for-dart.sh +++ b/android/package-for-dart.sh @@ -1,5 +1,5 @@ -#!/bin/bash -set -eu +#!/usr/bin/env bash +set -euo pipefail install_root=${1:?} python_version=${2:?} @@ -7,9 +7,61 @@ abi=${3:?} script_dir=$(dirname $(realpath $0)) +. abi-to-host.sh + +if [ -z "${NDK_HOME:-}" ]; then + sdk_candidates="" + if [ -n "${ANDROID_HOME:-}" ]; then + sdk_candidates+="$ANDROID_HOME " + fi + if [ -d "$HOME/Library/Android/sdk" ]; then + sdk_candidates+="$HOME/Library/Android/sdk " + fi + if [ -d "$HOME/Android/Sdk" ]; then + sdk_candidates+="$HOME/Android/Sdk " + fi + if [ -d "$script_dir/android-sdk" ]; then + sdk_candidates+="$script_dir/android-sdk " + fi + if [ -d "$HOME/ndk" ]; then + ndk_candidate=$(ls -d "$HOME/ndk"/* 2>/dev/null | sort -V | tail -n1 || true) + if [ -n "$ndk_candidate" ]; then + NDK_HOME="$ndk_candidate" + fi + fi + + for sdk_root in $sdk_candidates; do + if [ -n "${NDK_HOME:-}" ]; then + break + fi + ndk_candidate=$(ls -d "$sdk_root"/ndk/* 2>/dev/null | sort -V | tail -n1 || true) + if [ -n "$ndk_candidate" ]; then + NDK_HOME="$ndk_candidate" + break + fi + done +fi + +if [ -z "${NDK_HOME:-}" ] || [ ! -d "$NDK_HOME" ]; then + echo "NDK_HOME is not set and no NDK was found in Android SDK locations." + exit 1 +fi + +toolchain=$(echo "$NDK_HOME"/toolchains/llvm/prebuilt/*) +STRIP="$toolchain/bin/llvm-strip" +if [ ! -x "$STRIP" ]; then + echo "llvm-strip not found at: $STRIP" + exit 1 +fi + # build short Python version read python_version_major python_version_minor < <(echo $python_version | sed -E 's/^([0-9]+)\.([0-9]+).*/\1 \2/') python_version_short=$python_version_major.$python_version_minor +python_bin=$(command -v "python$python_version_short" || true) +if [ -z "$python_bin" ]; then + echo "python$python_version_short is required to compile stdlib bytecode" + exit 1 +fi # create build dir build_dir=build/python-$python_version/$abi @@ -23,6 +75,10 @@ mkdir -p dist # copy files to build rsync -av --exclude-from=$script_dir/python-android-dart.exclude $install_root/android/$abi/python-$python_version/* $build_dir +# strip binaries +chmod u+w $(find $build_dir -name *.so) +$STRIP $(find $build_dir -name *.so) + # create libpythonbundle.so bundle_dir=$build_dir/libpythonbundle mkdir -p $bundle_dir @@ -32,8 +88,8 @@ mv $build_dir/lib/python$python_version_short/lib-dynload $bundle_dir/modules # stdlib # stdlib_zip=$bundle_dir/stdlib.zip +"$python_bin" -I -m compileall -b "$build_dir/lib/python$python_version_short" cd $build_dir/lib/python$python_version_short -python -m compileall -b . find . \( -name '*.so' -or -name '*.py' -or -name '*.typed' \) -type f -delete rm -rf __pycache__ rm -rf **/__pycache__ @@ -47,9 +103,15 @@ zip -r ../libpythonbundle.so . cd - rm -rf $bundle_dir -# copy *.so from lib -cp $build_dir/lib/*.so $build_dir +# copy python*.so from lib +cp $build_dir/lib/libpython$python_version_short.so $build_dir + +# copy deps +for name in crypto ssl sqlite3; do + cp "$build_dir/lib/lib${name}_"python.so "$build_dir" +done + rm -rf $build_dir/lib # final archive -tar -czf dist/python-android-dart-$python_version_short-$abi.tar.gz -C $build_dir . \ No newline at end of file +tar -czf dist/python-android-dart-$python_version_short-$abi.tar.gz -C $build_dir . diff --git a/android/patches/bldlibrary.patch b/android/patches/bldlibrary.patch index 4dda79f..72b0bac 100644 --- a/android/patches/bldlibrary.patch +++ b/android/patches/bldlibrary.patch @@ -1,44 +1,55 @@ ---- Python-3.12.0-original/configure 2023-11-22 09:33:49 -+++ Python-3.12.0/configure 2023-11-22 10:13:05 -@@ -7476,6 +7476,7 @@ - case $ac_sys_system in - CYGWIN*) - LDLIBRARY='libpython$(LDVERSION).dll.a' -+ BLDLIBRARY='-L. -lpython$(LDVERSION)' - DLLLIBRARY='libpython$(LDVERSION).dll' - ;; - SunOS*) -@@ -24374,7 +24375,7 @@ - # On Android and Cygwin the shared libraries must be linked with libpython. +diff --git a/configure b/configure +index 1c75810d9e8..d883a00d548 100755 +--- a/configure ++++ b/configure +@@ -841,6 +841,7 @@ PY_ENABLE_SHARED + PLATLIBDIR + BINLIBDEST + LIBPYTHON ++MODULE_DEPS_SHARED + EXT_SUFFIX + ALT_SOABI + SOABI +@@ -24402,12 +24403,17 @@ LDVERSION='$(VERSION)$(ABIFLAGS)' + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDVERSION" >&5 + printf "%s\n" "$LDVERSION" >&6; } +-# On Android and Cygwin the shared libraries must be linked with libpython. ++# Configure the flags and dependencies used when compiling shared modules. ++# Do not rename LIBPYTHON - it's accessed via sysconfig by package build ++# systems (e.g. Meson) to decide whether to link extension modules against ++# libpython. ++MODULE_DEPS_SHARED='$(MODULE_DEPS_STATIC) $(EXPORTSYMS)' ++LIBPYTHON='' + ++# On Android and Cygwin the shared libraries must be linked with libpython. if test "$PY_ENABLE_SHARED" = "1" && ( test -n "$ANDROID_API_LEVEL" || test "$MACHDEP" = "cygwin"); then - LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" -+ LIBPYTHON="$BLDLIBRARY" - else - LIBPYTHON='' +-else +- LIBPYTHON='' ++ MODULE_DEPS_SHARED="$MODULE_DEPS_SHARED \$(LDLIBRARY)" ++ LIBPYTHON="\$(BLDLIBRARY)" fi ---- Python-3.12.0-original/Modules/makesetup 2023-10-02 12:48:14 -+++ Python-3.12.0/Modules/makesetup 2023-11-22 10:11:40 -@@ -86,18 +86,6 @@ - # Newline for sed i and a commands - NL='\ - ' -- --# Setup to link with extra libraries when making shared extensions. --# Currently, only Cygwin needs this baggage. --case `uname -s` in --CYGWIN*) if test $libdir = . -- then -- ExtraLibDir=. -- else -- ExtraLibDir='$(LIBPL)' -- fi -- ExtraLibs="-L$ExtraLibDir -lpython\$(LDVERSION)";; --esac - # Main loop - for i in ${*-Setup} -@@ -286,7 +274,7 @@ + +diff --git a/Makefile.pre.in b/Makefile.pre.in +index 0e64ccc5c21..c4217424508 100644 +--- a/Makefile.pre.in ++++ b/Makefile.pre.in +@@ -2797,7 +2797,7 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h + + # force rebuild when header file or module build flavor (static/shared) is changed + MODULE_DEPS_STATIC=Modules/config.c +-MODULE_DEPS_SHARED=$(MODULE_DEPS_STATIC) $(EXPORTSYMS) ++MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@ + + MODULE_CMATH_DEPS=$(srcdir)/Modules/_math.h + MODULE_MATH_DEPS=$(srcdir)/Modules/_math.h +diff --git a/Modules/makesetup b/Modules/makesetup +index f000c9cd673..3231044230e 100755 +--- a/Modules/makesetup ++++ b/Modules/makesetup +@@ -286,7 +286,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | ;; esac rule="$file: $objs" diff --git a/android/patches/dynload_shlib.patch b/android/patches/dynload_shlib.patch deleted file mode 100644 index 4260a05..0000000 --- a/android/patches/dynload_shlib.patch +++ /dev/null @@ -1,12 +0,0 @@ ---- a/Python/dynload_shlib.c -+++ b/Python/dynload_shlib.c -@@ -66,7 +66,8 @@ _PyImport_FindSharedFuncptr(const char *prefix, - char pathbuf[260]; - int dlopenflags=0; - -- if (strchr(pathname, '/') == NULL) { -+ // Chaquopy disabled: this interferes with our workaround in importer.prepare_dlopen. -+ if (0 && strchr(pathname, '/') == NULL) { - /* Prefix bare filename with "./" */ - PyOS_snprintf(pathbuf, sizeof(pathbuf), "./%-.255s", pathname); - pathname = pathbuf; diff --git a/android/patches/lfs.patch b/android/patches/lfs.patch deleted file mode 100644 index 4f32859..0000000 --- a/android/patches/lfs.patch +++ /dev/null @@ -1,16 +0,0 @@ ---- a/configure -+++ b/configure -@@ -8373,7 +8373,12 @@ $as_echo "#define HAVE_HTOLE64 1" >>confdefs.h - - fi - --use_lfs=yes -+# Chaquopy: changed "yes" to "no". _LARGEFILE_SOURCE has no effect on Android, and -+# _FILE_OFFSET_BITS=64 has no effect on 64-bit ABIs, but on 32-bit ABIs it causes many critical -+# functions to disappear on API levels older than 24. See -+# https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md. -+use_lfs=no -+ - # Don't use largefile support for GNU/Hurd - case $ac_sys_system in GNU*) - use_lfs=no diff --git a/darwin/Modules/module.modulemap b/darwin/Modules/module.modulemap index cf58b60..f22f264 100644 --- a/darwin/Modules/module.modulemap +++ b/darwin/Modules/module.modulemap @@ -107,6 +107,7 @@ framework module Python { exclude header "pyconfig-x86_64.h" exclude header "pydtrace.h" exclude header "pyexpat.h" + exclude header "pyport.h" exclude header "structmember.h" exclude header "token.h" } \ No newline at end of file diff --git a/darwin/PrivacyInfo.xcprivacy b/darwin/PrivacyInfo.xcprivacy new file mode 100644 index 0000000..6d94afc --- /dev/null +++ b/darwin/PrivacyInfo.xcprivacy @@ -0,0 +1,17 @@ + + + + + NSPrivacyTracking + + NSPrivacyCollectedDataTypes + + NSPrivacyAccessedAPITypes + + NSPrivacyTrackingDomains + + NSPrivacyUsesNonStandardAPIs + + + \ No newline at end of file diff --git a/darwin/package-ios-for-dart.sh b/darwin/package-ios-for-dart.sh index 111a615..1921851 100755 --- a/darwin/package-ios-for-dart.sh +++ b/darwin/package-ios-for-dart.sh @@ -1,5 +1,5 @@ -#!/bin/bash -set -eu +#!/usr/bin/env bash +set -euo pipefail python_apple_support_root=${1:?} python_version=${2:?} @@ -11,6 +11,11 @@ script_dir=$(dirname $(realpath $0)) # build short Python version read python_version_major python_version_minor < <(echo $python_version | sed -E 's/^([0-9]+)\.([0-9]+).*/\1 \2/') python_version_short=$python_version_major.$python_version_minor +python_bin=$(command -v "python$python_version_short" || true) +if [ -z "$python_bin" ]; then + echo "python$python_version_short is required to compile stdlib bytecode" + exit 1 +fi # create build directory build_dir=build/python-$python_version @@ -38,6 +43,10 @@ for arch in "${archs[@]}"; do rsync -av --exclude-from=$script_dir/python-darwin-stdlib.exclude $python_apple_support_root/install/iOS/$arch/python-*/lib/python$python_version_short/* $stdlib_dir/$arch done +echo "Copying privacy manifests..." +cp "$script_dir/PrivacyInfo.xcprivacy" "$stdlib_dir/${archs[0]}/lib-dynload/_hashlib.xcprivacy" +cp "$script_dir/PrivacyInfo.xcprivacy" "$stdlib_dir/${archs[0]}/lib-dynload/_ssl.xcprivacy" + echo "Converting lib-dynload to xcframeworks..." find "$stdlib_dir/${archs[0]}/lib-dynload" -name "*.$dylib_ext" | while read full_dylib; do dylib_relative_path=${full_dylib#$stdlib_dir/${archs[0]}/lib-dynload/} @@ -46,7 +55,7 @@ find "$stdlib_dir/${archs[0]}/lib-dynload" -name "*.$dylib_ext" | while read ful "$stdlib_dir/${archs[1]}/lib-dynload" \ "$stdlib_dir/${archs[2]}/lib-dynload" \ $dylib_relative_path \ - "Frameworks/serious_python_darwin.framework/python-stdlib/lib-dynload" \ + "Frameworks/serious_python_darwin.framework/python.bundle/stdlib/lib-dynload" \ $python_frameworks_dir #break # run for one lib only - for tests done @@ -59,13 +68,13 @@ for arch in "${archs[@]}"; do rm -rf $stdlib_dir/$arch done -# compile stdlib +# compile stdlib with an isolated interpreter, without importing from target stdlib dir. +"$python_bin" -I -m compileall -b "$stdlib_dir" cd $stdlib_dir -python -m compileall -b . find . \( -name '*.so' -or -name "*.$dylib_ext" -or -name '*.py' -or -name '*.typed' \) -type f -delete rm -rf __pycache__ rm -rf **/__pycache__ cd - # final archive -tar -czf dist/python-ios-dart-$python_version_short.tar.gz -C $build_dir . \ No newline at end of file +tar -czf dist/python-ios-dart-$python_version_short.tar.gz -C $build_dir . diff --git a/darwin/package-macos-for-dart.sh b/darwin/package-macos-for-dart.sh index 81803fb..913c872 100755 --- a/darwin/package-macos-for-dart.sh +++ b/darwin/package-macos-for-dart.sh @@ -1,5 +1,5 @@ -#!/bin/bash -set -eu +#!/usr/bin/env bash +set -euo pipefail python_apple_support_root=${1:?} python_version=${2:?} @@ -9,6 +9,11 @@ script_dir=$(dirname $(realpath $0)) # build short Python version read python_version_major python_version_minor < <(echo $python_version | sed -E 's/^([0-9]+)\.([0-9]+).*/\1 \2/') python_version_short=$python_version_major.$python_version_minor +python_bin=$(command -v "python$python_version_short" || true) +if [ -z "$python_bin" ]; then + echo "python$python_version_short is required to compile stdlib bytecode" + exit 1 +fi # create build directory build_dir=build/python-$python_version @@ -29,17 +34,18 @@ rsync -av --exclude-from=$script_dir/python-darwin-framework.exclude $python_app cp -r $script_dir/Modules $frameworks_dir/Python.xcframework/macos-arm64_x86_64/Python.framework mkdir -p $frameworks_dir/Python.xcframework/macos-arm64_x86_64/Python.framework/Headers cp -r $python_apple_support_root/support/$python_version_short/macOS/Python.xcframework/macos-arm64_x86_64/Python.framework/Versions/$python_version_short/include/python$python_version_short/* $frameworks_dir/Python.xcframework/macos-arm64_x86_64/Python.framework/Headers +rm $frameworks_dir/Python.xcframework/macos-arm64_x86_64/Python.framework/Headers/module.modulemap # copy stdlibs rsync -av --exclude-from=$script_dir/python-darwin-stdlib.exclude $python_apple_support_root/install/macOS/macosx/python-*/Python.framework/Versions/Current/lib/python$python_version_short/* $stdlib_dir -# compile stdlib +# compile stdlib with an isolated interpreter, without importing from target stdlib dir. +"$python_bin" -I -m compileall -b "$stdlib_dir" cd $stdlib_dir -python -m compileall -b . find . \( -name '*.py' -or -name '*.typed' \) -type f -delete rm -rf __pycache__ rm -rf **/__pycache__ cd - # final archive -tar -czf dist/python-macos-dart-$python_version.tar.gz -C $build_dir . \ No newline at end of file +tar -czf dist/python-macos-dart-$python_version.tar.gz -C $build_dir . diff --git a/darwin/python-darwin-framework.exclude b/darwin/python-darwin-framework.exclude index 5e5a095..afbea50 100644 --- a/darwin/python-darwin-framework.exclude +++ b/darwin/python-darwin-framework.exclude @@ -1,13 +1,29 @@ ios-arm64/bin ios-arm64/include ios-arm64/lib +ios-arm64/platform-config +ios-arm64/Python.framework/Headers/module.modulemap ios-arm64_x86_64-simulator/bin ios-arm64_x86_64-simulator/include ios-arm64_x86_64-simulator/lib +ios-arm64_x86_64-simulator/platform-config +ios-arm64_x86_64-simulator/Python.framework/Headers/module.modulemap macos-arm64_x86_64/Python.framework/Headers macos-arm64_x86_64/Python.framework/Versions/*/_CodeSignature macos-arm64_x86_64/Python.framework/Versions/*/Headers macos-arm64_x86_64/Python.framework/Versions/*/include -macos-arm64_x86_64/Python.framework/Versions/*/lib +macos-arm64_x86_64/Python.framework/Versions/*/lib/itcl* +macos-arm64_x86_64/Python.framework/Versions/*/lib/libform* +macos-arm64_x86_64/Python.framework/Versions/*/lib/libmenu* +macos-arm64_x86_64/Python.framework/Versions/*/lib/libncurses* +macos-arm64_x86_64/Python.framework/Versions/*/lib/libpanel* +macos-arm64_x86_64/Python.framework/Versions/*/lib/libpython* +macos-arm64_x86_64/Python.framework/Versions/*/lib/libtcl* +macos-arm64_x86_64/Python.framework/Versions/*/lib/libtk* +macos-arm64_x86_64/Python.framework/Versions/*/lib/pkgconfig +macos-arm64_x86_64/Python.framework/Versions/*/lib/python* +macos-arm64_x86_64/Python.framework/Versions/*/lib/sqlite* +macos-arm64_x86_64/Python.framework/Versions/*/lib/t* +macos-arm64_x86_64/Python.framework/Versions/*/lib/T* iphoneos iphonesimulator \ No newline at end of file diff --git a/darwin/python-darwin-stdlib.exclude b/darwin/python-darwin-stdlib.exclude index 5df694e..b06a320 100644 --- a/darwin/python-darwin-stdlib.exclude +++ b/darwin/python-darwin-stdlib.exclude @@ -3,6 +3,7 @@ lib-dynload/_ctypes_test*.so lib-dynload/xxlimited*.so lib-dynload/_xxtestfuzz*.so config-* +ctypes/macholib/fetch_macholib* curses ensurepip idlelib diff --git a/darwin/xcframework_utils.sh b/darwin/xcframework_utils.sh index 66b6d94..413ffeb 100644 --- a/darwin/xcframework_utils.sh +++ b/darwin/xcframework_utils.sh @@ -23,7 +23,7 @@ create_plist() { CFBundleInfoDictionaryVersion 6.0 CFBundlePackageType - APPL + FMWK CFBundleShortVersionString 1.0 CFBundleSupportedPlatforms @@ -31,7 +31,7 @@ create_plist() { iPhoneOS MinimumOSVersion - 12.0 + 13.0 CFBundleVersion 1 @@ -55,6 +55,10 @@ create_xcframework_from_dylibs() { dylib_without_ext=$(echo $dylib_relative_path | cut -d "." -f 1) framework=$(echo $dylib_without_ext | tr "/" ".") framework_identifier=${framework//_/-} + while [[ $framework_identifier == -* ]]; do + framework_identifier=${framework_identifier#-} + done + framework_identifier=${framework_identifier:-framework} # creating "iphoneos" framework fd=iphoneos/$framework.framework @@ -65,6 +69,11 @@ create_xcframework_from_dylibs() { create_plist $framework "org.python.$framework_identifier" $fd/Info.plist echo "$origin_prefix/$dylib_without_ext.fwork" > $fd/$framework.origin + # copy privacy manifest if any + if [ -f "$iphone_dir/$dylib_without_ext.xcprivacy" ]; then + cp "$iphone_dir/$dylib_without_ext.xcprivacy" "$fd/PrivacyInfo.xcprivacy" + fi + # creating "iphonesimulator" framework fd=iphonesimulator/$framework.framework mkdir -p $fd @@ -78,6 +87,11 @@ create_xcframework_from_dylibs() { create_plist $framework "org.python.$framework_identifier" $fd/Info.plist echo "$origin_prefix/$dylib_without_ext.fwork" > $fd/$framework.origin + # copy privacy manifest if any + if [ -f "$iphone_dir/$dylib_without_ext.xcprivacy" ]; then + mv "$iphone_dir/$dylib_without_ext.xcprivacy" "$fd/PrivacyInfo.xcprivacy" + fi + # merge frameworks info xcframework xcodebuild -create-xcframework \ -framework "iphoneos/$framework.framework" \ @@ -87,4 +101,4 @@ create_xcframework_from_dylibs() { # cleanup popd >/dev/null rm -rf "${tmp_dir}" >/dev/null -} \ No newline at end of file +} diff --git a/linux/package-for-linux.sh b/linux/package-for-linux.sh index 5c29211..00c5b5c 100755 --- a/linux/package-for-linux.sh +++ b/linux/package-for-linux.sh @@ -1,17 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + PYTHON_ARCH=${1:?} PYTHON_ARCH_VER=${2:-""} +if [ -z "${PYTHON_VERSION:-}" ]; then + echo "PYTHON_VERSION is required (e.g. 3.13.12)" + exit 1 +fi +if [ -z "${PYTHON_DIST_RELEASE:-}" ]; then + echo "PYTHON_DIST_RELEASE is required" + exit 1 +fi + +PYTHON_VERSION_SHORT=${PYTHON_VERSION_SHORT:-$(echo "$PYTHON_VERSION" | cut -d. -f1,2)} + DIST_FILE=cpython-${PYTHON_VERSION}+${PYTHON_DIST_RELEASE}-${PYTHON_ARCH}${PYTHON_ARCH_VER}-unknown-linux-gnu-install_only_stripped.tar.gz -curl -OL https://github.com/indygreg/python-build-standalone/releases/download/${PYTHON_DIST_RELEASE}/${DIST_FILE} +curl -OL https://github.com/astral-sh/python-build-standalone/releases/download/${PYTHON_DIST_RELEASE}/${DIST_FILE} mkdir -p $PYTHON_ARCH/build tar zxvf $DIST_FILE -C $PYTHON_ARCH/build # compile lib -python -m compileall -b $PYTHON_ARCH/build/python/lib/python3.12 +build_python=$(command -v "python$PYTHON_VERSION_SHORT" || true) +if [ -z "$build_python" ]; then + build_python=$(command -v python3 || true) +fi +if [ -z "$build_python" ]; then + build_python=$(command -v python || true) +fi +if [ -z "$build_python" ]; then + echo "Host Python interpreter not found for compileall" + exit 1 +fi +"$build_python" -I -m compileall -b "$PYTHON_ARCH/build/python/lib/python$PYTHON_VERSION_SHORT" # copy build to dist mkdir -p $PYTHON_ARCH/dist rsync -av --exclude-from=python-linux-dart.exclude $PYTHON_ARCH/build/python/* $PYTHON_ARCH/dist # archive -tar -czf python-linux-dart-$PYTHON_VERSION_SHORT-$PYTHON_ARCH.tar.gz -C $PYTHON_ARCH/dist . \ No newline at end of file +tar -czf "python-linux-dart-$PYTHON_VERSION_SHORT-$PYTHON_ARCH.tar.gz" -C "$PYTHON_ARCH/dist" . diff --git a/windows/exclude.txt b/windows/exclude.txt deleted file mode 100644 index 98cb92f..0000000 --- a/windows/exclude.txt +++ /dev/null @@ -1,3 +0,0 @@ -__pycache__ -*.py -*.exe \ No newline at end of file diff --git a/windows/package-for-dart.ps1 b/windows/package-for-dart.ps1 new file mode 100644 index 0000000..fd81a30 --- /dev/null +++ b/windows/package-for-dart.ps1 @@ -0,0 +1,166 @@ +param( + [Parameter(Mandatory = $true)] + [string]$PythonVersion, + + [Parameter(Mandatory = $true)] + [string]$PythonVersionShort +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$workspace = $env:GITHUB_WORKSPACE +if (-not $workspace) { + $workspace = Split-Path -Parent $PSScriptRoot +} + +$srcRoot = Join-Path $workspace "windows\build" +$srcArchive = Join-Path $srcRoot "Python-$PythonVersion.tgz" +$srcDir = Join-Path $srcRoot "Python-$PythonVersion" +$pcbuildDir = Join-Path $srcDir "PCbuild\amd64" +$pythonTag = $PythonVersionShort -replace '\.', '' + +$packageRoot = Join-Path $workspace "windows\python-windows-for-dart-$PythonVersionShort" +$zipPath = Join-Path $workspace "windows\python-windows-for-dart-$PythonVersionShort.zip" +$excludeListPath = Join-Path $workspace "windows\python-windows-dart.exclude" +$keepImportLibs = @("python3.lib", "python3_d.lib", "python$pythonTag.lib", "python${pythonTag}_d.lib") + +New-Item -ItemType Directory -Force -Path $srcRoot | Out-Null + +Write-Host "Downloading CPython source $PythonVersion" +Invoke-WebRequest -Uri "https://www.python.org/ftp/python/$PythonVersion/Python-$PythonVersion.tgz" -OutFile $srcArchive +tar -xf $srcArchive -C $srcRoot + +# Force PCbuild helper scripts to use configured Python, not py.exe launcher. +$pythonFromPath = (Get-Command python).Source +if (-not $pythonFromPath) { + throw "python was not found in PATH" +} +$env:PYTHON = $pythonFromPath +$env:PYTHON_FOR_BUILD = $pythonFromPath + +Push-Location $srcDir +cmd /c "PCbuild\build.bat -e -p x64 -c Release" +cmd /c "PCbuild\build.bat -e -p x64 -c Debug" +Pop-Location + +Remove-Item -Recurse -Force $packageRoot -ErrorAction SilentlyContinue +New-Item -ItemType Directory -Force -Path "$packageRoot\DLLs", "$packageRoot\include", "$packageRoot\Lib", "$packageRoot\libs", "$packageRoot\Scripts" | Out-Null + +Copy-Item -Path "$srcDir\Include\*" -Destination "$packageRoot\include" -Recurse -Force +Copy-Item -Path "$srcDir\Lib\*" -Destination "$packageRoot\Lib" -Recurse -Force + +# pyconfig.h location varies by CPython version/build layout. +$pyconfigCandidates = @( + (Join-Path $srcDir "PC\pyconfig.h"), + (Join-Path $srcDir "Include\pyconfig.h"), + (Join-Path $pcbuildDir "pyconfig.h") +) +$pyconfigHeader = $null +foreach ($candidate in $pyconfigCandidates) { + if (Test-Path $candidate) { + $pyconfigHeader = $candidate + break + } +} +if (-not $pyconfigHeader) { + $candidateList = $pyconfigCandidates -join ", " + throw "Missing required header. Checked: $candidateList" +} +Copy-Item -Path $pyconfigHeader -Destination "$packageRoot\include\pyconfig.h" -Force + +# Root binaries and symbols. +foreach ($name in @("LICENSE.txt", "NEWS.txt")) { + $src = Join-Path $srcDir $name + if (Test-Path $src) { + Copy-Item -Path $src -Destination $packageRoot -Force + } +} + +$rootFiles = @( + "python3.dll", + "python3_d.dll", + "python$pythonTag.dll", + "python${pythonTag}_d.dll", + "python${pythonTag}_d.pdb", + "python_d.pdb", + "pythonw_d.pdb" +) +foreach ($name in $rootFiles) { + $src = Join-Path $pcbuildDir $name + if (Test-Path $src) { + Copy-Item -Path $src -Destination $packageRoot -Force + } +} + +foreach ($name in @("vcruntime140.dll", "vcruntime140_1.dll")) { + $fromBuild = Join-Path $pcbuildDir $name + $fromSystem = Join-Path "$env:WINDIR\System32" $name + if (Test-Path $fromBuild) { + Copy-Item -Path $fromBuild -Destination $packageRoot -Force + } elseif (Test-Path $fromSystem) { + Copy-Item -Path $fromSystem -Destination $packageRoot -Force + } +} + +# Extension modules and supporting DLLs. +Get-ChildItem -Path $pcbuildDir -Filter "*.pyd" -File | Copy-Item -Destination "$packageRoot\DLLs" -Force +Get-ChildItem -Path $pcbuildDir -Filter "*.dll" -File | + Where-Object { $_.Name -notin @("python3.dll", "python3_d.dll", "python$pythonTag.dll", "python${pythonTag}_d.dll", "vcruntime140.dll", "vcruntime140_1.dll") } | + Copy-Item -Destination "$packageRoot\DLLs" -Force +foreach ($name in $keepImportLibs) { + $src = Join-Path $pcbuildDir $name + if (Test-Path $src) { + Copy-Item -Path $src -Destination "$packageRoot\libs" -Force + } +} + +# Cleanup using exclude list. +if (-not (Test-Path $excludeListPath)) { + throw "Exclude list not found: $excludeListPath" +} +$excludePatterns = Get-Content $excludeListPath | + ForEach-Object { $_.Trim() } | + Where-Object { $_ -and -not $_.StartsWith("#") } +foreach ($pattern in $excludePatterns) { + $fullPattern = Join-Path $packageRoot $pattern + $matches = Get-ChildItem -Path $fullPattern -Force -ErrorAction SilentlyContinue + foreach ($item in $matches) { + Remove-Item -Path $item.FullName -Recurse -Force + } +} + +# Match existing packaging behavior: bytecode-only stdlib. +& $pythonFromPath -I -m compileall -b "$packageRoot\Lib" +Get-ChildItem -Path "$packageRoot\Lib" -Recurse -File -Include *.py,*.typed | Remove-Item -Force +Get-ChildItem -Path "$packageRoot\Lib" -Recurse -Directory -Filter __pycache__ | Remove-Item -Recurse -Force + +# Remove empty directories left after exclusions/cleanup. +Get-ChildItem -Path $packageRoot -Recurse -Directory | + Sort-Object { $_.FullName.Length } -Descending | + Where-Object { (Get-ChildItem -Path $_.FullName -Force | Measure-Object).Count -eq 0 } | + Remove-Item -Force + +# Fail fast if required layout entries are missing. +$requiredEntries = @( + "$packageRoot\DLLs", + "$packageRoot\include", + "$packageRoot\Lib", + "$packageRoot\libs", + "$packageRoot\python3.dll", + "$packageRoot\python3_d.dll", + "$packageRoot\python$pythonTag.dll", + "$packageRoot\python${pythonTag}_d.dll", + "$packageRoot\python${pythonTag}_d.pdb", + "$packageRoot\python_d.pdb", + "$packageRoot\pythonw_d.pdb" +) +foreach ($entry in $requiredEntries) { + if (-not (Test-Path $entry)) { + Get-ChildItem $packageRoot + throw "Missing required package entry: $entry" + } +} + +Remove-Item -Force $zipPath -ErrorAction SilentlyContinue +7z a $zipPath "$packageRoot\*" diff --git a/windows/python-windows-dart.exclude b/windows/python-windows-dart.exclude new file mode 100644 index 0000000..886d977 --- /dev/null +++ b/windows/python-windows-dart.exclude @@ -0,0 +1,25 @@ +# Files and directories to exclude from Windows Dart package. +# Patterns are relative to package root: python-windows-for-dart-/ + +# Stdlib content not needed for embedded runtime package. +Lib/test +Lib/idlelib +Lib/tkinter +Lib/turtledemo +Lib/ensurepip +Lib/pydoc_data +Lib/lib2to3 +Lib/site-packages/pip* +Lib/site-packages/setuptools* +Lib/site-packages/wheel* + +# Optional/test/GUI extension modules and DLLs. +DLLs/_test*.pyd +DLLs/_ctypes_test*.pyd +DLLs/_tkinter*.pyd +DLLs/xxlimited*.pyd +DLLs/tcl86t.dll +DLLs/tk86t.dll +DLLs/zlib1.dll +DLLs/pyshellext.dll +DLLs/pyshellext_d.dll diff --git a/windows/unattend.xml b/windows/unattend.xml deleted file mode 100644 index 286ae72..0000000 --- a/windows/unattend.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - \ No newline at end of file