Add buildpack support for Heroku-24 (#1575)

Following on from #1574 (which added support for building
and releasing Python binaries for Heroku-24), this adds support
for the new stack to the buildpack itself.

Heroku-24's base images are published as both `amd64` and `arm64`
variants (primarily for CNB multi-arch support), so the buildpack now
checks the architecture where necessary to ensure it continues to
operate should it be run against the `arm64` image outside of Heroku.
(However, when run on Heroku, this buildpack is still only ever run
on `amd64`.)

GUS-W-14667590.

.github/workflows/build_python_runtime.yml

@@ -70,8 +70,8 @@ jobs:
         run: aws s3 sync ./upload "s3://${S3_BUCKET}"
 
   heroku-24:
-    # On Heroku-24 we only support Python 3.12+.
-    if: inputs.stack == 'heroku-24' || (inputs.stack == 'auto' && startsWith(inputs.python_version,'3.12.'))
+    # On Heroku-24 we only support Python 3.11+.
+    if: inputs.stack == 'heroku-24' || (inputs.stack == 'auto' && !startsWith(inputs.python_version,'3.8.') && !startsWith(inputs.python_version,'3.9.') && !startsWith(inputs.python_version,'3.10.'))
     strategy:
       fail-fast: false
       matrix:

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        stack: ["heroku-20", "heroku-22"]
+        stack: ["heroku-20", "heroku-22", "heroku-24"]
     env:
       HATCHET_APP_LIMIT: 200
       HATCHET_DEFAULT_STACK: ${{ matrix.stack }}

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Added support for Heroku-24. ([#1575](https://github.com/heroku/heroku-buildpack-python/pull/1575))
 
 ## [v249] - 2024-04-18
 

Makefile

@@ -1,8 +1,7 @@
 # These targets are not files
 .PHONY: lint lint-scripts lint-ruby compile publish
 
-STACK ?= heroku-22
-PLATFORM := linux/amd64
+STACK ?= heroku-24
 FIXTURE ?= spec/fixtures/python_version_unspecified
 
 # Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
@@ -26,7 +25,7 @@ lint-ruby:
 compile:
 	@echo "Running compile using: STACK=$(STACK) FIXTURE=$(FIXTURE)"
 	@echo
-	@docker run --rm -it -v $(PWD):/src:ro -e "STACK=$(STACK)" -w /buildpack --platform="$(PLATFORM)" "$(STACK_IMAGE_TAG)" \
+	@docker run --rm -it --user root -v $(PWD):/src:ro -e "STACK=$(STACK)" -w /buildpack "$(STACK_IMAGE_TAG)" \
 		bash -c 'cp -r /src/{bin,requirements,vendor} /buildpack && cp -r /src/$(FIXTURE) /build && mkdir /cache /env && bin/compile /build /cache /env'
 	@echo
 

README.md

@@ -62,6 +62,6 @@ Supported runtime options include:
 
 - `python-3.12.3` on all [supported stacks](https://devcenter.heroku.com/articles/stack#stack-support-details)
 - `python-3.11.9` on all [supported stacks](https://devcenter.heroku.com/articles/stack#stack-support-details)
-- `python-3.10.14` on all [supported stacks](https://devcenter.heroku.com/articles/stack#stack-support-details)
-- `python-3.9.19` on all [supported stacks](https://devcenter.heroku.com/articles/stack#stack-support-details)
+- `python-3.10.14` on Heroku-20 and Heroku-22 only
+- `python-3.9.19` on Heroku-20 and Heroku-22 only
 - `python-3.8.19` on Heroku-20 only

bin/steps/python

@@ -37,9 +37,9 @@ esac
 
 # The Python runtime archive filename is of form: 'python-X.Y.Z-ubuntu-22.04-amd64.tar.zst'
 # The Ubuntu version is calculated from `STACK` since it's faster than calling `lsb_release`.
-# TODO: Switch to dynamically calculating the architecture when adding support for Heroku-24.
 UBUNTU_VERSION="${STACK/heroku-}.04"
-PYTHON_URL="${S3_BASE_URL}/${PYTHON_VERSION}-ubuntu-${UBUNTU_VERSION}-amd64.tar.zst"
+ARCH=$(dpkg --print-architecture)
+PYTHON_URL="${S3_BASE_URL}/${PYTHON_VERSION}-ubuntu-${UBUNTU_VERSION}-${ARCH}.tar.zst"
 
 if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}"; then
   puts-warn

bin/steps/sqlite3

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# TODO: Remove this entirely since the Python stdlib now includes modern sqlite support,
+# and the APT buildpack should be used if an app needs the sqlite CLI/headers.
+
 # shellcheck source=bin/utils
 source "$BIN_DIR/utils"
 
@@ -38,14 +41,17 @@ sqlite3_install() {
   rm -f "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
   rm -f "$HEROKU_PYTHON_DIR/bin/sqlite3"
 
+  # eg: `x86_64` or `aarch64`
+  GNU_ARCH=$(arch)
+
   # copy over sqlite3 headers & bins and setup linking against the stack image library
   mv "$HEROKU_PYTHON_DIR/sqlite3/usr/include/"* "$HEROKU_PYTHON_DIR/include/"
-  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu"/libsqlite3.*a "$HEROKU_PYTHON_DIR/lib/"
+  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/${GNU_ARCH}-linux-gnu"/libsqlite3.*a "$HEROKU_PYTHON_DIR/lib/"
   mkdir -p "$HEROKU_PYTHON_DIR/lib/pkgconfig"
   # set the right prefix/lib directories
-  sed -e 's/prefix=\/usr/prefix=\/app\/.heroku\/python/' -e 's/\/x86_64-linux-gnu//' "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/pkgconfig/sqlite3.pc" > "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
+  sed -e 's/prefix=\/usr/prefix=\/app\/.heroku\/python/' -e "s/\/${GNU_ARCH}-linux-gnu//" "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/${GNU_ARCH}-linux-gnu/pkgconfig/sqlite3.pc" > "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
   # need to point the libsqlite3.so to the stack image library for /usr/bin/ld -lsqlite3
-  SQLITE3_LIBFILE="/usr/lib/x86_64-linux-gnu/$(readlink -n "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/libsqlite3.so")"
+  SQLITE3_LIBFILE="/usr/lib/${GNU_ARCH}-linux-gnu/$(readlink -n "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/${GNU_ARCH}-linux-gnu/libsqlite3.so")"
   ln -s "$SQLITE3_LIBFILE" "$HEROKU_PYTHON_DIR/lib/libsqlite3.so"
   if [ -z "$HEADERS_ONLY" ]; then
     mv "$HEROKU_PYTHON_DIR/sqlite3/usr/bin"/* "$HEROKU_PYTHON_DIR/bin/"

builds/build_python_runtime.sh

@@ -22,6 +22,7 @@ function error() {
 case "${STACK}" in
   heroku-24)
     SUPPORTED_PYTHON_VERSIONS=(
+      "3.11"
       "3.12"
     )
     ;;

spec/hatchet/pipenv_spec.rb

@@ -159,7 +159,7 @@
       end
     end
 
-    context 'when using Heroku-22', stacks: %w[heroku-22] do
+    context 'when using Heroku-22 or newer', stacks: %w[heroku-22 heroku-24] do
       let(:allow_failure) { true }
 
       # We only support Python 3.8 on Heroku-20 and older.
@@ -168,15 +168,35 @@
   end
 
   context 'with a Pipfile.lock containing python_version 3.9' do
-    let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_python_3.9') }
+    let(:allow_failure) { false }
+    let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_python_3.9', allow_failure:) }
+
+    context 'when using Heroku-22 or older', stacks: %w[heroku-20 heroku-22] do
+      include_examples 'builds using Pipenv with the requested Python version', LATEST_PYTHON_3_9
+    end
 
-    include_examples 'builds using Pipenv with the requested Python version', LATEST_PYTHON_3_9
+    context 'when using Heroku-24', stacks: %w[heroku-24] do
+      let(:allow_failure) { true }
+
+      # We only support Python 3.9 on Heroku-22 and older.
+      include_examples 'aborts the build with a runtime not available message (Pipenv)', LATEST_PYTHON_3_9
+    end
   end
 
   context 'with a Pipfile.lock containing python_version 3.10' do
-    let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_python_3.10') }
+    let(:allow_failure) { false }
+    let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_python_3.10', allow_failure:) }
+
+    context 'when using Heroku-22 or older', stacks: %w[heroku-20 heroku-22] do
+      include_examples 'builds using Pipenv with the requested Python version', LATEST_PYTHON_3_10
+    end
 
-    include_examples 'builds using Pipenv with the requested Python version', LATEST_PYTHON_3_10
+    context 'when using Heroku-24', stacks: %w[heroku-24] do
+      let(:allow_failure) { true }
+
+      # We only support Python 3.10 on Heroku-22 and older.
+      include_examples 'aborts the build with a runtime not available message (Pipenv)', LATEST_PYTHON_3_10
+    end
   end
 
   context 'with a Pipfile.lock containing python_version 3.11' do

spec/hatchet/python_update_warning_spec.rb

@@ -66,7 +66,7 @@
       end
     end
 
-    context 'when using Heroku-22', stacks: %w[heroku-22] do
+    context 'when using Heroku-22 or newer', stacks: %w[heroku-22 heroku-24] do
       let(:allow_failure) { true }
 
       # We only support Python 3.8 on Heroku-20 and older.
@@ -75,16 +75,35 @@
   end
 
   context 'with a runtime.txt containing python-3.9.12' do
-    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.9_outdated') }
+    let(:allow_failure) { false }
+    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.9_outdated', allow_failure:) }
+
+    context 'when using Heroku-22 or older', stacks: %w[heroku-20 heroku-22] do
+      include_examples 'warns there is a Python update available', '3.9.12', LATEST_PYTHON_3_9
+    end
 
-    include_examples 'warns there is a Python update available', '3.9.12', LATEST_PYTHON_3_9
+    context 'when using Heroku-24', stacks: %w[heroku-24] do
+      let(:allow_failure) { true }
+
+      # We only support Python 3.9 on Heroku-22 and older.
+      include_examples 'aborts the build without showing an update warning', '3.9.12'
+    end
   end
 
   context 'with a runtime.txt containing python-3.10.5' do
     let(:allow_failure) { false }
     let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.10_outdated', allow_failure:) }
 
-    include_examples 'warns there is a Python update available', '3.10.5', LATEST_PYTHON_3_10
+    context 'when using Heroku-22 or older', stacks: %w[heroku-20 heroku-22] do
+      include_examples 'warns there is a Python update available', '3.10.5', LATEST_PYTHON_3_10
+    end
+
+    context 'when using Heroku-24', stacks: %w[heroku-24] do
+      let(:allow_failure) { true }
+
+      # We only support Python 3.10 on Heroku-22 and older.
+      include_examples 'aborts the build without showing an update warning', '3.10.5'
+    end
   end
 
   context 'with a runtime.txt containing python-3.11.0' do

spec/hatchet/python_version_spec.rb

@@ -54,7 +54,11 @@
       end
     end
 
-    context 'with an app last built using an older default Python version' do
+    # TODO: Enable on Heroku-24 after the default Python version next changes (for the 3.12.4
+    # release), since for now there isn't a historic buildpack version we can use in this test
+    # that is both compatible with the new Heroku-24 S3 asset URLs and also has a different
+    # default Python version so that we can test the sticky versions feature.
+    context 'with an app last built using an older default Python version', stacks: %w[heroku-20 heroku-22] do
       # This test performs an initial build using an older buildpack version, followed
       # by a build using the current version. This ensures that the current buildpack
       # can successfully read the version metadata written to the build cache in the past.
@@ -163,7 +167,7 @@
       end
     end
 
-    context 'when using Heroku-22', stacks: %w[heroku-22] do
+    context 'when using Heroku-22 or newer', stacks: %w[heroku-22 heroku-24] do
       let(:allow_failure) { true }
 
       # We only support Python 3.8 on Heroku-20 and older.
@@ -172,15 +176,35 @@
   end
 
   context 'when runtime.txt contains python-3.9.19' do
-    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.9') }
+    let(:allow_failure) { false }
+    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.9', allow_failure:) }
+
+    context 'when using Heroku-22 or older', stacks: %w[heroku-20 heroku-22] do
+      include_examples 'builds with the requested Python version', LATEST_PYTHON_3_9
+    end
 
-    include_examples 'builds with the requested Python version', LATEST_PYTHON_3_9
+    context 'when using Heroku-24', stacks: %w[heroku-24] do
+      let(:allow_failure) { true }
+
+      # We only support Python 3.9 on Heroku-22 and older.
+      include_examples 'aborts the build with a runtime not available message', "python-#{LATEST_PYTHON_3_9}"
+    end
   end
 
   context 'when runtime.txt contains python-3.10.14' do
-    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.10') }
+    let(:allow_failure) { false }
+    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.10', allow_failure:) }
 
-    include_examples 'builds with the requested Python version', LATEST_PYTHON_3_10
+    context 'when using Heroku-22 or older', stacks: %w[heroku-20 heroku-22] do
+      include_examples 'builds with the requested Python version', LATEST_PYTHON_3_10
+    end
+
+    context 'when using Heroku-24', stacks: %w[heroku-24] do
+      let(:allow_failure) { true }
+
+      # We only support Python 3.10 on Heroku-22 and older.
+      include_examples 'aborts the build with a runtime not available message', "python-#{LATEST_PYTHON_3_10}"
+    end
   end
 
   context 'when runtime.txt contains python-3.11.9' do
@@ -214,20 +238,20 @@
   end
 
   context 'when the requested Python version has changed since the last build' do
-    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.9') }
+    let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.11') }
 
     it 'builds with the new Python version after removing the old install' do
       app.deploy do |app|
-        File.write('runtime.txt', "python-#{LATEST_PYTHON_3_10}")
+        File.write('runtime.txt', "python-#{LATEST_PYTHON_3_12}")
         app.commit!
         app.push!
         # TODO: The output shouldn't say "installing from cache", since it's not.
         expect(clean_output(app.output)).to include(<<~OUTPUT)
           remote: -----> Python app detected
           remote: -----> Using Python version specified in runtime.txt
-          remote: -----> Python version has changed from python-#{LATEST_PYTHON_3_9} to python-#{LATEST_PYTHON_3_10}, clearing cache
+          remote: -----> Python version has changed from python-#{LATEST_PYTHON_3_11} to python-#{LATEST_PYTHON_3_12}, clearing cache
           remote: -----> No change in requirements detected, installing from cache
-          remote: -----> Installing python-#{LATEST_PYTHON_3_10}
+          remote: -----> Installing python-#{LATEST_PYTHON_3_12}
           remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION}
           remote: -----> Installing SQLite3
           remote: -----> Installing requirements with pip

spec/hatchet/stack_spec.rb

@@ -3,7 +3,7 @@
 require_relative '../spec_helper'
 
 RSpec.describe 'Stack changes' do
-  context 'when the stack is upgraded from Heroku-20 to Heroku-22', stacks: %w[heroku-20] do
+  context 'when the stack is upgraded from Heroku-22 to Heroku-24', stacks: %w[heroku-22] do
     # This test performs an initial build using an older buildpack version, followed
     # by a build using the current version. This ensures that the current buildpack
     # can successfully read the stack metadata written to the build cache in the past.
@@ -14,8 +14,8 @@
 
     it 'clears the cache before installing again whilst preserving the sticky Python version' do
       app.deploy do |app|
-        expect(app.output).to include('Building on the Heroku-20 stack')
-        app.update_stack('heroku-22')
+        expect(app.output).to include('Building on the Heroku-22 stack')
+        app.update_stack('heroku-24')
         update_buildpacks(app, [:default])
         app.commit!
         app.push!
@@ -28,7 +28,7 @@
           remote:  !     A Python security update is available! Upgrade as soon as possible to: python-#{LATEST_PYTHON_3_12}
           remote:  !     See: https://devcenter.heroku.com/articles/python-runtimes
           remote:  !     
-          remote: -----> Stack has changed from heroku-20 to heroku-22, clearing cache
+          remote: -----> Stack has changed from heroku-22 to heroku-24, clearing cache
           remote: -----> No change in requirements detected, installing from cache
           remote: -----> Installing python-3.12.2
           remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION}
@@ -40,20 +40,20 @@
     end
   end
 
-  context 'when the stack is downgraded from Heroku-22 to Heroku-20', stacks: %w[heroku-22] do
+  context 'when the stack is downgraded from Heroku-24 to Heroku-22', stacks: %w[heroku-24] do
     let(:app) { Hatchet::Runner.new('spec/fixtures/python_version_unspecified') }
 
     it 'clears the cache before installing again' do
       app.deploy do |app|
-        expect(app.output).to include('Building on the Heroku-22 stack')
-        app.update_stack('heroku-20')
+        expect(app.output).to include('Building on the Heroku-24 stack')
+        app.update_stack('heroku-22')
         app.commit!
         app.push!
         expect(clean_output(app.output)).to include(<<~OUTPUT)
           remote: -----> Python app detected
           remote: -----> No Python version was specified. Using the same version as the last build: python-#{DEFAULT_PYTHON_VERSION}
           remote:        To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes
-          remote: -----> Stack has changed from heroku-22 to heroku-20, clearing cache
+          remote: -----> Stack has changed from heroku-24 to heroku-22, clearing cache
           remote: -----> No change in requirements detected, installing from cache
           remote: -----> Installing python-#{DEFAULT_PYTHON_VERSION}
           remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION}

spec/spec_helper.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 ENV['HATCHET_BUILDPACK_BASE'] ||= 'https://github.com/heroku/heroku-buildpack-python.git'
-ENV['HATCHET_DEFAULT_STACK'] ||= 'heroku-22'
+ENV['HATCHET_DEFAULT_STACK'] ||= 'heroku-24'
 
 require 'rspec/core'
 require 'hatchet'