Pipenv editable WIP

.github/workflows/ci.yml

@@ -3,8 +3,8 @@ name: CI
 on:
   push:
     # Avoid duplicate builds on PRs.
-    branches:
-      - main
+    # branches:
+      # - main
   pull_request:
 
 permissions:

bin/steps/pipenv

@@ -6,66 +6,50 @@
 source "$BIN_DIR/utils"
 set -e
 
-if [[ -f Pipfile.lock ]]; then
-    if [[ -f .heroku/python/Pipfile.lock.sha256 ]]; then
-        if [[ $(openssl dgst -sha256 Pipfile.lock) == $(cat .heroku/python/Pipfile.lock.sha256) ]]; then
-            # Don't skip installation if there are git deps.
-            if ! grep -q 'git' Pipfile.lock; then
-                echo "Skipping installation, as Pipfile.lock hasn't changed since last deploy." | indent
+# Previous versions of the buildpack used to cache the checksum of the lockfile to allow
+# for skipping pipenv install if the lockfile was unchanged. However, this is not always safe
+# to do (the lockfile can refer to dependencies that can change independently of the lockfile,
+# for example, when using a local non-editable file dependency), so we no longer ever skip
+# install, and instead defer to pipenv to determine whether install is actually a no-op.
+rm -f .heroku/python/Pipfile.lock.sha256
+
+if [[ -f Pipfile ]]; then
+    # Measure that we're using Pipenv.
+    mcount "tool.pipenv"
+
+    # Skip installing dependencies using pip later.
+    export SKIP_PIP_INSTALL=1
 
-                mcount "tool.pipenv"
-                export SKIP_PIPENV_INSTALL=1
-                export SKIP_PIP_INSTALL=1
-            fi
-        fi
+    # Set Pip env vars
+    # This reads certain environment variables set on the Heroku app config
+    # and makes them accessible to the pip install process.
+    #
+    # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used.
+    if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then
+        PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")"
+        export PIP_EXTRA_INDEX_URL
+        mcount "buildvar.PIP_EXTRA_INDEX_URL"
     fi
-fi
-
-
-if [ ! "$SKIP_PIPENV_INSTALL" ]; then
-    # Pipenv support (Generate requirements.txt with pipenv).
-    if [[ -f Pipfile ]]; then
-        # Measure that we're using Pipenv.
-        mcount "tool.pipenv"
 
-        # Skip pip install, later.
-        export SKIP_PIP_INSTALL=1
+    PIPENV_VERSION=$(get_requirement_version 'pipenv')
 
-        # Set Pip env vars
-        # This reads certain environment variables set on the Heroku app config
-        # and makes them accessible to the pip install process.
-        #
-        # PIP_EXTRA_INDEX_URL allows for an alternate pypi URL to be used.
-        if [[ -r "$ENV_DIR/PIP_EXTRA_INDEX_URL" ]]; then
-            PIP_EXTRA_INDEX_URL="$(cat "$ENV_DIR/PIP_EXTRA_INDEX_URL")"
-            export PIP_EXTRA_INDEX_URL
-            mcount "buildvar.PIP_EXTRA_INDEX_URL"
-        fi
+    /app/.heroku/python/bin/pip install --quiet --disable-pip-version-check --no-cache-dir "pipenv==${PIPENV_VERSION}"
 
-        PIPENV_VERSION=$(get_requirement_version 'pipenv')
+    # Install the test dependencies, for CI.
+    if [ "$INSTALL_TEST" ]; then
+        puts-step "Installing test dependencies"
+        /app/.heroku/python/bin/pipenv install --dev --system --deploy 2>&1 | cleanup | indent
 
-        /app/.heroku/python/bin/pip install --quiet --disable-pip-version-check --no-cache-dir "pipenv==${PIPENV_VERSION}"
+    # Install the dependencies.
+    elif [[ ! -f Pipfile.lock ]]; then
+        puts-step "Installing dependencies with Pipenv ${PIPENV_VERSION}"
+        /app/.heroku/python/bin/pipenv install --system --skip-lock 2>&1 | indent
 
-        # Install the test dependencies, for CI.
-        if [ "$INSTALL_TEST" ]; then
-            puts-step "Installing test dependencies"
-            /app/.heroku/python/bin/pipenv install --dev --system --deploy 2>&1 | cleanup | indent
+    else
+        pipenv-to-pip Pipfile.lock > requirements.txt
+        cp requirements.txt .heroku/python/requirements-declared.txt
 
-        # Install the dependencies.
-        elif [[ ! -f Pipfile.lock ]]; then
-            puts-step "Installing dependencies with Pipenv ${PIPENV_VERSION}"
-            /app/.heroku/python/bin/pipenv install --system --skip-lock 2>&1 | indent
-
-        else
-            pipenv-to-pip Pipfile.lock > requirements.txt
-            cp requirements.txt .heroku/python/requirements-declared.txt
-            openssl dgst -sha256 Pipfile.lock > .heroku/python/Pipfile.lock.sha256
-
-            puts-step "Installing dependencies with Pipenv ${PIPENV_VERSION}"
-            /app/.heroku/python/bin/pipenv install --system --deploy 2>&1 | indent
-        fi
+        puts-step "Installing dependencies with Pipenv ${PIPENV_VERSION}"
+        /app/.heroku/python/bin/pipenv install --system --deploy 2>&1 | indent
     fi
-else
-    export SKIP_PIP_INSTALL=1
-    pipenv-to-pip Pipfile.lock > requirements.txt
 fi

spec/fixtures/pipenv_editable/Pipfile

@@ -0,0 +1,9 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+local-package-pyproject-toml = {file = "packages/local_package_pyproject_toml", editable = true}
+local-package-setup-py = {file = "packages/local_package_setup_py", editable = true}
+gunicorn = {git = "git+https://github.com/benoitc/gunicorn", ref = "20.1.0", editable = true}

spec/fixtures/pipenv_editable/bin/compile

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# This file is run by the inline buildpack, and tests that editable requirements are
+# usable by buildpacks that run after the Python buildpack during the build.
+
+set -euo pipefail
+
+BUILD_DIR="${1}"
+
+cd "${BUILD_DIR}"
+
+exec bin/test-entrypoints

spec/fixtures/pipenv_editable/bin/detect

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# This file is run by the inline buildpack.
+
+set -euo pipefail
+
+echo "Inline"

spec/fixtures/pipenv_editable/bin/post_compile

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+exec bin/test-entrypoints

spec/fixtures/pipenv_editable/bin/test-entrypoints

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# List the filenames and contents of all .egg-link, .pth, and finder files in site-packages.
+find .heroku/python/lib/python*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) | sort | xargs -exec tail -n +1
+echo
+
+echo -n "Running entrypoint for the pyproject.toml-based local package: "
+local_package_pyproject_toml
+
+echo -n "Running entrypoint for the setup.py-based local package: "
+local_package_setup_py
+
+echo -n "Running entrypoint for the VCS package: "
+gunicorn --version

spec/fixtures/pipenv_editable/packages/local_package_pyproject_toml/local_package_pyproject_toml.egg-info/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.1
+Name: local_package_pyproject_toml
+Version: 0.0.1

spec/fixtures/pipenv_editable/packages/local_package_pyproject_toml/local_package_pyproject_toml.egg-info/SOURCES.txt

@@ -0,0 +1,7 @@
+pyproject.toml
+local_package_pyproject_toml/__init__.py
+local_package_pyproject_toml.egg-info/PKG-INFO
+local_package_pyproject_toml.egg-info/SOURCES.txt
+local_package_pyproject_toml.egg-info/dependency_links.txt
+local_package_pyproject_toml.egg-info/entry_points.txt
+local_package_pyproject_toml.egg-info/top_level.txt

spec/fixtures/pipenv_editable/packages/local_package_pyproject_toml/local_package_pyproject_toml.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

spec/fixtures/pipenv_editable/packages/local_package_pyproject_toml/local_package_pyproject_toml.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+local_package_pyproject_toml = local_package_pyproject_toml:hello

spec/fixtures/pipenv_editable/packages/local_package_pyproject_toml/local_package_pyproject_toml.egg-info/top_level.txt

@@ -0,0 +1 @@
+local_package_pyproject_toml

spec/fixtures/pipenv_editable/packages/local_package_pyproject_toml/local_package_pyproject_toml/__init__.py

@@ -0,0 +1,2 @@
+def hello():
+    print("Hello pyproject.toml!")

spec/fixtures/pipenv_editable/packages/local_package_pyproject_toml/pyproject.toml

@@ -0,0 +1,6 @@
+[project]
+name = "local_package_pyproject_toml"
+version = "0.0.1"
+
+[project.scripts]
+local_package_pyproject_toml = "local_package_pyproject_toml:hello"

spec/fixtures/pipenv_editable/packages/local_package_setup_py/local_package_setup_py/__init__.py

@@ -0,0 +1,2 @@
+def hello():
+    print("Hello setup.py!")

spec/fixtures/pipenv_editable/packages/local_package_setup_py/setup.cfg

@@ -0,0 +1,10 @@
+[metadata]
+name = local_package_setup_py
+version = 0.0.1
+
+[options]
+packages = local_package_setup_py
+
+[options.entry_points]
+console_scripts =
+    local_package_setup_py = local_package_setup_py:hello

spec/fixtures/pipenv_editable/packages/local_package_setup_py/setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()

spec/fixtures/requirements_editable/bin/test-entrypoints

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 # List the filenames and contents of all .egg-link, .pth, and finder files in site-packages.
-find .heroku/python/lib*/*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) | sort | xargs -exec tail -n +1
+find .heroku/python/lib/python*/site-packages/ -type f -and \( -name '*.egg-link' -or -name '*.pth' -or -name '__editable___*_finder.py' \) | sort | xargs -exec tail -n +1
 echo
 
 echo -n "Running entrypoint for the pyproject.toml-based local package: "