From 4ddd50bb970bc160a72af01961c422ff103cd6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Delmas?= Date: Thu, 16 Nov 2023 16:39:12 -0500 Subject: [PATCH] Set python to 3.7 in docker container and release action (#2879) An attempt to fix a release build issue due to cbmc-viewer depending on voluptuous which now depends on python 3.7 for type annotations. This adds a ppa in the dockerfile and github release action for ubuntu 18.04 and installs python 3.7. By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 and MIT licenses. --------- Co-authored-by: Remi Delmas Co-authored-by: jaisnan --- .github/workflows/release.yml | 12 ++- docs/src/install-guide.md | 2 +- .../ci/Dockerfile.bundle-test-ubuntu-18-04 | 14 ++- src/os_hacks.rs | 91 ------------------- src/setup.rs | 9 +- 5 files changed, 24 insertions(+), 104 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aa120dafb790..74af1c0be544 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,17 +68,23 @@ jobs: df -h # This is required before checkout because the container does not - # have Git installed, so cannot run checkout action. The checkout - # action requires Git >=2.18, so use the Git maintainers' PPA. + # have Git installed, so cannot run checkout action. + # The checkout action requires Git >=2.18 and python 3.7, so use the Git maintainers' PPA. + # and the "deadsnakes" PPA, as the default version of python on ubuntu 22.04 is Python 3.10 - name: Install system dependencies run: | apt-get update apt-get install -y software-properties-common apt-utils add-apt-repository ppa:git-core/ppa + add-apt-repository ppa:deadsnakes/ppa apt-get update apt-get install -y \ build-essential bash-completion curl lsb-release sudo g++ gcc flex \ - bison make patch git + bison make patch git python3.7 python3.7-dev python3.7-distutils + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 + curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py + python3 get-pip.py --force-reinstall + rm get-pip.py - name: Checkout Kani uses: actions/checkout@v3 diff --git a/docs/src/install-guide.md b/docs/src/install-guide.md index 7f5bd0a7f232..776209987ba1 100644 --- a/docs/src/install-guide.md +++ b/docs/src/install-guide.md @@ -14,7 +14,7 @@ GitHub CI workflows, see [GitHub CI Action](./install-github-ci.md). The following must already be installed: -* **Python version 3.6 or newer** and the package installer `pip`. +* **Python version 3.7 or newer** and the package installer `pip`. * Rust 1.58 or newer installed via `rustup`. * `ctags` is required for Kani's `--visualize` option to work correctly. [Universal ctags](https://ctags.io/) is recommended. diff --git a/scripts/ci/Dockerfile.bundle-test-ubuntu-18-04 b/scripts/ci/Dockerfile.bundle-test-ubuntu-18-04 index a4cc79c80f07..d3ddfd20fdbf 100644 --- a/scripts/ci/Dockerfile.bundle-test-ubuntu-18-04 +++ b/scripts/ci/Dockerfile.bundle-test-ubuntu-18-04 @@ -10,9 +10,19 @@ FROM ubuntu:18.04 ENV DEBIAN_FRONTEND=noninteractive \ DEBCONF_NONINTERACTIVE_SEEN=true + RUN apt-get update && \ - apt-get install -y python3 python3-pip curl ctags && \ - curl -sSf https://sh.rustup.rs | sh -s -- -y + apt-get install --no-install-recommends -y build-essential software-properties-common && \ + add-apt-repository -y ppa:deadsnakes/ppa && \ + apt install --no-install-recommends -y python3.7 python3.7-dev python3.7-distutils curl ctags + +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 + +RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py --force-reinstall && \ + rm get-pip.py + +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" WORKDIR /tmp/kani diff --git a/src/os_hacks.rs b/src/os_hacks.rs index cf39c39be82d..ca064a662901 100644 --- a/src/os_hacks.rs +++ b/src/os_hacks.rs @@ -5,7 +5,6 @@ //! "flow" of code in setup.rs, this module contains all functions that implement os-specific //! workarounds. -use std::ffi::OsString; use std::path::Path; use std::process::Command; @@ -14,74 +13,6 @@ use os_info::Info; use crate::cmd::AutoRun; -pub fn should_apply_ubuntu_18_04_python_hack(os: &os_info::Info) -> Result { - if os.os_type() != os_info::Type::Ubuntu { - return Ok(false); - } - // Check both versions: https://github.com/stanislav-tkach/os_info/issues/318 - if *os.version() != os_info::Version::Semantic(18, 4, 0) - && *os.version() != os_info::Version::Custom("18.04".into()) - { - return Ok(false); - } - // It's not enough to check that we're on Ubuntu 18.04 because the user may have - // manually updated to a newer version of Python instead of using what the OS ships. - // So check if it looks like the OS-shipped version as best we can. - let cmd = Command::new("python3").args(["-m", "pip", "--version"]).output()?; - let output = std::str::from_utf8(&cmd.stdout)?; - // The problem version looks like: - // 'pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)' - // So we'll test for version 9. - Ok(pip_major_version(output)? == 9) -} - -/// Unit testable parsing function for extracting pip version numbers, from strings that look like: -/// 'pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)' -fn pip_major_version(output: &str) -> Result { - // We don't want dependencies so parse with stdlib string functions as best we can. - let mut words = output.split_whitespace(); - let _pip = words.next().context("No pip output")?; - let version = words.next().context("No pip version")?; - - let mut versions = version.split('.'); - let major = versions.next().context("No pip major version")?; - - Ok(major.parse()?) -} - -/// See [`crate::setup::setup_python_deps`] -pub fn setup_python_deps_on_ubuntu_18_04(pyroot: &Path, pkg_versions: &[&str]) -> Result<()> { - println!("Applying a workaround for 18.04..."); - // https://github.com/pypa/pip/issues/3826 - // Ubuntu 18.04 has a patched-to-be-broken version of pip that just straight-up makes `--target` not work. - // Worse still, there is no apparent way to replicate the correct behavior cleanly. - - // This is a really awful hack to simulate getting the same result. I can find no other solution. - // Example failed approach: `--system --target pyroot` fails to create a `pyroot/bin` with binaries. - - // Step 1: use `--system --prefix pyroot`. This disables the broken behavior, and creates `bin` but... - Command::new("python3") - .args(["-m", "pip", "install", "--system", "--prefix"]) - .arg(pyroot) - .args(pkg_versions) - .run()?; - - // Step 2: move `pyroot/lib/python3.6/site-packages/*` up to `pyroot` - // This seems to successfully replicate the behavior of `--target` - // "mv" is not idempotent however so we need to do "cp -r" then delete - let mut cp_cmd = OsString::new(); - cp_cmd.push("cp -r "); - cp_cmd.push(pyroot.as_os_str()); - cp_cmd.push("/lib/python*/site-packages/* "); - cp_cmd.push(pyroot.as_os_str()); - Command::new("bash").arg("-c").arg(cp_cmd).run()?; - - // `lib` is the directory `--prefix` creates that `--target` does not. - std::fs::remove_dir_all(pyroot.join("lib"))?; - - Ok(()) -} - /// This is the final step of setup, where we look for OSes that require additional setup steps /// beyond the usual ones that we have done already. pub fn setup_os_hacks(kani_dir: &Path, os: &Info) -> Result<()> { @@ -159,25 +90,3 @@ fn setup_nixos_patchelf(kani_dir: &Path) -> Result<()> { Ok(()) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn check_pip_major_version() -> Result<()> { - // These read a lot better formatted on one line, so shorten them: - use pip_major_version as p; - // 18.04 example: (with extra newline to test whitespace handling) - assert_eq!(p("pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)\n")?, 9); - // a mac - assert_eq!(p("pip 21.1.1 from /usr/local/python3.9/site-packages/pip (python 3.9)")?, 21); - // 20.04 - assert_eq!(p("pip 20.0.2 from /usr/lib/python3/dist-packages/pip (python 3.8)")?, 20); - // How mangled can we get and still "work"? - assert_eq!(p("pip 1")?, 1); - assert_eq!(p("p 1")?, 1); - assert_eq!(p("\n\n p 1 p")?, 1); - Ok(()) - } -} diff --git a/src/setup.rs b/src/setup.rs index 747d3435108a..ffdcf340e336 100644 --- a/src/setup.rs +++ b/src/setup.rs @@ -83,7 +83,7 @@ pub fn setup(use_local_bundle: Option) -> Result<()> { setup_rust_toolchain(&kani_dir)?; - setup_python_deps(&kani_dir, &os)?; + setup_python_deps(&kani_dir)?; os_hacks::setup_os_hacks(&kani_dir, &os)?; @@ -152,18 +152,13 @@ fn setup_rust_toolchain(kani_dir: &Path) -> Result { } /// Install into the pyroot the python dependencies we need -fn setup_python_deps(kani_dir: &Path, os: &os_info::Info) -> Result<()> { +fn setup_python_deps(kani_dir: &Path) -> Result<()> { println!("[4/5] Installing Kani python dependencies..."); let pyroot = kani_dir.join("pyroot"); // TODO: this is a repetition of versions from kani/kani-dependencies let pkg_versions = &["cbmc-viewer==3.8"]; - if os_hacks::should_apply_ubuntu_18_04_python_hack(os)? { - os_hacks::setup_python_deps_on_ubuntu_18_04(&pyroot, pkg_versions)?; - return Ok(()); - } - Command::new("python3") .args(["-m", "pip", "install", "--target"]) .arg(&pyroot)