Set python to 3.7 in docker container and release action (#2879)

An attempt to fix a release build issue due to cbmc-viewer depending on
voluptuous which now depends on python 3.7 for type annotations.

This adds a ppa in the dockerfile and github release action for ubuntu
18.04 and installs python 3.7.


By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 and MIT licenses.

---------

Co-authored-by: Remi Delmas <delmasrd@amazon.com>
Co-authored-by: jaisnan <jaisnan@amazon.com>

.github/workflows/release.yml

@@ -68,17 +68,23 @@ jobs:
  df -h
 
  # This is required before checkout because the container does not
- # have Git installed, so cannot run checkout action. The checkout
- # action requires Git >=2.18, so use the Git maintainers' PPA.
+ # have Git installed, so cannot run checkout action.
+ # The checkout action requires Git >=2.18 and python 3.7, so use the Git maintainers' PPA.
+ # and the "deadsnakes" PPA, as the default version of python on ubuntu 22.04 is Python 3.10
  - name: Install system dependencies
  run: |
  apt-get update
  apt-get install -y software-properties-common apt-utils
  add-apt-repository ppa:git-core/ppa
+ add-apt-repository ppa:deadsnakes/ppa
  apt-get update
  apt-get install -y \
  build-essential bash-completion curl lsb-release sudo g++ gcc flex \
- bison make patch git
+ bison make patch git python3.7 python3.7-dev python3.7-distutils
+ update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
+ curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+ python3 get-pip.py --force-reinstall
+ rm get-pip.py
 
  - name: Checkout Kani
  uses: actions/checkout@v3

docs/src/install-guide.md

@@ -14,7 +14,7 @@ GitHub CI workflows, see [GitHub CI Action](./install-github-ci.md).
 
 The following must already be installed:
 
-* **Python version 3.6 or newer** and the package installer `pip`.
+* **Python version 3.7 or newer** and the package installer `pip`.
 * Rust 1.58 or newer installed via `rustup`.
 * `ctags` is required for Kani's `--visualize` option to work correctly. [Universal ctags](https://ctags.io/) is recommended.
 

scripts/ci/Dockerfile.bundle-test-ubuntu-18-04

@@ -10,9 +10,19 @@
 FROM ubuntu:18.04
 ENV DEBIAN_FRONTEND=noninteractive \
  DEBCONF_NONINTERACTIVE_SEEN=true
+
 RUN apt-get update && \
- apt-get install -y python3 python3-pip curl ctags && \
- curl -sSf https://sh.rustup.rs | sh -s -- -y
+ apt-get install --no-install-recommends -y build-essential software-properties-common && \
+ add-apt-repository -y ppa:deadsnakes/ppa && \
+ apt install --no-install-recommends -y python3.7 python3.7-dev python3.7-distutils curl ctags
+
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
+
+RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+ python3 get-pip.py --force-reinstall && \
+ rm get-pip.py
+
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 WORKDIR /tmp/kani

src/os_hacks.rs

@@ -5,7 +5,6 @@
 //! "flow" of code in setup.rs, this module contains all functions that implement os-specific
 //! workarounds.
 
-use std::ffi::OsString;
 use std::path::Path;
 use std::process::Command;
 
@@ -14,74 +13,6 @@ use os_info::Info;
 
 use crate::cmd::AutoRun;
 
-pub fn should_apply_ubuntu_18_04_python_hack(os: &os_info::Info) -> Result<bool> {
- if os.os_type() != os_info::Type::Ubuntu {
- return Ok(false);
- }
- // Check both versions: https://github.com/stanislav-tkach/os_info/issues/318
- if *os.version() != os_info::Version::Semantic(18, 4, 0)
- && *os.version() != os_info::Version::Custom("18.04".into())
- {
- return Ok(false);
- }
- // It's not enough to check that we're on Ubuntu 18.04 because the user may have
- // manually updated to a newer version of Python instead of using what the OS ships.
- // So check if it looks like the OS-shipped version as best we can.
- let cmd = Command::new("python3").args(["-m", "pip", "--version"]).output()?;
- let output = std::str::from_utf8(&cmd.stdout)?;
- // The problem version looks like:
- // 'pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)'
- // So we'll test for version 9.
- Ok(pip_major_version(output)? == 9)
-}
-
-/// Unit testable parsing function for extracting pip version numbers, from strings that look like:
-/// 'pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)'
-fn pip_major_version(output: &str) -> Result<u32> {
- // We don't want dependencies so parse with stdlib string functions as best we can.
- let mut words = output.split_whitespace();
- let _pip = words.next().context("No pip output")?;
- let version = words.next().context("No pip version")?;
-
- let mut versions = version.split('.');
- let major = versions.next().context("No pip major version")?;
-
- Ok(major.parse()?)
-}
-
-/// See [`crate::setup::setup_python_deps`]
-pub fn setup_python_deps_on_ubuntu_18_04(pyroot: &Path, pkg_versions: &[&str]) -> Result<()> {
- println!("Applying a workaround for 18.04...");
- // https://github.com/pypa/pip/issues/3826
- // Ubuntu 18.04 has a patched-to-be-broken version of pip that just straight-up makes `--target` not work.
- // Worse still, there is no apparent way to replicate the correct behavior cleanly.
-
- // This is a really awful hack to simulate getting the same result. I can find no other solution.
- // Example failed approach: `--system --target pyroot` fails to create a `pyroot/bin` with binaries.
-
- // Step 1: use `--system --prefix pyroot`. This disables the broken behavior, and creates `bin` but...
- Command::new("python3")
- .args(["-m", "pip", "install", "--system", "--prefix"])
- .arg(pyroot)
- .args(pkg_versions)
- .run()?;
-
- // Step 2: move `pyroot/lib/python3.6/site-packages/*` up to `pyroot`
- // This seems to successfully replicate the behavior of `--target`
- // "mv" is not idempotent however so we need to do "cp -r" then delete
- let mut cp_cmd = OsString::new();
- cp_cmd.push("cp -r ");
- cp_cmd.push(pyroot.as_os_str());
- cp_cmd.push("/lib/python*/site-packages/* ");
- cp_cmd.push(pyroot.as_os_str());
- Command::new("bash").arg("-c").arg(cp_cmd).run()?;
-
- // `lib` is the directory `--prefix` creates that `--target` does not.
- std::fs::remove_dir_all(pyroot.join("lib"))?;
-
- Ok(())
-}
-
 /// This is the final step of setup, where we look for OSes that require additional setup steps
 /// beyond the usual ones that we have done already.
 pub fn setup_os_hacks(kani_dir: &Path, os: &Info) -> Result<()> {
@@ -159,25 +90,3 @@ fn setup_nixos_patchelf(kani_dir: &Path) -> Result<()> {
 
  Ok(())
 }
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn check_pip_major_version() -> Result<()> {
- // These read a lot better formatted on one line, so shorten them:
- use pip_major_version as p;
- // 18.04 example: (with extra newline to test whitespace handling)
- assert_eq!(p("pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6)\n")?, 9);
- // a mac
- assert_eq!(p("pip 21.1.1 from /usr/local/python3.9/site-packages/pip (python 3.9)")?, 21);
- // 20.04
- assert_eq!(p("pip 20.0.2 from /usr/lib/python3/dist-packages/pip (python 3.8)")?, 20);
- // How mangled can we get and still "work"?
- assert_eq!(p("pip 1")?, 1);
- assert_eq!(p("p 1")?, 1);
- assert_eq!(p("\n\n p 1 p")?, 1);
- Ok(())
- }
-}

src/setup.rs

@@ -83,7 +83,7 @@ pub fn setup(use_local_bundle: Option<OsString>) -> Result<()> {
 
  setup_rust_toolchain(&kani_dir)?;
 
- setup_python_deps(&kani_dir, &os)?;
+ setup_python_deps(&kani_dir)?;
 
  os_hacks::setup_os_hacks(&kani_dir, &os)?;
 
@@ -152,18 +152,13 @@ fn setup_rust_toolchain(kani_dir: &Path) -> Result<String> {
 }
 
 /// Install into the pyroot the python dependencies we need
-fn setup_python_deps(kani_dir: &Path, os: &os_info::Info) -> Result<()> {
+fn setup_python_deps(kani_dir: &Path) -> Result<()> {
  println!("[4/5] Installing Kani python dependencies...");
  let pyroot = kani_dir.join("pyroot");
 
  // TODO: this is a repetition of versions from kani/kani-dependencies
  let pkg_versions = &["cbmc-viewer==3.8"];
 
- if os_hacks::should_apply_ubuntu_18_04_python_hack(os)? {
- os_hacks::setup_python_deps_on_ubuntu_18_04(&pyroot, pkg_versions)?;
- return Ok(());
- }
-
  Command::new("python3")
  .args(["-m", "pip", "install", "--target"])
  .arg(&pyroot)