From 91ea74b0ff71d068fab43bac40124b0ef4e164f4 Mon Sep 17 00:00:00 2001 From: Wiktor Ogrodnik Date: Mon, 24 Apr 2023 13:42:05 +0200 Subject: [PATCH] Adds action inputs for python packages and repos (#20) - Adds `python-packages` action parameter to add python packages that will be later sideloaded (with dependencies) to the emulated Linux. - Adds `repos` action parameter to add custom git repositories that will be later sideloaded to the emulated Linux. - Changes emulation speedup method from increasing MIPS to setting the advanceImmediately flag to Renode - Adds additional comments and updates README --- .github/workflows/build_and_test.yml | 5 +- .github/workflows/pull_request.yml | 5 +- README.md | 5 +- action.yml | 31 ++++- action/hifive.resc | 2 +- action/run-in-renode.py | 183 +++++++++++++++++++++++++-- run-locally.sh | 2 +- 7 files changed, 202 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 8839e48..20632e8 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -57,23 +57,20 @@ jobs: sudo apt update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ python3-pip telnet iptables iproute2 - - run: git clone https://github.com/antmicro/pyrav4l2.git tests/pyrav4l2 - - name: test uses: ./ with: shared-dir: ./tests image: https://github.com/${{ github.repository }}/releases/download/latest/images.tar.xz renode-run: | - ping -c 3 172.16.0.1 wget example.org gpiodetect sh gpio.sh sh i2c.sh python test.py - pip install ./pyrav4l2 python controls-enumeration.py devices: | vivid gpio 0 16 i2c 0x1C + python-packages: git+https://github.com/antmicro/pyrav4l2.git diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 4ec8269..267142e 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -16,23 +16,20 @@ jobs: - name: build kernel run: ./scripts/compile.sh - - run: git clone https://github.com/antmicro/pyrav4l2.git tests/pyrav4l2 - - name: test uses: ./ with: shared-dir: ./tests image: ./images.tar.xz renode-run: | - ping -c 3 172.16.0.1 wget example.org gpiodetect sh gpio.sh sh i2c.sh python test.py - pip install ./pyrav4l2 python controls-enumeration.py devices: | vivid gpio 0 16 i2c 0x1C + python-packages: git+https://github.com/antmicro/pyrav4l2.git diff --git a/README.md b/README.md index 41ad74f..2b40862 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,11 @@ renode-linux-runner-action is a GitHub Action that can run scripts on Linux insi The emulated system is based on the [Buildroot 2022.11.3](https://github.com/buildroot/buildroot/tree/2022.11.3) and it runs on the RISC-V/HiFive Unleashed platform in [Renode 1.13.3](https://github.com/renode/renode). It contains the Linux kernel configured with some emulated devices enabled and it has the following packages installed: -- Python 3.10.7 +- Python 3.10.8 - pip 21.2.4 - v4l2-utils 1.22.1 - libgpiod tools 1.6.3 +- git 2.31.7 ## Parameters @@ -20,6 +21,8 @@ It contains the Linux kernel configured with some emulated devices enabled and i - `renode-run` - A command or a list of commands to run in Renode. - `devices` - List of devices to add to the workflow. If not specified, the action will not install any devices. - `image` - url of path to tar.xz archive with compiled embedded Linux image. If not specified, the action will use the default one. See releases for examples. +- `python-packages` - python packages from pypi library or git repository that will be sideloaded into emulated Linux. +- `repos` - git repositories that will be sideloaded into emulated Linux. ### Devices syntax diff --git a/action.yml b/action.yml index f2b4b48..5433f5e 100644 --- a/action.yml +++ b/action.yml @@ -14,6 +14,14 @@ inputs: description: Compiled linux image source required: false default: "https://github.com/antmicro/renode-linux-runner-action/releases/download/latest/images.tar.xz" + python-packages: + description: Custom python packages that will be added to your shared dir + required: false + default: "" + repos: + description: Custom git repos that will be added to your shared dir + required: false + default: "" runs: using: composite steps: @@ -40,17 +48,26 @@ runs: run: cd ${{github.action_path}} && tar xf images.tar.xz shell: bash + - id: install-packages + run: | + wget -q --no-verbose https://github.com/renode/renode/releases/download/v1.13.3/renode_1.13.3_amd64.deb && \ + sudo DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ + ./renode_1.13.3_amd64.deb python3-venv + shell: bash + - id: install-pip-packages - run: pip3 install pexpect + run: pip3 install pexpect virtualenv shell: bash - - id: download-renode - run: | - wget -q --no-verbose https://github.com/renode/renode/releases/download/v1.13.3/renode_1.13.3_amd64.deb && - sudo DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \ - ./renode_1.13.3_amd64.deb + - id: create-pip-virtual-environment + run: cd ${{github.action_path}} && mkdir -p venv-dir && python3 -m venv venv-dir shell: bash - id: test - run: cd ${{github.action_path}} && sudo python action/run-in-renode.py "${{ inputs.renode-run }}" "${{ inputs.devices }}" + run: | + cd ${{github.action_path}} && sudo python3 action/run-in-renode.py \ + "${{ inputs.renode-run }}" \ + "${{ inputs.devices }}" \ + "${{ inputs.python-packages }}" \ + "${{ inputs.repos }}" shell: bash diff --git a/action/hifive.resc b/action/hifive.resc index c62bbcd..f4d26de 100644 --- a/action/hifive.resc +++ b/action/hifive.resc @@ -13,7 +13,7 @@ virtio LoadImage @drive.img connector Connect sysbus.ethernet switch0 # This setting increases emulation speed, thus mitigates networks errors and speedup user scripts. -u54_1 PerformanceInMips 150 +machine SetAdvanceImmediately true showAnalyzer uart0 e51 LogFunctionNames true diff --git a/action/run-in-renode.py b/action/run-in-renode.py index 0be1a65..f289f83 100755 --- a/action/run-in-renode.py +++ b/action/run-in-renode.py @@ -23,10 +23,13 @@ from typing import Any, Protocol from string import hexdigits from datetime import datetime +from json import loads as json_loads +from os import getcwd as os_getcwd CR = r'\r' HOST_INTERFACE = "eth0" TAP_INTERFACE = "tap0" +RENODE_PIP_PACKAGES_DIR = "./pip_packages" class FilteredStdout(object): @@ -144,7 +147,7 @@ class Device_Prototype: commands that is needed to add device params: list[str] default parameters - command_action: list[tuple[str, int]] + command_action: list[tuple[Action, int]] defines number of parameters needed and the Action itself """ @@ -187,6 +190,15 @@ class Device: added_devices: list[Device] = [] +default_packages = [] + + +downloaded_packages = [] + + +downloaded_repos = [] + + def add_devices(devices: str): """ Parses arguments and commands, and adds devices to the @@ -242,14 +254,125 @@ def add_devices(devices: str): print(f"WARNING: Device {device_name} not found") +def get_package(child: px_spawn, package_name: str): + """ + Download selected python package for riscv64 platform. + + Parameters + ---------- + child: px_spawn + pexpect spawn with shell and python virtual environment enabled + package_name: str + package to download + """ + + global downloaded_packages + + child.sendcontrol('c') + run_cmd(child, "(venv-dir) #", f"pip download {package_name} --platform=linux_riscv64 --no-deps --progress-bar off --disable-pip-version-check") + child.expect_exact('(venv-dir) #') + + # Removes strange ASCII control codes that appear during some 'pip download' runs. + ansi_escape = re_compile(r'\x1B[@-_][0-?]*[ -/]*[@-~]') + output_str: str = ansi_escape.sub('', child.before) + + downloaded_packages += [file.split(' ')[1] for file in output_str.splitlines() if file.startswith('Saved')] + + +def add_packages(packages: str): + """ + Download all selected python packages and their dependencies + for the riscv64 platform to sideload it later to emulated Linux. + Parameters + ---------- + packages: str + raw string from github action, syntax defined in README.md + """ + + child = px_spawn(f'sh -c "cd {os_getcwd()};exec /bin/sh"', encoding="utf-8", timeout=60) + + try: + child.expect_exact('#') + child.sendline('') + + # FilteredStdout is used to remove \r characters from telnet output. + # GitHub workflow log GUI interprets this sequence as newline. + child.logfile_read = FilteredStdout(sys_stdout, CR, "") + + run_cmd(child, "#", ". ./venv-dir/bin/activate") + + # Since the pip version in Ubuntu 22.04 is 22.0.2 and the first stable pip that supporting the --report flag is 23.0, + # pip needs to be updated in venv. This workaround may be removed later. + run_cmd(child, "(venv-dir) #", "pip -q install pip==23.0.1 --progress-bar off --disable-pip-version-check") + + for package in default_packages + packages.splitlines(): + + print(f"processing: {package}") + + # prepare report + run_cmd(child, "(venv-dir) #", f"pip install -q {package} --dry-run --report report.json --progress-bar off --disable-pip-version-check") + child.expect_exact("(venv-dir)") + + with open("report.json", "r", encoding="utf-8") as report_file: + report = json_loads(report_file.read()) + + print(f"Packages to install: {len(report['install'])}") + + for dependency in report["install"]: + + dependency_name = dependency["metadata"]["name"] + "==" + dependency["metadata"]["version"] \ + if "vcs_info" not in dependency["download_info"] \ + else "git+" + dependency["download_info"]["url"] + "@" + dependency["download_info"]["vcs_info"]["commit_id"] + get_package(child, dependency_name) + + child.sendcontrol("c") + + run_cmd(child, "(venv-dir) #", "deactivate") + child.expect_exact("#") + except px_TIMEOUT: + print("Timeout!") + sys_exit(1) + + +def add_repos(repos: str): + """ + Download all selected git repos to sideload it later to emulated Linux. + + Parameters + ---------- + repos: str + raw string from github action, syntax defined in README.md + """ + + global downloaded_repos + + for repo in repos.splitlines(): + + repo = repo.split(' ') + repo, folder = repo[0], repo[1] if len(repo) > 1 else repo[0].split('/')[-1] + + print(f'Cloning {repo}' + f'to {folder}' if folder != '' else '') + run(['git', 'clone', repo, folder],) + + downloaded_repos += [folder] + + def create_shared_directory_image(): """ Creates an image of the shared directory that will be mounted into the Renode machine. When creating the image fails, it exits from the script with the same error code as failing command. """ + run(["mkdir", "-p", f"/mnt/user/{RENODE_PIP_PACKAGES_DIR}"]) + + for package in downloaded_packages: + run(['mv', package, f"/mnt/user/{RENODE_PIP_PACKAGES_DIR}"]) + + for repo in downloaded_repos: + run(['mv', repo, "/mnt/user"]) + try: - run(["truncate", "drive.img", "-s", "100M"], check=True) + run(["truncate", "drive.img", "-s", "200M"], check=True) run(["mkfs.ext4", "-d", "/mnt/user", "drive.img"], check=True, stdout=DEVNULL) @@ -300,6 +423,9 @@ def run_cmd(child_process: px_spawn, def setup_network(): + """ + Setups host tap interface and connect it to Renode. + """ child = px_spawn("sh", encoding="utf-8", timeout=10) @@ -371,9 +497,8 @@ def setup_renode(): # Extracting files from Virtio - run_cmd(child, "#", "mkdir /mnt/drive") - run_cmd(child, "#", "mount /dev/vda /mnt/drive") - run_cmd(child, "#", "cd /mnt/drive") + run_cmd(child, "#", "mount /dev/vda /mnt") + run_cmd(child, "#", "cd /mnt") run_cmd(child, "#", "mkdir -p /sys/kernel/debug") run_cmd(child, "#", "mount -t debugfs nodev /sys/kernel/debug") @@ -389,13 +514,6 @@ def setup_renode(): run_cmd(child, "#", f'date -s "{now.strftime("%Y-%m-%d %H:%M:%S")}"') - # pip configuration - # Disable pip version checking. Pip runs very slowly in Renode without this setting. - - run_cmd(child, "#", "mkdir -p $HOME/.config/pip") - run_cmd(child, "#", 'echo "[global]" >> $HOME/.config/pip/pip.conf') - run_cmd(child, "#", 'echo "disable-pip-version-check = True" >> $HOME/.config/pip/pip.conf') - # increase git buffers # mitigates issues with broken pipe `Send failure: Broken pipe` run_cmd(child, "#", 'git config --global http.maxRequestBuffer 240M') @@ -412,6 +530,38 @@ def setup_renode(): sys_exit(1) +def setup_python(): + """ + Install previously downloaded python packages in emulated linux. + """ + + child = px_spawn("telnet 127.0.0.1 1234", encoding="utf-8", timeout=None) + + try: + child.expect_exact("'^]'.") + child.sendcontrol("c") + + # FilteredStdout is used to remove \r characters from telnet output. + # GitHub workflow log GUI interprets this sequence as newline. + child.logfile_read = FilteredStdout(sys_stdout, CR, "") + + # pip configuration + # Disable pip version checking. Pip runs very slowly in Renode without this setting. + + run_cmd(child, "#", "mkdir -p $HOME/.config/pip") + run_cmd(child, "#", 'echo "[global]" >> $HOME/.config/pip/pip.conf') + run_cmd(child, "#", 'echo "disable-pip-version-check = True" >> $HOME/.config/pip/pip.conf') + + run_cmd(child, "#", f"pip install {' '.join([f'{RENODE_PIP_PACKAGES_DIR}/{package}' for package in downloaded_packages])} --no-index --no-deps --no-build-isolation") + + run_cmd(child, "#", f"rm -r {RENODE_PIP_PACKAGES_DIR}/") + + child.expect_exact("#") + except px_TIMEOUT: + print("Timeout!") + sys_exit(1) + + def run_cmds_in_renode(commands_to_run: str): """ Runs commands specified by user in the Renode instance. @@ -459,11 +609,18 @@ def run_cmds_in_renode(commands_to_run: str): print("Not enough input arguments") sys_exit(1) - if len(sys_argv) == 3 and sys_argv[2] != "": + if len(sys_argv) >= 3 and sys_argv[2] != "": add_devices(sys_argv[2]) + if len(sys_argv) >= 4 and sys_argv[3] != "": + add_packages(sys_argv[3]) + + if len(sys_argv) >= 5 and sys_argv[4] != "": + add_repos(sys_argv[4]) + create_shared_directory_image() run_renode_in_background() setup_network() setup_renode() + setup_python() run_cmds_in_renode(sys_argv[1]) diff --git a/run-locally.sh b/run-locally.sh index c592d90..aa7da79 100755 --- a/run-locally.sh +++ b/run-locally.sh @@ -14,4 +14,4 @@ fi echo -e "{\n \"act\": true\n}" > env.json act -j $RUNNER -e env.json --container-options "--privileged" -rm env.json \ No newline at end of file +rm env.json