Adds action inputs for python packages and repos (#20)

- Adds `python-packages` action parameter to add python packages
 that will be later sideloaded (with dependencies) to the
 emulated Linux.
- Adds `repos` action parameter to add custom git repositories
 that will be later sideloaded to the emulated Linux.
- Changes emulation speedup method from increasing MIPS to setting
 the advanceImmediately flag to Renode
- Adds additional comments and updates README

.github/workflows/build_and_test.yml

@@ -57,23 +57,20 @@ jobs:
          sudo apt update -qq && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \
          python3-pip telnet iptables iproute2
 
-      - run: git clone https://github.com/antmicro/pyrav4l2.git tests/pyrav4l2
-
       - name: test
         uses: ./
         with:
           shared-dir: ./tests
           image: https://github.com/${{ github.repository }}/releases/download/latest/images.tar.xz
           renode-run: |
-            ping -c 3 172.16.0.1
             wget example.org
             gpiodetect
             sh gpio.sh
             sh i2c.sh
             python test.py
-            pip install ./pyrav4l2
             python controls-enumeration.py
           devices: |
             vivid
             gpio 0 16
             i2c 0x1C
+          python-packages: git+https://github.com/antmicro/pyrav4l2.git

.github/workflows/pull_request.yml

@@ -16,23 +16,20 @@ jobs:
       - name: build kernel
         run: ./scripts/compile.sh
 
-      - run: git clone https://github.com/antmicro/pyrav4l2.git tests/pyrav4l2
-
       - name: test
         uses: ./
         with:
           shared-dir: ./tests
           image: ./images.tar.xz
           renode-run: |
-            ping -c 3 172.16.0.1
             wget example.org
             gpiodetect
             sh gpio.sh
             sh i2c.sh
             python test.py
-            pip install ./pyrav4l2
             python controls-enumeration.py
           devices: |
             vivid
             gpio 0 16
             i2c 0x1C
+          python-packages: git+https://github.com/antmicro/pyrav4l2.git

README.md

@@ -9,17 +9,20 @@ renode-linux-runner-action is a GitHub Action that can run scripts on Linux insi
 The emulated system is based on the [Buildroot 2022.11.3](https://github.com/buildroot/buildroot/tree/2022.11.3) and it runs on the RISC-V/HiFive Unleashed platform in [Renode 1.13.3](https://github.com/renode/renode).
 It contains the Linux kernel configured with some emulated devices enabled and it has the following packages installed:
 
-- Python 3.10.7
+- Python 3.10.8
 - pip 21.2.4
 - v4l2-utils 1.22.1
 - libgpiod tools 1.6.3
+- git 2.31.7
 
 ## Parameters
 
 - `shared-dir` - Path to the shared directory. Contents of this directory will be mounted in Renode. The embedded Linux in Renode will start in this directory.
 - `renode-run` - A command or a list of commands to run in Renode.
 - `devices` - List of devices to add to the workflow. If not specified, the action will not install any devices.
 - `image` - url of path to tar.xz archive with compiled embedded Linux image. If not specified, the action will use the default one. See releases for examples.
+- `python-packages` - python packages from pypi library or git repository that will be sideloaded into emulated Linux.
+- `repos` - git repositories that will be sideloaded into emulated Linux.
 
 ### Devices syntax
 

action.yml

@@ -14,6 +14,14 @@ inputs:
     description: Compiled linux image source
     required: false
     default: "https://github.com/antmicro/renode-linux-runner-action/releases/download/latest/images.tar.xz"
+  python-packages:
+    description: Custom python packages that will be added to your shared dir
+    required: false
+    default: ""
+  repos:
+    description: Custom git repos that will be added to your shared dir
+    required: false
+    default: ""
 runs:
   using: composite
   steps:
@@ -40,17 +48,26 @@ runs:
       run: cd ${{github.action_path}} && tar xf images.tar.xz
       shell: bash
 
+    - id: install-packages
+      run: |
+        wget -q --no-verbose https://github.com/renode/renode/releases/download/v1.13.3/renode_1.13.3_amd64.deb && \
+        sudo DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends                             \
+        ./renode_1.13.3_amd64.deb python3-venv
+      shell: bash
+
     - id: install-pip-packages
-      run: pip3 install pexpect
+      run: pip3 install pexpect virtualenv
       shell: bash
 
-    - id: download-renode
-      run: |
-        wget -q --no-verbose https://github.com/renode/renode/releases/download/v1.13.3/renode_1.13.3_amd64.deb &&
-        sudo DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \
-        ./renode_1.13.3_amd64.deb
+    - id: create-pip-virtual-environment
+      run: cd ${{github.action_path}} && mkdir -p venv-dir && python3 -m venv venv-dir
       shell: bash
 
     - id: test
-      run: cd ${{github.action_path}} && sudo python action/run-in-renode.py "${{ inputs.renode-run }}" "${{ inputs.devices }}"
+      run: |
+        cd ${{github.action_path}} && sudo python3 action/run-in-renode.py \
+        "${{ inputs.renode-run }}"                                         \
+        "${{ inputs.devices }}"                                            \
+        "${{ inputs.python-packages }}"                                    \
+        "${{ inputs.repos }}"
       shell: bash

action/hifive.resc

@@ -13,7 +13,7 @@ virtio LoadImage @drive.img
 connector Connect sysbus.ethernet switch0
 
 # This setting increases emulation speed, thus mitigates networks errors and speedup user scripts.
-u54_1 PerformanceInMips 150
+machine SetAdvanceImmediately true
 
 showAnalyzer uart0
 e51 LogFunctionNames true

action/run-in-renode.py

@@ -23,10 +23,13 @@
 from typing import Any, Protocol
 from string import hexdigits
 from datetime import datetime
+from json import loads as json_loads
+from os import getcwd as os_getcwd
 
 CR = r'\r'
 HOST_INTERFACE = "eth0"
 TAP_INTERFACE = "tap0"
+RENODE_PIP_PACKAGES_DIR = "./pip_packages"
 
 
 class FilteredStdout(object):
@@ -144,7 +147,7 @@ class Device_Prototype:
         commands that is needed to add device
     params: list[str]
         default parameters
-    command_action: list[tuple[str, int]]
+    command_action: list[tuple[Action, int]]
         defines number of parameters needed and the
         Action itself
     """
@@ -187,6 +190,15 @@ class Device:
 added_devices: list[Device] = []
 
 
+default_packages = []
+
+
+downloaded_packages = []
+
+
+downloaded_repos = []
+
+
 def add_devices(devices: str):
     """
     Parses arguments and commands, and adds devices to the
@@ -242,14 +254,125 @@ def add_devices(devices: str):
             print(f"WARNING: Device {device_name} not found")
 
 
+def get_package(child: px_spawn, package_name: str):
+    """
+    Download selected python package for riscv64 platform.
+
+    Parameters
+    ----------
+    child: px_spawn
+        pexpect spawn with shell and python virtual environment enabled
+    package_name: str
+        package to download
+    """
+
+    global downloaded_packages
+
+    child.sendcontrol('c')
+    run_cmd(child, "(venv-dir) #", f"pip download {package_name} --platform=linux_riscv64 --no-deps --progress-bar off --disable-pip-version-check")
+    child.expect_exact('(venv-dir) #')
+
+    # Removes strange ASCII control codes that appear during some 'pip download' runs.
+    ansi_escape = re_compile(r'\x1B[@-_][0-?]*[ -/]*[@-~]')
+    output_str: str = ansi_escape.sub('', child.before)
+
+    downloaded_packages += [file.split(' ')[1] for file in output_str.splitlines() if file.startswith('Saved')]
+
+
+def add_packages(packages: str):
+    """
+    Download all selected python packages and their dependencies
+    for the riscv64 platform to sideload it later to emulated Linux.
+    Parameters
+    ----------
+    packages: str
+        raw string from github action, syntax defined in README.md
+    """
+
+    child = px_spawn(f'sh -c "cd {os_getcwd()};exec /bin/sh"', encoding="utf-8", timeout=60)
+
+    try:
+        child.expect_exact('#')
+        child.sendline('')
+
+        # FilteredStdout is used to remove \r characters from telnet output.
+        # GitHub workflow log GUI interprets this sequence as newline.
+        child.logfile_read = FilteredStdout(sys_stdout, CR, "")
+
+        run_cmd(child, "#", ". ./venv-dir/bin/activate")
+
+        # Since the pip version in Ubuntu 22.04 is 22.0.2 and the first stable pip that supporting the --report flag is 23.0,
+        # pip needs to be updated in venv. This workaround may be removed later.
+        run_cmd(child, "(venv-dir) #", "pip -q install pip==23.0.1 --progress-bar off --disable-pip-version-check")
+
+        for package in default_packages + packages.splitlines():
+
+            print(f"processing: {package}")
+
+            # prepare report
+            run_cmd(child, "(venv-dir) #", f"pip install -q {package} --dry-run --report report.json --progress-bar off --disable-pip-version-check")
+            child.expect_exact("(venv-dir)")
+
+            with open("report.json", "r", encoding="utf-8") as report_file:
+                report = json_loads(report_file.read())
+
+            print(f"Packages to install: {len(report['install'])}")
+
+            for dependency in report["install"]:
+
+                dependency_name = dependency["metadata"]["name"] + "==" + dependency["metadata"]["version"] \
+                    if "vcs_info" not in dependency["download_info"] \
+                    else "git+" + dependency["download_info"]["url"] + "@" + dependency["download_info"]["vcs_info"]["commit_id"]
+                get_package(child, dependency_name)
+
+            child.sendcontrol("c")
+
+        run_cmd(child, "(venv-dir) #", "deactivate")
+        child.expect_exact("#")
+    except px_TIMEOUT:
+        print("Timeout!")
+        sys_exit(1)
+
+
+def add_repos(repos: str):
+    """
+    Download all selected git repos to sideload it later to emulated Linux.
+
+    Parameters
+    ----------
+    repos: str
+        raw string from github action, syntax defined in README.md
+    """
+
+    global downloaded_repos
+
+    for repo in repos.splitlines():
+
+        repo = repo.split(' ')
+        repo, folder = repo[0], repo[1] if len(repo) > 1 else repo[0].split('/')[-1]
+
+        print(f'Cloning {repo}' + f'to {folder}' if folder != '' else '')
+        run(['git', 'clone', repo, folder],)
+
+        downloaded_repos += [folder]
+
+
 def create_shared_directory_image():
     """
     Creates an image of the shared directory that will be mounted into the Renode machine.
     When creating the image fails, it exits from the script with the same error code as failing command.
     """
 
+    run(["mkdir", "-p", f"/mnt/user/{RENODE_PIP_PACKAGES_DIR}"])
+
+    for package in downloaded_packages:
+        run(['mv', package, f"/mnt/user/{RENODE_PIP_PACKAGES_DIR}"])
+
+    for repo in downloaded_repos:
+        run(['mv', repo, "/mnt/user"])
+
     try:
-        run(["truncate", "drive.img", "-s", "100M"], check=True)
+        run(["truncate", "drive.img", "-s", "200M"], check=True)
         run(["mkfs.ext4", "-d", "/mnt/user", "drive.img"],
             check=True,
             stdout=DEVNULL)
@@ -300,6 +423,9 @@ def run_cmd(child_process: px_spawn,
 
 
 def setup_network():
+    """
+    Setups host tap interface and connect it to Renode.
+    """
 
     child = px_spawn("sh", encoding="utf-8", timeout=10)
 
@@ -371,9 +497,8 @@ def setup_renode():
 
         # Extracting files from Virtio
 
-        run_cmd(child, "#", "mkdir /mnt/drive")
-        run_cmd(child, "#", "mount /dev/vda /mnt/drive")
-        run_cmd(child, "#", "cd /mnt/drive")
+        run_cmd(child, "#", "mount /dev/vda /mnt")
+        run_cmd(child, "#", "cd /mnt")
         run_cmd(child, "#", "mkdir -p /sys/kernel/debug")
         run_cmd(child, "#", "mount -t debugfs nodev /sys/kernel/debug")
 
@@ -389,13 +514,6 @@ def setup_renode():
 
         run_cmd(child, "#", f'date -s "{now.strftime("%Y-%m-%d %H:%M:%S")}"')
 
-        # pip configuration
-        # Disable pip version checking. Pip runs very slowly in Renode without this setting.
-
-        run_cmd(child, "#", "mkdir -p $HOME/.config/pip")
-        run_cmd(child, "#", 'echo "[global]" >> $HOME/.config/pip/pip.conf')
-        run_cmd(child, "#", 'echo "disable-pip-version-check = True" >> $HOME/.config/pip/pip.conf')
-
         # increase git buffers
         # mitigates issues with broken pipe `Send failure: Broken pipe`
         run_cmd(child, "#", 'git config --global http.maxRequestBuffer 240M')
@@ -412,6 +530,38 @@ def setup_renode():
         sys_exit(1)
 
 
+def setup_python():
+    """
+    Install previously downloaded python packages in emulated linux.
+    """
+
+    child = px_spawn("telnet 127.0.0.1 1234", encoding="utf-8", timeout=None)
+
+    try:
+        child.expect_exact("'^]'.")
+        child.sendcontrol("c")
+
+        # FilteredStdout is used to remove \r characters from telnet output.
+        # GitHub workflow log GUI interprets this sequence as newline.
+        child.logfile_read = FilteredStdout(sys_stdout, CR, "")
+
+        # pip configuration
+        # Disable pip version checking. Pip runs very slowly in Renode without this setting.
+
+        run_cmd(child, "#", "mkdir -p $HOME/.config/pip")
+        run_cmd(child, "#", 'echo "[global]" >> $HOME/.config/pip/pip.conf')
+        run_cmd(child, "#", 'echo "disable-pip-version-check = True" >> $HOME/.config/pip/pip.conf')
+
+        run_cmd(child, "#", f"pip install {' '.join([f'{RENODE_PIP_PACKAGES_DIR}/{package}' for package in downloaded_packages])} --no-index --no-deps --no-build-isolation")
+
+        run_cmd(child, "#", f"rm -r {RENODE_PIP_PACKAGES_DIR}/")
+
+        child.expect_exact("#")
+    except px_TIMEOUT:
+        print("Timeout!")
+        sys_exit(1)
+
+
 def run_cmds_in_renode(commands_to_run: str):
     """
     Runs commands specified by user in the Renode instance.
@@ -459,11 +609,18 @@ def run_cmds_in_renode(commands_to_run: str):
         print("Not enough input arguments")
         sys_exit(1)
 
-    if len(sys_argv) == 3 and sys_argv[2] != "":
+    if len(sys_argv) >= 3 and sys_argv[2] != "":
         add_devices(sys_argv[2])
 
+    if len(sys_argv) >= 4 and sys_argv[3] != "":
+        add_packages(sys_argv[3])
+
+    if len(sys_argv) >= 5 and sys_argv[4] != "":
+        add_repos(sys_argv[4])
+
     create_shared_directory_image()
     run_renode_in_background()
     setup_network()
     setup_renode()
+    setup_python()
     run_cmds_in_renode(sys_argv[1])

run-locally.sh

@@ -14,4 +14,4 @@ fi
 echo -e "{\n  \"act\": true\n}" > env.json
 
 act -j $RUNNER -e env.json --container-options "--privileged"
-rm env.json
+rm env.json