Merge remote-tracking branch 'upstream/master' into find_error

.github/workflows/main.yml

@@ -18,14 +18,20 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04]
-        python: ["3.7", "3.8", "3.9", "3.10"]
+        python: ["3.8", "3.9", "3.10"]
         include:
           - os: ubuntu-20.04
-            python: 3.8
+            python: 3.9
             # the following has no effect with manual trigger
             # where the ramp-workflow is specified anyway
             ramp_workflow_version: master
 
+    # Do not ignore bash profile files. From:
+    # https://github.com/marketplace/actions/setup-miniconda
+    defaults:
+      run:
+        shell: bash -l {0}
+
 
     services:
       postgres:
@@ -43,26 +49,23 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Setup conda
-        uses: s-weigand/setup-conda@v1
+        uses: conda-incubator/setup-miniconda@v2
         with:
-          update-conda: true
-          activate-conda: false
+          #mamba-version: "*"
+          channels: conda-forge
+          activate-environment: testenv
+          environment-file: environment.yml
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
           python-version: ${{ matrix.python }}
-          conda-channels: anaconda
 
-      - name: Create envs
+      - name: Create iris_kit env
         run: |
-          conda install --yes mamba -n base -c conda-forge
-          rm -f /usr/share/miniconda/pkgs/cache/*.json # workaround for mamba-org/mamba#488
-          mamba create --yes -n testenv python=$PYTHON_VERSION
-          mamba env update -n testenv -f environment.yml
           mamba env create -f ci_tools/environment_iris_kit.yml
-        env:
-          PYTHON_VERSION: ${{ matrix.python }}
 
       - name: Install ramp-board
         run: |
-          source activate testenv
+          conda activate testenv
           if [ "$PYTHON_VERSION" == "3.8" ]; then
               python -m pip install "dask==2021.4.1" "distributed==2021.4.1"
           fi
@@ -79,33 +82,33 @@ jobs:
       - name: Test ramp-frontend
         run: |
           set -e
-          source activate testenv
+          conda activate testenv
           python -m smtpd -n -c DebuggingServer 127.0.0.1:8025 &
           sleep 3 # wait for the server to start
           pytest -rvsl ramp-frontend --cov=ramp_frontend --cov-report=term-missing --cov-report=xml --cov-append
 
       - name: Test ramp-engine
         run: |
           set -e
-          source activate testenv
+          conda activate testenv
           pytest -rvsl ramp-engine/ --cov=ramp_engine --cov-report=term-missing --cov-report=xml --cov-append
 
       - name: Test ramp-database
         run: |
           set -e
-          source activate testenv
+          conda activate testenv
           pytest -rvsl ramp-database/ --cov=ramp_database --cov-report=term-missing  --cov-report=xml --cov-append
 
       - name: Test ramp-utils
         run: |
           set -e
-          source activate testenv
+          conda activate testenv
           pytest -rvsl ramp-utils/ --cov=ramp_utils --cov-report=term-missing --cov-report=xml --cov-append
 
       - name: Test integration
         run: |
           set -e
-          source activate testenv
+          conda activate testenv
           bash ci_tools/actions/integration_test.sh
 
       - uses: codecov/codecov-action@v3
@@ -123,10 +126,10 @@ jobs:
       - uses: actions/setup-python@v4
         name: Install Python
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - name: Install dependencies
-        run: pip install flake8 black==22.3.0
+        run: pip install flake8==6.0.0  black==22.3.0
 
       - name: Run flake8
         run: flake8 ramp-*

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     rev: 22.3.0
     hooks:
     -   id: black
--   repo: https://gitlab.com/pycqa/flake8
+-   repo: https://github.com/pycqa/flake8
     rev: 3.7.8
     hooks:
     -   id: flake8
@@ -19,4 +19,4 @@ repos:
     hooks:
     -   id: mypy
         types: [file, python]
-        additional_dependencies: [types-PyYAML, types-click, types-itsdangerous, types-Flask, types-Werkzeug]
+        additional_dependencies: [types-PyYAML, types-click, types-itsdangerous, types-Flask, types-Werkzeug, types-requests]

ci_tools/circle/build_doc.sh

@@ -4,14 +4,14 @@ set -e
 
 # Install dependencies with miforge
 curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh
-bash Miniforge3-$(uname)-$(uname -m).sh -b -p $MINIFORGE_PATH
-export PATH="$MINIFORGE_PATH/bin:$PATH"
+bash Miniforge3-$(uname)-$(uname -m).sh -b -p /home/circleci/project/miniforge/
+export PATH="/home/circleci/project/miniforge/bin:$PATH"
 
 # create the environment
 conda create --yes -n testenv python=3.8
 conda env update -n testenv -f environment.yml
 source activate testenv
-conda install --yes sphinx=3.5.4 jinja2=3.0.3 sphinx_rtd_theme numpydoc pygraphviz
+conda install --yes sphinx=3.5.4 jinja2=3.0.3 sphinx_rtd_theme numpydoc==1.2.1 pygraphviz
 pip install eralchemy sphinx-click
 
 # Build and install scikit-learn in dev mode

doc/whats_new/v0.11.rst

@@ -0,0 +1,13 @@
+.. _changes_0_10:
+
+Version 0.11
+============
+
+Changelog
+---------
+
+
+`ramp-database`
+...............
+
+- Switch to fetching starting kit repos via HTTP rather than using git clone to avoid being blocked by Github :pr:`592`.

environment.yml

@@ -19,13 +19,13 @@ dependencies:
   - itsdangerous=2.1.2
   - nbconvert
   - numpy
-  - pandas=1.3.5
+  - pandas>1.3.5,<1.5.0
   - pip
   - psycopg2=2.9.3
   - pyyaml=6.0
   - pytest
   - pytest-cov
-  - scikit-image=0.19.2
   - sqlalchemy==1.4.32
+  - scikit-image=0.19.*
   - pip:
     - ramp-workflow==0.5.0

ramp-database/ramp_database/testing.py

@@ -7,8 +7,9 @@
 import os
 import shutil
 import subprocess
-
-from git import Repo
+import requests
+from io import BytesIO
+from zipfile import ZipFile
 
 from ramp_utils import read_config
 from ramp_utils import generate_ramp_config
@@ -123,6 +124,28 @@ def _delete_line_from_file(f_name, line_to_delete):
         f.truncate()
 
 
+def _fetch_github_repo(url, output_dir):
+    """Fetch a zip of the github repo"""
+    response = requests.get(url)
+    if response.status_code != 200:
+        raise ValueError(
+            f"Failed to download the archive. HTTP status code:"
+            f"{response.status_code}"
+        )
+
+    with ZipFile(BytesIO(response.content)) as fh:
+        for member in fh.namelist():
+            # Skip directories
+            if member.endswith("/"):
+                continue
+            # there is an extra root folder f"{problem-name}-master" in the zip
+            # we want to remove when extracing
+            target_path = Path(output_dir) / "/".join(member.split("/")[1:])
+            target_path.parent.mkdir(exist_ok=True, parents=True)
+            with fh.open(member) as source, open(target_path, "wb") as target:
+                shutil.copyfileobj(source, target)
+
+
 def setup_ramp_kit_ramp_data(
     ramp_config,
     problem_name,
@@ -158,11 +181,11 @@ def setup_ramp_kit_ramp_data(
                 'it, you need to set "force=True".'
             )
         shutil.rmtree(problem_kit_path, ignore_errors=True)
-    ramp_kit_url = "https://github.com/ramp-kits/{}.git".format(problem_name)
-    kwargs = {}
-    if depth is not None:
-        kwargs["depth"] = depth
-    Repo.clone_from(ramp_kit_url, problem_kit_path, **kwargs)
+
+    ramp_kit_url = (
+        f"https://github.com/ramp-kits/{problem_name}/archive/refs/heads/master.zip"
+    )
+    _fetch_github_repo(ramp_kit_url, problem_kit_path)
 
     problem_data_path = ramp_config["ramp_data_dir"]
     if os.path.exists(problem_data_path):
@@ -172,8 +195,10 @@ def setup_ramp_kit_ramp_data(
                 'it, you need to set "force=True".'
             )
         shutil.rmtree(problem_data_path, ignore_errors=True)
-    ramp_data_url = "https://github.com/ramp-data/{}.git".format(problem_name)
-    Repo.clone_from(ramp_data_url, problem_data_path, **kwargs)
+    ramp_data_url = (
+        f"https://github.com/ramp-data/{problem_name}/archive/refs/heads/master.zip"
+    )
+    _fetch_github_repo(ramp_data_url, problem_data_path)
 
     current_directory = os.getcwd()
     os.chdir(problem_data_path)

ramp-database/ramp_database/tools/leaderboard.py

@@ -427,12 +427,19 @@ def get_leaderboard(
             columns = [
                 "team",
                 "submission",
+                "id",
                 "submitted at (UTC)",
                 "state",
                 "waiting list",
             ]
         else:
-            columns = ["team", "submission", "submitted at (UTC)", "error"]
+            columns = [
+                "team",
+                "submission",
+                "id",
+                "submitted at (UTC)",
+                "error",
+            ]
 
         # we rely on the zip function ignore the submission state if the error
         # column was not appended
@@ -444,6 +451,7 @@ def get_leaderboard(
                     [
                         sub.event_team.team.name,
                         sub.name_with_link,
+                        sub.id,
                         pd.Timestamp(sub.submission_timestamp),
                         (
                             sub.state_with_link
@@ -461,6 +469,13 @@ def get_leaderboard(
             for sub in submissions
         ]
         df = pd.DataFrame(data, columns=columns)
+        if leaderboard_type == "new":
+
+            def create_button(cell_value):
+                return f"<button onclick=\"alert(('You killed ' \
+                    + 'submission {cell_value}!'))\">{cell_value}</button>"
+
+            df["Stop"] = df["id"].apply(lambda x: create_button(x))
     else:
         # make some extra filtering
         submissions = [sub for sub in submissions if sub.is_public_leaderboard]

ramp-database/ramp_database/tools/tests/test_leaderboard.py

@@ -257,6 +257,7 @@ def test_get_leaderboard(session_toy_db):
     assert (
         """<th>team</th>
       <th>submission</th>
+      <th>id</th>
       <th>submitted at (UTC)</th>
       <th>error</th>"""
         in leaderboard_failed

ramp-engine/ramp_engine/remote.py

@@ -64,7 +64,26 @@ def _deserialize_folder(stream: bytes, out_dir: Union[str, Path]):
     with BytesIO(stream) as fh:
         fh.seek(0)
         with tarfile.open(fileobj=fh, mode="r:gz") as fh_tar:
-            fh_tar.extractall(out_dir)
+
+            def is_within_directory(directory, target):
+
+                abs_directory = os.path.abspath(directory)
+                abs_target = os.path.abspath(target)
+
+                prefix = os.path.commonprefix([abs_directory, abs_target])
+
+                return prefix == abs_directory
+
+            def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
+
+                for member in tar.getmembers():
+                    member_path = os.path.join(path, member.name)
+                    if not is_within_directory(path, member_path):
+                        raise Exception("Attempted Path Traversal in Tar File")
+
+                tar.extractall(path, members, numeric_owner=numeric_owner)
+
+            safe_extract(fh_tar, out_dir)
 
 
 def _remove_link_or_dir(path: str):

ramp-frontend/ramp_frontend/tests/test_ramp.py

@@ -476,6 +476,7 @@ def test_ask_for_event_mail(client_session):
                 # check that the email has been sent
                 # shutdown the threadpool and wait for the future (email) to be sent
                 client.application.pool.shutdown(wait=True)
+                pytest.xfail(reason="assert 2 == 1")
                 assert len(outbox) == 1
                 assert "User test_user asked to add a new event" in outbox[0].body
 

requirements.txt

@@ -11,7 +11,6 @@ Flask-Login==0.5.0
 Flask-Mail==0.9.1
 Flask-SQLAlchemy==2.5.1
 Flask-Wtf==1.0.0
-gitpython
 ipykernel
 jupyter
 numpy