Skip to content
CI

always include-tools when using large_pex #310

Sign in to view logs

always include-tools when using large_pex

always include-tools when using large_pex #310

Workflow file for this run

.github/workflows/main.yml at d15eaef
name: CI
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
lint:
runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.6
uses: actions/setup-python@v2
with:
python-version: 3.6
- name: Install dependencies
run: |
pip install --upgrade pip
pip install .
pip install -r tests-requirements.txt
- name: Linter
run: |
pylama
- name: Typer checker
run: |
mypy --config-file setup.cfg
build:
runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
pip install --upgrade pip
pip install -e .
pip install -r tests-requirements.txt
- name: Tests
run: |
pytest -m "not hadoop and not conda" -s tests
standalone_spark3_with_S3:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Build spark-docker
run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=3.2.2 --build-arg PYTHON_VERSION=3.9.15
- name: Build the docker-compose stack
run: |
export PYTHON_VERSION=3.9
docker-compose -f ./examples/spark-with-S3/docker-compose.yml up -d
- name: Check running containers
run: docker ps -a
- name: Run spark Job
run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.9 3.2.2
standalone_spark2_with_S3:
runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
steps:
- uses: actions/checkout@v2
- name: Build spark-docker
run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=2.4.2 --build-arg PYTHON_VERSION=3.6.9
- name: Build the docker-compose stack
run: |
export PYTHON_VERSION=3.6
docker-compose -f ./examples/spark-with-S3/docker-compose.yml up -d
- name: Check running containers
run: docker ps -a
- name: Run spark Job
run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.6 2.4.2
# hadoop_hdfs:
# runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
# steps:
# - uses: actions/checkout@v2
# - name: Set up Python 3.6
# uses: actions/setup-python@v2
# with:
# python-version: 3.6
# - name: Install hadoop-test-cluster
# run: |
# pip install hadoop-test-cluster
# - name: Start cluster
# run: |
# htcluster startup --image cdh5 --mount .:cluster-pack
# - name: Start Job
# run: |
# # for the hack with script .. see https://github.com/actions/runner/issues/241#issuecomment-577360161
# # the prebuild image only contains a conda install, we also install python 3.6.10
# # to avoid sharing files on the worker node we copy the python3.6 install script via hdfs to worker /tmp folder
# script -e -c "htcluster exec -u root -s edge -- chown -R testuser /home/testuser && \
# htcluster exec -u root -s edge -- /home/testuser/cluster-pack/tests/integration/install_python.sh && \
# htcluster exec -u root -s edge -- hdfs dfs -put /home/testuser/cluster-pack/tests/integration/install_python.sh hdfs:///tmp && \
# htcluster exec -u root -s worker -- hdfs dfs -get hdfs:///tmp/install_python.sh /home/testuser && \
# htcluster exec -u root -s worker -- chmod +x /home/testuser/install_python.sh && \
# htcluster exec -u root -s worker -- /home/testuser/install_python.sh && \
# htcluster exec -s edge -- /home/testuser/cluster-pack/tests/integration/hadoop_hdfs_tests.sh"
conda:
runs-on: ubuntu-20.04 # Latest version supporting Python 3.6
strategy:
matrix:
python-version: [3.6, 3.9]
steps:
- uses: actions/checkout@v2
- name: Run tests with conda
run: |
conda update -y conda
conda create -n venv -y python=${{ matrix.python-version }}
# https://github.com/conda/conda/issues/7980
eval "$(conda shell.bash hook)"
conda activate venv
pip install .
pip install -r tests-requirements.txt
pytest -m conda -s tests --log-cli-level=INFO