Merge pull request #1017 from maxplanck-ie/snakepipes_pytest

Real-data pytests

.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
  docs:
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - uses: mamba-org/setup-micromamba@main
  with:
  environment-file: .github/snakePipesEnvCI.yml
@@ -34,7 +34,7 @@ jobs:
  flake:
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - uses: mamba-org/setup-micromamba@main
  with:
  environment-file: .github/snakePipesEnvCI.yml
@@ -56,7 +56,7 @@ jobs:
  CI:
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - uses: mamba-org/setup-micromamba@main
  with:
  environment-file: .github/snakePipesEnvCI.yml
@@ -75,11 +75,6 @@ jobs:
  micromamba activate snakePipes_CI
  snakePipes config --tempDir /tmp
  ./.ci_stuff/test_dag.sh
- - name: pytest
- run: |
- micromamba activate snakePipes_CI
- snakePipes config --tempDir /tmp
- pytest --verbosity=2
  createEnvs:
  needs: CI
  strategy:

.github/workflows/osx.yml

@@ -35,7 +35,7 @@ jobs:
  ]
  runs-on: macos-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - uses: mamba-org/setup-micromamba@main
  with:
  environment-file: .github/snakePipesEnvCI.yml

.github/workflows/pytest.yml

@@ -0,0 +1,72 @@
+name: pytest
+
+on: [pull_request, push]
+
+defaults:
+ run:
+ shell: bash -l {0}
+
+jobs:
+ test_mRNA:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: mamba-org/setup-micromamba@main
+ with:
+ environment-file: snakePipes/shared/rules/envs/rna_seq.yaml
+ environment-name: rnaseqenv
+ condarc: |
+ channels:
+ - conda-forge
+ - bioconda
+ - defaults
+ channel_priority: 'strict'
+ cache-downloads: true
+ post-cleanup: 'none'
+ - name: create_starix
+ run: |
+ micromamba activate rnaseqenv
+ gunzip -c tests/data/genomes/genome_chr17.fa.gz > genome_chr17.fa
+ gunzip -c tests/data/genomes/genes_chr17.gtf.gz > genes_chr17.gtf
+ STAR --runThreadN 4 --runMode genomeGenerate --genomeDir tests/data/mRNA_STAR --genomeFastaFiles genome_chr17.fa --sjdbGTFfile genes_chr17.gtf --sjdbOverhang 100 --genomeSAindexNbases 12
+ - uses: mamba-org/setup-micromamba@main
+ with:
+ environment-file: .github/snakePipesEnvCI.yml
+ condarc: |
+ channels:
+ - conda-forge
+ - bioconda
+ - defaults
+ channel_priority: 'strict'
+ cache-downloads: true
+ - name: Install snakePipes
+ run: |
+ micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
+ - name: pytest
+ run: |
+ micromamba activate snakePipes_CI
+ snakePipes config --tempDir /tmp --condaEnvDir ./
+ snakePipes createEnvs --only CONDA_SHARED_ENV CONDA_RNASEQ_ENV
+ pytest --verbosity=2 -rP tests/test_mRNA.py
+ CI_jobcounts:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: mamba-org/setup-micromamba@main
+ with:
+ environment-file: .github/snakePipesEnvCI.yml
+ condarc: |
+ channels:
+ - conda-forge
+ - bioconda
+ - defaults
+ channel_priority: 'strict'
+ cache-downloads: true
+ - name: Install snakePipes
+ run: |
+ micromamba run -n snakePipes_CI python -m pip install . --no-deps --ignore-installed -vvv
+ - name: pytest
+ run: |
+ micromamba activate snakePipes_CI
+ snakePipes config --tempDir /tmp --condaEnvDir ./
+ pytest --verbosity=2 -rP tests/test_jobcounts.py

.gitignore

@@ -30,3 +30,7 @@ snakePipes.egg-info
 
 # misc
 .vscode/
+
+# tests
+*fa
+*gtf

tests/data/cluster_config.yaml

@@ -0,0 +1 @@
+snakemake_cluster_cmd: ''

tests/test_mRNA.py

@@ -0,0 +1,87 @@
+from pathlib import Path
+import subprocess as sp
+from ruamel.yaml import YAML
+import shutil
+import gzip
+import pytest
+
+def extract_gz(i, o):
+ '''
+ extracts i into file o
+ '''
+ with gzip.open(i, 'rb') as f:
+ with open(o, 'wb') as of:
+ shutil.copyfileobj(f, of)
+
+def createTestData(fp):
+ '''
+ Sets up fasta, gtf and organism yaml in a factory (fp)
+ '''
+ # uncompress fna into tmp factory
+ fnagz = Path('tests') / 'data' / 'genomes' / 'genome_chr17.fa.gz'
+ faout = fp / 'genome.fa'
+ extract_gz(fnagz, faout)
+ # uncompress gtf into tmp factory
+ gtfgz = Path('tests') / 'data' / 'genomes' / 'genes_chr17.gtf.gz'
+ gtfout = fp / 'genes.gtf'
+ extract_gz(gtfgz, gtfout)
+ # Path to STAR index (created in action)
+ STARpath = Path('tests') / 'data' / 'mRNA_STAR'
+ STARpath = STARpath.resolve()
+
+ orgyaml = {
+ "genome_size": 94987271 , #we can also extract genome size from STARindex output
+ "genome_fasta": faout.as_posix(),
+ "star_index": STARpath.as_posix(),
+ "genes_gtf" : gtfout.as_posix(),
+ "extended_coding_regions_gtf" : "",
+ "blacklist_bed": "",
+ "ignoreForNormalization": "" 
+ }
+ # set up yaml
+ yaml = YAML()
+ yaml.boolean_representation = ['False', 'True']
+ with open(fp / 'org.yaml', 'w') as of:
+ yaml.dump(orgyaml, of)
+
+@pytest.fixture(scope='session')
+def ifs(tmp_path_factory):
+ fp = tmp_path_factory.mktemp("data")
+ createTestData(fp)
+ return fp
+
+class TestmRNAseq:
+ def test_mrna(self, ifs):
+ org = ifs / 'org.yaml'
+ clusterconfig = Path('tests') / 'data' / 'cluster_config.yaml'
+ sp.run(
+ [
+ 'mRNA-seq',
+ '-i',
+ Path('tests') / 'data' / 'mRNA_mIFNB',
+ '-o',
+ 'test_mrna',
+ '--clusterConfig',
+ clusterconfig,
+ org
+ ]
+ )
+ assert Path('test_mrna/mRNA-seq_snakePipes.done').is_file() == True
+
+ def test_mrna4(self, ifs):
+ org = ifs / 'org.yaml'
+ clusterconfig = Path('tests') / 'data' / 'cluster_config.yaml'
+ sp.run(
+ [
+ 'mRNA-seq',
+ '-i',
+ Path('tests') / 'data' / 'mRNA_BcellPancreas',
+ '-o',
+ 'test_mrna_4sample',
+ '--clusterConfig',
+ clusterconfig,
+ org
+ ]
+ )
+ assert Path('test_mrna_4sample/mRNA-seq_snakePipes.done').is_file() == True
+