Merge branch 'release/v3.0.4'

.travis.yml

@@ -14,7 +14,7 @@ before_install:
   - mv .cache ./repseqio/.cache
   - cd repseqio && mvn clean install -DskipTests -B && cd ..
   - mv ./repseqio/.cache .cache
-
+  - ./ensure-test-data.sh
 
 script:
   - mvn test -B && ./itests.sh test

CHANGELOG

@@ -1,4 +1,21 @@
 
+MiXCR 3.0.4 (15 Feb 2019)
+========================
+
+-- Fixed problem with possibly incorrect clone-to-alignment mapping and clonotype sorting. The 
+   problem manifests itself in rare cercumstances, with `-OseparateByV/J/C=true` (special thanks to 
+   Victor Greiff @GreiffLab for finding the bug and helping with testing)
+-- Fixes exception while reading alignments from `clna` files (thanks to Dr. Ma @JohnMCMa for 
+   finding the bug and providing the data to reproduce it)
+-- Integration with new MiLib abstraction for CLI, to automatically infer when reanalysis is nedded 
+   and when not (this mechanism extracts paramteres from the target files, and checks if current 
+   command line sets the same options, if the options are the same, mixcr skips the step, otherwize 
+   restarts the analysis)
+-- Report for the `assemble` command now contains chain distribution statistics
+-- `analyze` command now is case-insensitive for all options like `--starting-material`
+-- Fixed exception arising in rare cases with sequence imputation on the export step
+
+
 MiXCR 3.0.3 (18 Nov 2018)
 ========================
 

README.md

@@ -104,7 +104,7 @@ To build MiXCR from source:
 
 ## License
 
-Copyright (c) 2014-2015, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
+Copyright (c) 2014-2018, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
 (here and after addressed as Inventors)
 All Rights Reserved
 

doc/install.rst

@@ -17,8 +17,15 @@ To install MiXCR using Homebrew just type the following commands:
 ::
 
     brew tap milaboratory/all
+    brew install mixcr3
+
+To install the old MiXCR 2.x series you can use
+
+::
+
     brew install mixcr
 
+
 Installation on Mac OS X / Linux / FreeBSD from zip distribution
 ----------------------------------------------------------------
 

itests.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-set -e
-set -o pipefail
+set -eo pipefail
 
 # "Integration" tests for MiXCR
 # Test standard analysis pipeline results
@@ -49,6 +48,8 @@ case $os in
     ;;
 esac
 
+tests=("case1" "case2" "case3" "case4" "case5")
+
 create_standard_results=false
 run_tests=false
 while [[ $# > 0 ]]
@@ -62,6 +63,9 @@ do
         test)
             run_tests=true
         ;;
+        case*)
+            tests=("$key")
+        ;;
         *)
             echo "Unknown option $key";
             exit 1
@@ -75,6 +79,8 @@ mkdir ${dir}/test_target
 cp ${dir}/src/test/resources/sequences/*.fastq ${dir}/test_target/
 
 cd ${dir}/test_target/
+ln -s ../src/test/resources/sequences/big/CD4M1_test_R1.fastq.gz ${dir}/test_target/CD4M1_test_R1.fastq.gz
+ln -s ../src/test/resources/sequences/big/CD4M1_test_R2.fastq.gz ${dir}/test_target/CD4M1_test_R2.fastq.gz
 
 PATH=${dir}:${PATH}
 
@@ -86,7 +92,7 @@ function go_assemble {
   mixcr assemble -r $1.clns.report $1.vdjca $1.clns
   for c in TCR IG TRB TRA TRG TRD IGH IGL IGK ALL
   do
-    mixcr exportClones -c ${c} -s $1.clns $1.clns.${c}.txt
+    mixcr exportClones -c ${c} $1.clns $1.clns.${c}.txt
   done
 }
 
@@ -100,18 +106,33 @@ if [[ $create_standard_results == true ]]; then
   done
 fi
 
-# UseCase 1
+function run_test() {
+  cd ${dir}/test_target
+  echo "========================"
+  echo "Running: $1"
+  echo "========================"
+
+  if ../itests/${1}; then
+    echo "========================"
+    echo "$1 executed successfully"
+  else
+    echo "========================"
+    echo "$1 executed with error"
+    touch ${1}.error
+  fi
+  echo "========================"
+}
 
 if [[ $run_tests == true ]]; then
-  echo "Running test case 1"
-  mixcr align -s hs -OvParameters.geneFeatureToAlign=VGeneWithP -OsaveOriginalReads=true test_R1.fastq test_R2.fastq case1.vdjca
-  mixcr assemble case1.vdjca case1.clns
-
-  mixcr exportAlignments -nFeatureImputed VDJRegion -descrsR1 -descrsR2 case1.vdjca case1.alignments.txt
-
-  echo "Running test case 2"
-  mixcr analyze shotgun -f --species hs --contig-assembly --impute-germline-on-export --starting-material rna test_R1.fastq test_R2.fastq case2
+  for testName in ${tests[@]} ; do
+      run_test "${testName}.sh"
+  done
 
-  echo "Running test case 3"
-  mixcr analyze amplicon --receptor-type tra --impute-germline-on-export -s hs --starting-material rna --contig-assembly --5-end v-primers --3-end j-primers --adapters no-adapters test_R1.fastq test_R2.fastq case3
+  if ls ${dir}/test_target/*.error 1> /dev/null 2>&1; then
+    echo "There are tests with errors."
+    exit 1
+  else
+    echo "All tests finished successfully."
+    exit 0
+  fi
 fi

itests/case1.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+mixcr align -s hs -OvParameters.geneFeatureToAlign=VGeneWithP -OsaveOriginalReads=true test_R1.fastq test_R2.fastq case1.vdjca
+mixcr assemble case1.vdjca case1.clns

itests/case2.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+mixcr analyze shotgun -f --species hs --contig-assembly --impute-germline-on-export --starting-material rna test_R1.fastq test_R2.fastq case2

itests/case3.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+mixcr analyze amplicon --receptor-type tra --impute-germline-on-export -s hs --starting-material rna --contig-assembly --5-end v-primers --3-end j-primers --adapters no-adapters test_R1.fastq test_R2.fastq case3

itests/case4.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+# Checking generic pipeline with relatively big input files
+mixcr analyze amplicon --receptor-type tra --impute-germline-on-export -s hs --starting-material rna --contig-assembly --5-end v-primers --3-end j-primers --adapters adapters-present CD4M1_test_R1.fastq.gz CD4M1_test_R2.fastq.gz case4
+
+# Checking skip steps behaviour
+mixcr analyze amplicon --receptor-type tra --impute-germline-on-export -s hs --starting-material rna --contig-assembly --5-end v-primers --3-end j-primers --adapters adapters-present CD4M1_test_R1.fastq.gz CD4M1_test_R2.fastq.gz case4

itests/case5.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+gzip -dc CD4M1_test_R1.fastq.gz CD4M1_test_R1.fastq.gz | tr 'N' 'A' > case5_R1.fastq
+gzip -dc CD4M1_test_R2.fastq.gz CD4M1_test_R2.fastq.gz | tr 'N' 'A' > case5_R2.fastq
+
+mixcr analyze amplicon --assemble '-OseparateByC=true' --assemble '-OseparateByV=true' --assemble '-OseparateByJ=true' --impute-germline-on-export -s hs --starting-material rna --contig-assembly --5-end v-primers --3-end j-primers --adapters adapters-present case5_R1.fastq case5_R2.fastq case5
+#mixcr analyze amplicon --assemble '-OcloneClusteringParameters=null' --impute-germline-on-export -s hs --starting-material rna --contig-assembly --5-end v-primers --3-end j-primers --adapters adapters-present case5_R1.fastq case5_R2.fastq case5
+
+mixcr exportAlignments -f -readIds -cloneIdWithMappingType case5.clna case5.als.txt
+
+sort -t $'\t' --key=1 -n case5.als.txt | tail -n +2 > case5.als.sorted.txt
+
+lines=$(cat case5.als.sorted.txt | wc -l)
+
+head -n $((lines/2)) case5.als.sorted.txt | cut -f2 > case5.als.sorted.1.txt
+tail -n $((lines/2)) case5.als.sorted.txt | cut -f2 > case5.als.sorted.2.txt
+
+if cmp case5.als.sorted.1.txt case5.als.sorted.2.txt; then
+  echo "All good"
+else
+  echo "Results are different"
+  diff case5.als.sorted.1.txt case5.als.sorted.2.txt
+fi

pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 <!--
-  ~ Copyright (c) 2014-2018, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
+  ~ Copyright (c) 2014-2019, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
   ~ (here and after addressed as Inventors)
   ~ All Rights Reserved
   ~
@@ -33,26 +33,20 @@
 
     <groupId>com.milaboratory</groupId>
     <artifactId>mixcr</artifactId>
-    <version>3.0.3</version>
+    <version>3.0.4</version>
     <packaging>jar</packaging>
     <name>MiXCR</name>
 
-    <parent>
-        <groupId>org.sonatype.oss</groupId>
-        <artifactId>oss-parent</artifactId>
-        <version>7</version>
-    </parent>
-
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <milib.version>1.9</milib.version>
+        <milib.version>1.10</milib.version>
     </properties>
 
     <dependencies>
         <dependency>
             <groupId>io.repseq</groupId>
             <artifactId>repseqio</artifactId>
-            <version>1.3</version>
+            <version>1.3.1</version>
             <exclusions>
                 <exclusion>
                     <groupId>com.milaboratory</groupId>
@@ -189,13 +183,13 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.17</version>
+                <version>2.22.1</version>
             </plugin>
 
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
-                <version>2.2</version>
+                <version>3.1.1</version>
                 <executions>
                     <execution>
                         <goals>
@@ -208,11 +202,10 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
-                <version>2.3.2</version>
+                <version>3.8.0</version>
                 <configuration>
                     <source>1.8</source>
                     <target>1.8</target>
-                    <verbose>true</verbose>
                 </configuration>
             </plugin>
 
@@ -228,7 +221,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>2.4</version>
+                <version>3.2.1</version>
                 <executions>
                     <execution>
                         <phase>pre-integration-test</phase>

src/main/java/com/milaboratory/mixcr/assembler/AlignmentsMappingMerger.java

@@ -1,3 +1,32 @@
+/*
+ * Copyright (c) 2014-2019, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
+ * (here and after addressed as Inventors)
+ * All Rights Reserved
+ *
+ * Permission to use, copy, modify and distribute any part of this program for
+ * educational, research and non-profit purposes, by non-profit institutions
+ * only, without fee, and without a written agreement is hereby granted,
+ * provided that the above copyright notice, this paragraph and the following
+ * three paragraphs appear in all copies.
+ *
+ * Those desiring to incorporate this work into commercial products or use for
+ * commercial purposes should contact MiLaboratory LLC, which owns exclusive
+ * rights for distribution of this program for commercial purposes, using the
+ * following email address: licensing@milaboratory.com.
+ *
+ * IN NO EVENT SHALL THE INVENTORS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
+ * SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
+ * ARISING OUT OF THE USE OF THIS SOFTWARE, EVEN IF THE INVENTORS HAS BEEN
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE SOFTWARE PROVIDED HEREIN IS ON AN "AS IS" BASIS, AND THE INVENTORS HAS
+ * NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR
+ * MODIFICATIONS. THE INVENTORS MAKES NO REPRESENTATIONS AND EXTENDS NO
+ * WARRANTIES OF ANY KIND, EITHER IMPLIED OR EXPRESS, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A
+ * PARTICULAR PURPOSE, OR THAT THE USE OF THE SOFTWARE WILL NOT INFRINGE ANY
+ * PATENT, TRADEMARK OR OTHER RIGHTS.
+ */
 package com.milaboratory.mixcr.assembler;
 
 import cc.redberry.pipe.OutputPort;

src/main/java/com/milaboratory/mixcr/assembler/AlignmentsProvider.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2018, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
+ * Copyright (c) 2014-2019, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
  * (here and after addressed as Inventors)
  * All Rights Reserved
  *

src/main/java/com/milaboratory/mixcr/assembler/AssembledReadsPort.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2018, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
+ * Copyright (c) 2014-2019, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
  * (here and after addressed as Inventors)
  * All Rights Reserved
  *

src/main/java/com/milaboratory/mixcr/assembler/AssemblerEvent.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2018, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
+ * Copyright (c) 2014-2019, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
  * (here and after addressed as Inventors)
  * All Rights Reserved
  *

src/main/java/com/milaboratory/mixcr/assembler/AssemblerEventLogger.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2018, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
+ * Copyright (c) 2014-2019, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
  * (here and after addressed as Inventors)
  * All Rights Reserved
  *

src/main/java/com/milaboratory/mixcr/assembler/AssemblerUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2018, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
+ * Copyright (c) 2014-2019, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
  * (here and after addressed as Inventors)
  * All Rights Reserved
  *
@@ -42,6 +42,17 @@ public static long mappingVariantsCount(int numberOfBadPoints, int maxAllowedMut
         return variants;
     }
 
+    /**
+     * Calculates maximal number of registered mismatches for the number of possible mismatch positions, to control the
+     * probability of non-specific mapping in cases with big number of low quality positions. Maximal probability of
+     * random mapping is defined through the numberOfVariants parameter.
+     *
+     * Basically, if the target read has only one low quality position and you found the clonotype with the different
+     * nucleotide in exactly this position, it is not the same as if the sequence had all low quality positions and you
+     * found the clonotype differing in one of them. In the first case you had only 3 possible sequences to match with,
+     * and in the second case - 3*L, where L is the length of the clonal sequence. To address for this effect, MiXCR
+     * controls the number of possible matching sequence, instead of a raw number of allowed mismatches.
+     */
     public static class MappingThresholdCalculator {
         final int[] values;
         final int oneThreshold;
@@ -61,6 +72,13 @@ public MappingThresholdCalculator(long numberOfVariants, int maxN) {
             this.values = thresholds.toArray();
         }
 
+        /**
+         * Returns maximal number of allowed mismatches to keep the number of possible mapping variants lower then the
+         * value of numberOfVariants parameter, provided in constructor.
+         *
+         * @param N number of low quality positions
+         * @return
+         */
         public int getThreshold(int N) {
             if (N < values.length)
                 return values[N];

src/main/java/com/milaboratory/mixcr/assembler/CloneAccumulator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2018, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
+ * Copyright (c) 2014-2019, Bolotin Dmitry, Chudakov Dmitry, Shugay Mikhail
  * (here and after addressed as Inventors)
  * All Rights Reserved
  *