Merge branch 'nextflow-io:master' into master

VERSION

@@ -1 +1 @@
-24.07.0-edge
+24.08.0-edge

changelog.txt

@@ -1,10 +1,30 @@
 NEXTFLOW CHANGE-LOG
 ===================
+24.08.0-edge - 4 Sep 2024
+- Add Google Batch warning when for conflicting disk image config (#5279) [96cb57cb]
+- Add support for Google Batch used specified boot images  (#5268) [0aaa6482]
+- Disable AWS spot retry (#5215) [f28fcb25]
+- Disable Google Batch automatic spot retries (#5223) [aad21533]
+- Disable automatic detection of virtual threads (#5270) [b3ba2c2d]
+- Fix  missing .command.env when eval is used and task runs on a cloud env [4a6b54aa]
+- Fix job array syntax for PBS/Torque executor (#5281) [d59f5fae]
+- Fix k8s client status cond is possible null in podState (#5264) [46672415]
+- Fix non-determinist behaviour of join operator with bag array as key (#5189) [e7dc0d69]
+- Fix stage retry on corrupted HTTP downloads  (#5275) [bf0cd326]
+- Support Azure Managed Identities in Fusion configuration logic (#5278) [a0bf8b40]
+- Use public.cr.seqera.io in place of AWS ECR [5a01f277]
+- Wave client logs improvement [5a37e617]
+- Bump amazoncorretto:21-al2023 [59aed581]
+- Bump nf-wave@1.5.1 [97c4e08f]
+- Bump nf-google@1.15.0 [24133f2a]
+- Bump nf-azure@1.9.0 [29f49ba7]
+- Bump nf-amazon@2.8.0 [bbc3adca]
+
 24.07.0-edge - 8 Aug 2024
 - Add runtime error for missing channel factory (#5170) [1f9210ab]
 - Apply k8s.cpuLimits to kuberun driver pod (#5160) [4300adf1]
 - Await build completion for all Wave containers [2b8117e9]
-- Deprecate module addParams() and params() (#5200) [ci fast] [82c97f8c]
+- Deprecate module addParams() and params() (#5200) [82c97f8c]
 - Remove capsule launcher dependencies (#3395) [f15e4246]
 - Fix AWS Cloudwatch access when using custom log group name [30195838]
 - Fix Invalid AWS Fargate CPUs usage error reporting [d9c50e59]
@@ -61,7 +81,7 @@ NEXTFLOW CHANGE-LOG
 - Bump jgit 6.10.0 [4cf6b9f7]
 
 24.04.3 - 9 Jul 2024
-- Add ability to override failOnError setting default via env variable (#5117) [ci fast] [6852429c]
+- Add ability to override failOnError setting default via env variable (#5117) [6852429c]
 - Fix normalization of consecutive slashes in uri path (#5114) [3f366b7e]
 - Fix executions hangs on finalisation exception (#5070) [4c207c23]
 - Bump nf-google@1.13.2-patch1 [55ec5ec5]

docker/Makefile

@@ -24,14 +24,10 @@ build-arm: dist/docker/arm64
 	docker buildx build --platform linux/arm64 --output=type=docker --progress=plain --tag nextflow/nextflow:${version} --build-arg TARGETPLATFORM=linux/arm64 .
 
 release: build
-	docker tag nextflow/nextflow:${version} nextflow/nextflow:latest
 	docker push nextflow/nextflow:${version}
-	docker push nextflow/nextflow:latest
 	#
-	docker tag nextflow/nextflow:${version} public.ecr.aws/seqera-labs/nextflow:${version}
-	docker tag nextflow/nextflow:${version} public.ecr.aws/seqera-labs/nextflow:latest
-	docker push public.ecr.aws/seqera-labs/nextflow:${version}
-	docker push public.ecr.aws/seqera-labs/nextflow:latest
+	docker tag nextflow/nextflow:${version} public.cr.seqera.io/nextflow/nextflow:${version}
+	docker push public.cr.seqera.io/nextflow/nextflow:${version}
 
 #Static builds can now be found at:
 #

docs/google.md

@@ -156,7 +156,7 @@ process myTask {
 
 :::{note}
 Using an instance template will overwrite the `accelerator` and `disk` directives, as well as the following Google Batch
-config options: `cpuPlatform`, `preemptible`, and `spot`.
+config options: `bootDiskImage`, `cpuPlatform`, `preemptible`, and `spot`.
 :::
 
 To use an instance template with GPUs, you must also set the `google.batch.installGpuDrivers` config option to `true`.

modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy

@@ -46,7 +46,7 @@ class PbsExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
 
         if( task instanceof TaskArrayRun ) {
             final arraySize = task.getArraySize()
-            result << '-J' << "0-${arraySize - 1}".toString()
+            result << '-t' << "0-${arraySize - 1}".toString()
         }
 
         result << '-N' << getJobNameFor(task)
@@ -188,7 +188,7 @@ class PbsExecutor extends AbstractGridExecutor implements TaskArrayExecutor {
 
     @Override
     String getArrayIndexName() {
-        return 'PBS_ARRAY_INDEX'
+        return 'PBS_ARRAYID'
     }
 
     @Override

modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy

@@ -129,4 +129,9 @@ class PbsProExecutor extends PbsExecutor {
         DECODE_STATUS.get(status)
     }
 
+    @Override
+    String getArrayIndexName() {
+        return 'PBS_ARRAY_INDEX'
+    }
+
 }

modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy

@@ -63,53 +63,66 @@ class ArrayBag<E> implements Bag<E>, List<E>, KryoSerializable {
         InvokerHelper.inspect(this)
     }
 
-//    E getAt( int index )  {
-//        target.get(index)
-//    }
-//
-//    void putAt( int index, E value ) {
-//        target.set(index, value)
-//    }
-//
-//    @Override
-//    int hashCode() {
-//        int h = 0;
-//        Iterator<E> i = target.iterator();
-//        while (i.hasNext()) {
-//            E obj = i.next();
-//            if (obj != null)
-//                h += obj.hashCode();
-//        }
-//        return h;
-//    }
-//
-//    @Override
-//    boolean equals(Object o) {
-//        if ( o.is(this) )
-//            return true;
-//
-//        if (!(o instanceof ArrayBag))
-//            return false;
-//
-//        Collection other = ((ArrayBag)o).target
-//        if (other.size() != target.size())
-//            return false;
-//
-//        try {
-//            return target.containsAll(other);
-//        }
-//        catch (ClassCastException unused)   {
-//            return false;
-//        }
-//        catch (NullPointerException unused) {
-//            return false;
-//        }
-//    }
+    @Override
+    int hashCode() {
+        int h = 0;
+        Iterator<E> i = target.iterator();
+        while (i.hasNext()) {
+            E obj = i.next();
+            if (obj != null)
+                h += obj.hashCode();
+        }
+        return h;
+    }
+
+    /**
+     * NOTE!!! this method implements an equality is NOT used when invoking `equals` method
+     * or using `==` operator from Groovy code. This because the Groovy runtime implements its
+     * own equality logic both for {@link Map} and {@link Collection} logic.
+     *
+     * See
+     * https://issues.apache.org/jira/browse/GROOVY-9003
+     *
+     * However this is still applied when equality is checked by Java compiled code e.g. Java SDK.
+     * For this reason is necessary to implement the expected equals (and hashCode) semantic by `join`
+     * (and other) operators.
+     *
+     * See issue
+     * https://github.com/nextflow-io/nextflow/issues/5187
+     *
+     * @return
+     *      {@code true} is the content of the bag is equals to another bag irrespective the elements order,
+     *      {@code false} otherwise
+     */
+    @Override
+    boolean equals(Object o) {
+        if ( o.is(this) )
+            return true;
+
+        if (!(o instanceof ArrayBag))
+            return false;
 
+        Collection other = ((ArrayBag)o).target
+        if (other.size() != target.size())
+            return false;
+
+        try {
+            return target.containsAll(other);
+        }
+        catch (ClassCastException unused)   {
+            return false;
+        }
+        catch (NullPointerException unused) {
+            return false;
+        }
+    }
+
+    @Override
     void read (Kryo kryo, Input input) {
         target = kryo.readObject(input,ArrayList)
     }
 
+    @Override
     void write (Kryo kryo, Output output) {
         kryo.writeObject(output, target)
     }

modules/nextflow/src/main/resources/META-INF/build-info.properties

@@ -1,4 +1,4 @@
-build=5921
-version=24.07.0-edge
-timestamp=1722884291592
-commitId=e8d643c2f
+build=5922
+version=24.08.0-edge
+timestamp=1725461287713
+commitId=97c4e08f3

modules/nextflow/src/main/resources/META-INF/plugins-info.txt

@@ -1,8 +1,8 @@
-nf-amazon@2.7.0
-nf-azure@1.8.1
+nf-amazon@2.8.0
+nf-azure@1.9.0
 nf-cloudcache@0.4.2
 nf-codecommit@0.2.2
 nf-console@1.1.4
-nf-google@1.14.0
+nf-google@1.15.0
 nf-tower@1.9.2
-nf-wave@1.5.0
+nf-wave@1.5.1

modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy

@@ -172,7 +172,7 @@ class PbsExecutorTest extends Specification {
         }
         then:
         executor.getHeaders(taskArray) == '''
-                #PBS -J 0-4
+                #PBS -t 0-4
                 #PBS -N nf-task_name
                 #PBS -o /dev/null
                 #PBS -j oe
@@ -321,7 +321,7 @@ class PbsExecutorTest extends Specification {
         given:
         def executor = Spy(PbsExecutor)
         expect:
-        executor.getArrayIndexName() == 'PBS_ARRAY_INDEX'
+        executor.getArrayIndexName() == 'PBS_ARRAYID'
         executor.getArrayIndexStart() == 0
     }
 

modules/nextflow/src/test/groovy/nextflow/extension/JoinOpTest.groovy

@@ -21,7 +21,9 @@ import nextflow.Channel
 import nextflow.Global
 import nextflow.Session
 import nextflow.exception.AbortOperationException
+import nextflow.util.ArrayBag
 import spock.lang.Specification
+
 /**
  *
  * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
@@ -207,6 +209,35 @@ class JoinOpTest extends Specification {
 
     }
 
+    def 'should be able to use identical ArrayBags join key' () {
+        given:
+        def key1 = new ArrayBag(["key"])
+        def key2 = new ArrayBag(["key"])
+        def ch1 = Channel.of([key1, "foo"])
+        def ch2 = Channel.of([key2, "bar"])
+
+        when:
+        def op = new JoinOp(ch1 as DataflowReadChannel, ch2 as DataflowReadChannel)
+        List result = op.apply().toList().getVal()
+
+        then:
+        !result.isEmpty()
+    }
+
+    def 'should differentiate nonidentical ArrayBags join key' () {
+        given:
+        def key1 = new ArrayBag(["key", "key", "quay"])
+        def key2 = new ArrayBag(["quay", "quay", "key"])
+        def ch1 = Channel.of([key1, "foo"])
+        def ch2 = Channel.of([key2, "bar"])
+
+        when:
+        def op = new JoinOp(ch1 as DataflowReadChannel, ch2 as DataflowReadChannel)
+        List result = op.apply().toList().getVal()
+
+        then:
+        result.isEmpty()
+    }
 
     def 'should not fail on mismatches' () {
         given:

modules/nextflow/src/test/groovy/nextflow/util/ArrayBagTest.groovy

@@ -74,4 +74,35 @@ class ArrayBagTest extends Specification {
         String.valueOf(bag) == '[1, 2, 3]'
     }
 
+    def 'hashCode should be invariant to order' () {
+        given:
+        def bag1 = new ArrayBag([1,2,3])
+        def bag2 = new ArrayBag([3,1,2])
+        def bag3 = new ArrayBag([4,1,2])
+
+        expect:
+        bag1.hashCode() == bag2.hashCode()
+        bag1.hashCode() != bag3.hashCode()
+
+        /**
+         * NOTE!!! equality cannot be checked due to groovy overriding the equals implementation
+         * see {@link ArrayBag#equals(java.lang.Object)}
+         */
+    }
+
+    def 'should access map entry using bag as key' () {
+        given:
+        def bag1 = new ArrayBag([1,2,3])
+        def bag2 = new ArrayBag([3,1,2])
+        def bag3 = new ArrayBag([4,1,2])
+        and:
+        def map = [(bag1):'foo']
+
+        expect:
+        map.get(bag1) == 'foo'
+        map.get(bag2) == 'foo'
+        map.get(bag3) == null
+
+    }
+
 }

nextflow

@@ -15,7 +15,7 @@
 #  limitations under the License.
 
 [[ "$NXF_DEBUG" == 'x' ]] && set -x
-NXF_VER=${NXF_VER:-'24.07.0-edge'}
+NXF_VER=${NXF_VER:-'24.08.0-edge'}
 NXF_ORG=${NXF_ORG:-'nextflow-io'}
 NXF_HOME=${NXF_HOME:-$HOME/.nextflow}
 NXF_PROT=${NXF_PROT:-'https'}

nextflow.md5

@@ -1 +1 @@
-de590cf568c61fa941e4013d7f407902
+09e28262a4fc4ebc1ffbccbe9ab7678d

nextflow.sha1

@@ -1 +1 @@
-3cc9902cd2713262f5fb29d652b749ef5676b3ed
+52be33b40ba95be215f2927a5fc6975ec475fbee

nextflow.sha256

@@ -1 +1 @@
-19515441f324cd32e272352b056d800293b639e1bfcc455257b4192115029831
+dc03db53bd7c3692c2e76af34fb850cbbbb9f15ee3d2b3c7c4dcd79559880144

plugins/nf-amazon/changelog.txt

@@ -1,5 +1,8 @@
 nf-amazon changelog
 ===================
+2.8.0 - 4 Sep 2024
+- Disable AWS spot retry (#5215) [f28fcb25]
+
 2.7.0 - 5 Aug 2024
 - More robust parsing of shm-size containerOptions (#5177) [b56802a3]
 - Fix AWS Cloudwatch access when using custom log group name [30195838]

plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF

@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.cloud.aws.AmazonPlugin
 Plugin-Id: nf-amazon
-Plugin-Version: 2.7.0
+Plugin-Version: 2.8.0
 Plugin-Provider: Seqera Labs
 Plugin-Requires: >=24.04.4

plugins/nf-azure/changelog.txt

@@ -1,5 +1,8 @@
 nf-azure changelog
 ===================
+1.9.0 - 4 Sep 2024
+- Support Azure Managed Identities in Fusion configuration logic (#5278) [a0bf8b40]
+
 1.8.1 - 5 Aug 2024
 - Bump pf4j to version 3.12.0 [96117b9a]
 

plugins/nf-azure/src/resources/META-INF/MANIFEST.MF

@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.cloud.azure.AzurePlugin
 Plugin-Id: nf-azure
-Plugin-Version: 1.8.1
+Plugin-Version: 1.9.0
 Plugin-Provider: Seqera Labs
 Plugin-Requires: >=24.04.4

plugins/nf-google/changelog.txt

@@ -1,5 +1,10 @@
 nf-google changelog
 ===================
+1.15.0 - 4 Sep 2024
+- Add Google Batch warning when for conflicting disk image config (#5279) [ci fast] [96cb57cb]
+- Add support for Google Batch used specified boot images  (#5268) [0aaa6482]
+- Disable Google Batch automatic spot retries (#5223) [aad21533]
+
 1.14.0 - 5 Aug 2024
 - Bump pf4j to version 3.12.0 [96117b9a]
 - Make Google Batch auto retry codes configurable (#5148) [e562ce06]