diff --git a/VERSION b/VERSION index d4c0237907..2607bd405f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.07.0-edge +24.08.0-edge diff --git a/changelog.txt b/changelog.txt index a77bcf5a44..e2323b311e 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,10 +1,30 @@ NEXTFLOW CHANGE-LOG =================== +24.08.0-edge - 4 Sep 2024 +- Add Google Batch warning when for conflicting disk image config (#5279) [96cb57cb] +- Add support for Google Batch used specified boot images (#5268) [0aaa6482] +- Disable AWS spot retry (#5215) [f28fcb25] +- Disable Google Batch automatic spot retries (#5223) [aad21533] +- Disable automatic detection of virtual threads (#5270) [b3ba2c2d] +- Fix missing .command.env when eval is used and task runs on a cloud env [4a6b54aa] +- Fix job array syntax for PBS/Torque executor (#5281) [d59f5fae] +- Fix k8s client status cond is possible null in podState (#5264) [46672415] +- Fix non-determinist behaviour of join operator with bag array as key (#5189) [e7dc0d69] +- Fix stage retry on corrupted HTTP downloads (#5275) [bf0cd326] +- Support Azure Managed Identities in Fusion configuration logic (#5278) [a0bf8b40] +- Use public.cr.seqera.io in place of AWS ECR [5a01f277] +- Wave client logs improvement [5a37e617] +- Bump amazoncorretto:21-al2023 [59aed581] +- Bump nf-wave@1.5.1 [97c4e08f] +- Bump nf-google@1.15.0 [24133f2a] +- Bump nf-azure@1.9.0 [29f49ba7] +- Bump nf-amazon@2.8.0 [bbc3adca] + 24.07.0-edge - 8 Aug 2024 - Add runtime error for missing channel factory (#5170) [1f9210ab] - Apply k8s.cpuLimits to kuberun driver pod (#5160) [4300adf1] - Await build completion for all Wave containers [2b8117e9] -- Deprecate module addParams() and params() (#5200) [ci fast] [82c97f8c] +- Deprecate module addParams() and params() (#5200) [82c97f8c] - Remove capsule launcher dependencies (#3395) [f15e4246] - Fix AWS Cloudwatch access when using custom log group name [30195838] - Fix Invalid AWS Fargate CPUs usage error reporting [d9c50e59] @@ -61,7 +81,7 @@ NEXTFLOW CHANGE-LOG - Bump jgit 6.10.0 [4cf6b9f7] 24.04.3 - 9 Jul 2024 -- Add ability to override failOnError setting default via env variable (#5117) [ci fast] [6852429c] +- Add ability to override failOnError setting default via env variable (#5117) [6852429c] - Fix normalization of consecutive slashes in uri path (#5114) [3f366b7e] - Fix executions hangs on finalisation exception (#5070) [4c207c23] - Bump nf-google@1.13.2-patch1 [55ec5ec5] diff --git a/docker/Makefile b/docker/Makefile index 894e568577..e65294f7fe 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -24,14 +24,10 @@ build-arm: dist/docker/arm64 docker buildx build --platform linux/arm64 --output=type=docker --progress=plain --tag nextflow/nextflow:${version} --build-arg TARGETPLATFORM=linux/arm64 . release: build - docker tag nextflow/nextflow:${version} nextflow/nextflow:latest docker push nextflow/nextflow:${version} - docker push nextflow/nextflow:latest # - docker tag nextflow/nextflow:${version} public.ecr.aws/seqera-labs/nextflow:${version} - docker tag nextflow/nextflow:${version} public.ecr.aws/seqera-labs/nextflow:latest - docker push public.ecr.aws/seqera-labs/nextflow:${version} - docker push public.ecr.aws/seqera-labs/nextflow:latest + docker tag nextflow/nextflow:${version} public.cr.seqera.io/nextflow/nextflow:${version} + docker push public.cr.seqera.io/nextflow/nextflow:${version} #Static builds can now be found at: # diff --git a/docs/google.md b/docs/google.md index 232fcc29fd..30bd8d4f84 100644 --- a/docs/google.md +++ b/docs/google.md @@ -156,7 +156,7 @@ process myTask { :::{note} Using an instance template will overwrite the `accelerator` and `disk` directives, as well as the following Google Batch -config options: `cpuPlatform`, `preemptible`, and `spot`. +config options: `bootDiskImage`, `cpuPlatform`, `preemptible`, and `spot`. ::: To use an instance template with GPUs, you must also set the `google.batch.installGpuDrivers` config option to `true`. diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy index 8e968f99c7..b25f020648 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/PbsExecutor.groovy @@ -46,7 +46,7 @@ class PbsExecutor extends AbstractGridExecutor implements TaskArrayExecutor { if( task instanceof TaskArrayRun ) { final arraySize = task.getArraySize() - result << '-J' << "0-${arraySize - 1}".toString() + result << '-t' << "0-${arraySize - 1}".toString() } result << '-N' << getJobNameFor(task) @@ -188,7 +188,7 @@ class PbsExecutor extends AbstractGridExecutor implements TaskArrayExecutor { @Override String getArrayIndexName() { - return 'PBS_ARRAY_INDEX' + return 'PBS_ARRAYID' } @Override diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy index c171657224..bcb0cbfa84 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/PbsProExecutor.groovy @@ -129,4 +129,9 @@ class PbsProExecutor extends PbsExecutor { DECODE_STATUS.get(status) } + @Override + String getArrayIndexName() { + return 'PBS_ARRAY_INDEX' + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy b/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy index 64e849085c..bd7dcff3b8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/util/ArrayBag.groovy @@ -63,53 +63,66 @@ class ArrayBag implements Bag, List, KryoSerializable { InvokerHelper.inspect(this) } -// E getAt( int index ) { -// target.get(index) -// } -// -// void putAt( int index, E value ) { -// target.set(index, value) -// } -// -// @Override -// int hashCode() { -// int h = 0; -// Iterator i = target.iterator(); -// while (i.hasNext()) { -// E obj = i.next(); -// if (obj != null) -// h += obj.hashCode(); -// } -// return h; -// } -// -// @Override -// boolean equals(Object o) { -// if ( o.is(this) ) -// return true; -// -// if (!(o instanceof ArrayBag)) -// return false; -// -// Collection other = ((ArrayBag)o).target -// if (other.size() != target.size()) -// return false; -// -// try { -// return target.containsAll(other); -// } -// catch (ClassCastException unused) { -// return false; -// } -// catch (NullPointerException unused) { -// return false; -// } -// } + @Override + int hashCode() { + int h = 0; + Iterator i = target.iterator(); + while (i.hasNext()) { + E obj = i.next(); + if (obj != null) + h += obj.hashCode(); + } + return h; + } + + /** + * NOTE!!! this method implements an equality is NOT used when invoking `equals` method + * or using `==` operator from Groovy code. This because the Groovy runtime implements its + * own equality logic both for {@link Map} and {@link Collection} logic. + * + * See + * https://issues.apache.org/jira/browse/GROOVY-9003 + * + * However this is still applied when equality is checked by Java compiled code e.g. Java SDK. + * For this reason is necessary to implement the expected equals (and hashCode) semantic by `join` + * (and other) operators. + * + * See issue + * https://github.com/nextflow-io/nextflow/issues/5187 + * + * @return + * {@code true} is the content of the bag is equals to another bag irrespective the elements order, + * {@code false} otherwise + */ + @Override + boolean equals(Object o) { + if ( o.is(this) ) + return true; + + if (!(o instanceof ArrayBag)) + return false; + Collection other = ((ArrayBag)o).target + if (other.size() != target.size()) + return false; + + try { + return target.containsAll(other); + } + catch (ClassCastException unused) { + return false; + } + catch (NullPointerException unused) { + return false; + } + } + + @Override void read (Kryo kryo, Input input) { target = kryo.readObject(input,ArrayList) } + @Override void write (Kryo kryo, Output output) { kryo.writeObject(output, target) } diff --git a/modules/nextflow/src/main/resources/META-INF/build-info.properties b/modules/nextflow/src/main/resources/META-INF/build-info.properties index 4365a054ab..fd89d26b7f 100644 --- a/modules/nextflow/src/main/resources/META-INF/build-info.properties +++ b/modules/nextflow/src/main/resources/META-INF/build-info.properties @@ -1,4 +1,4 @@ -build=5921 -version=24.07.0-edge -timestamp=1722884291592 -commitId=e8d643c2f +build=5922 +version=24.08.0-edge +timestamp=1725461287713 +commitId=97c4e08f3 diff --git a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt index f76e2fd995..39dc609f48 100644 --- a/modules/nextflow/src/main/resources/META-INF/plugins-info.txt +++ b/modules/nextflow/src/main/resources/META-INF/plugins-info.txt @@ -1,8 +1,8 @@ -nf-amazon@2.7.0 -nf-azure@1.8.1 +nf-amazon@2.8.0 +nf-azure@1.9.0 nf-cloudcache@0.4.2 nf-codecommit@0.2.2 nf-console@1.1.4 -nf-google@1.14.0 +nf-google@1.15.0 nf-tower@1.9.2 -nf-wave@1.5.0 \ No newline at end of file +nf-wave@1.5.1 \ No newline at end of file diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy index b279c7eee2..174a7b3b12 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/PbsExecutorTest.groovy @@ -172,7 +172,7 @@ class PbsExecutorTest extends Specification { } then: executor.getHeaders(taskArray) == ''' - #PBS -J 0-4 + #PBS -t 0-4 #PBS -N nf-task_name #PBS -o /dev/null #PBS -j oe @@ -321,7 +321,7 @@ class PbsExecutorTest extends Specification { given: def executor = Spy(PbsExecutor) expect: - executor.getArrayIndexName() == 'PBS_ARRAY_INDEX' + executor.getArrayIndexName() == 'PBS_ARRAYID' executor.getArrayIndexStart() == 0 } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/JoinOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/JoinOpTest.groovy index 9893d9a797..673686fdf9 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/JoinOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/JoinOpTest.groovy @@ -21,7 +21,9 @@ import nextflow.Channel import nextflow.Global import nextflow.Session import nextflow.exception.AbortOperationException +import nextflow.util.ArrayBag import spock.lang.Specification + /** * * @author Paolo Di Tommaso @@ -207,6 +209,35 @@ class JoinOpTest extends Specification { } + def 'should be able to use identical ArrayBags join key' () { + given: + def key1 = new ArrayBag(["key"]) + def key2 = new ArrayBag(["key"]) + def ch1 = Channel.of([key1, "foo"]) + def ch2 = Channel.of([key2, "bar"]) + + when: + def op = new JoinOp(ch1 as DataflowReadChannel, ch2 as DataflowReadChannel) + List result = op.apply().toList().getVal() + + then: + !result.isEmpty() + } + + def 'should differentiate nonidentical ArrayBags join key' () { + given: + def key1 = new ArrayBag(["key", "key", "quay"]) + def key2 = new ArrayBag(["quay", "quay", "key"]) + def ch1 = Channel.of([key1, "foo"]) + def ch2 = Channel.of([key2, "bar"]) + + when: + def op = new JoinOp(ch1 as DataflowReadChannel, ch2 as DataflowReadChannel) + List result = op.apply().toList().getVal() + + then: + result.isEmpty() + } def 'should not fail on mismatches' () { given: diff --git a/modules/nextflow/src/test/groovy/nextflow/util/ArrayBagTest.groovy b/modules/nextflow/src/test/groovy/nextflow/util/ArrayBagTest.groovy index 0eb91521a6..b4e0b8efb4 100644 --- a/modules/nextflow/src/test/groovy/nextflow/util/ArrayBagTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/util/ArrayBagTest.groovy @@ -74,4 +74,35 @@ class ArrayBagTest extends Specification { String.valueOf(bag) == '[1, 2, 3]' } + def 'hashCode should be invariant to order' () { + given: + def bag1 = new ArrayBag([1,2,3]) + def bag2 = new ArrayBag([3,1,2]) + def bag3 = new ArrayBag([4,1,2]) + + expect: + bag1.hashCode() == bag2.hashCode() + bag1.hashCode() != bag3.hashCode() + + /** + * NOTE!!! equality cannot be checked due to groovy overriding the equals implementation + * see {@link ArrayBag#equals(java.lang.Object)} + */ + } + + def 'should access map entry using bag as key' () { + given: + def bag1 = new ArrayBag([1,2,3]) + def bag2 = new ArrayBag([3,1,2]) + def bag3 = new ArrayBag([4,1,2]) + and: + def map = [(bag1):'foo'] + + expect: + map.get(bag1) == 'foo' + map.get(bag2) == 'foo' + map.get(bag3) == null + + } + } diff --git a/nextflow b/nextflow index 293452710b..5a830d760c 100755 --- a/nextflow +++ b/nextflow @@ -15,7 +15,7 @@ # limitations under the License. [[ "$NXF_DEBUG" == 'x' ]] && set -x -NXF_VER=${NXF_VER:-'24.07.0-edge'} +NXF_VER=${NXF_VER:-'24.08.0-edge'} NXF_ORG=${NXF_ORG:-'nextflow-io'} NXF_HOME=${NXF_HOME:-$HOME/.nextflow} NXF_PROT=${NXF_PROT:-'https'} diff --git a/nextflow.md5 b/nextflow.md5 index 763bb14a5c..f84d9f76d9 100644 --- a/nextflow.md5 +++ b/nextflow.md5 @@ -1 +1 @@ -de590cf568c61fa941e4013d7f407902 +09e28262a4fc4ebc1ffbccbe9ab7678d diff --git a/nextflow.sha1 b/nextflow.sha1 index 36b7629f1e..5de50736aa 100644 --- a/nextflow.sha1 +++ b/nextflow.sha1 @@ -1 +1 @@ -3cc9902cd2713262f5fb29d652b749ef5676b3ed +52be33b40ba95be215f2927a5fc6975ec475fbee diff --git a/nextflow.sha256 b/nextflow.sha256 index b5f27a0a76..aaa74b964c 100644 --- a/nextflow.sha256 +++ b/nextflow.sha256 @@ -1 +1 @@ -19515441f324cd32e272352b056d800293b639e1bfcc455257b4192115029831 +dc03db53bd7c3692c2e76af34fb850cbbbb9f15ee3d2b3c7c4dcd79559880144 diff --git a/plugins/nf-amazon/changelog.txt b/plugins/nf-amazon/changelog.txt index 911ac7d8f1..f153a7d3f9 100644 --- a/plugins/nf-amazon/changelog.txt +++ b/plugins/nf-amazon/changelog.txt @@ -1,5 +1,8 @@ nf-amazon changelog =================== +2.8.0 - 4 Sep 2024 +- Disable AWS spot retry (#5215) [f28fcb25] + 2.7.0 - 5 Aug 2024 - More robust parsing of shm-size containerOptions (#5177) [b56802a3] - Fix AWS Cloudwatch access when using custom log group name [30195838] diff --git a/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF b/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF index 1e34af703f..d97464362f 100644 --- a/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-amazon/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.cloud.aws.AmazonPlugin Plugin-Id: nf-amazon -Plugin-Version: 2.7.0 +Plugin-Version: 2.8.0 Plugin-Provider: Seqera Labs Plugin-Requires: >=24.04.4 diff --git a/plugins/nf-azure/changelog.txt b/plugins/nf-azure/changelog.txt index 3fba12494f..17cfc997d4 100644 --- a/plugins/nf-azure/changelog.txt +++ b/plugins/nf-azure/changelog.txt @@ -1,5 +1,8 @@ nf-azure changelog =================== +1.9.0 - 4 Sep 2024 +- Support Azure Managed Identities in Fusion configuration logic (#5278) [a0bf8b40] + 1.8.1 - 5 Aug 2024 - Bump pf4j to version 3.12.0 [96117b9a] diff --git a/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF b/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF index 939940fc14..a653c0076b 100644 --- a/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-azure/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.cloud.azure.AzurePlugin Plugin-Id: nf-azure -Plugin-Version: 1.8.1 +Plugin-Version: 1.9.0 Plugin-Provider: Seqera Labs Plugin-Requires: >=24.04.4 diff --git a/plugins/nf-google/changelog.txt b/plugins/nf-google/changelog.txt index 5b8a54a4dc..3d4d356ecd 100644 --- a/plugins/nf-google/changelog.txt +++ b/plugins/nf-google/changelog.txt @@ -1,5 +1,10 @@ nf-google changelog =================== +1.15.0 - 4 Sep 2024 +- Add Google Batch warning when for conflicting disk image config (#5279) [ci fast] [96cb57cb] +- Add support for Google Batch used specified boot images (#5268) [0aaa6482] +- Disable Google Batch automatic spot retries (#5223) [aad21533] + 1.14.0 - 5 Aug 2024 - Bump pf4j to version 3.12.0 [96117b9a] - Make Google Batch auto retry codes configurable (#5148) [e562ce06] diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy index b7a33cafed..389f2247b6 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy @@ -303,6 +303,9 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask { if( task.config.getDisk() ) log.warn1 'Process directive `disk` ignored because an instance template was specified' + if( executor.config.getBootDiskImage() ) + log.warn1 'Config option `google.batch.bootDiskImage` ignored because an instance template was specified' + if( executor.config.cpuPlatform ) log.warn1 'Config option `google.batch.cpuPlatform` ignored because an instance template was specified' diff --git a/plugins/nf-google/src/resources/META-INF/MANIFEST.MF b/plugins/nf-google/src/resources/META-INF/MANIFEST.MF index 92494dcf38..d7c884312b 100644 --- a/plugins/nf-google/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-google/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.cloud.google.GoogleCloudPlugin Plugin-Id: nf-google -Plugin-Version: 1.14.0 +Plugin-Version: 1.15.0 Plugin-Provider: Seqera Labs Plugin-Requires: >=24.04.4 diff --git a/plugins/nf-wave/changelog.txt b/plugins/nf-wave/changelog.txt index 57b257b5ef..880c9d89ed 100644 --- a/plugins/nf-wave/changelog.txt +++ b/plugins/nf-wave/changelog.txt @@ -1,5 +1,8 @@ nf-wave changelog ================== +1.5.1 - 4 Sep 2024 +- Wave client logs improvement [5a37e617] + 1.5.0 - 5 Aug 2024 - Await build completion for all Wave containers [2b8117e9] - Bump pf4j to version 3.12.0 [96117b9a] diff --git a/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF b/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF index 696026aa07..472417014b 100644 --- a/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-wave/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: io.seqera.wave.plugin.WavePlugin Plugin-Id: nf-wave -Plugin-Version: 1.5.0 +Plugin-Version: 1.5.1 Plugin-Provider: Seqera Labs Plugin-Requires: >=24.04.4