From 05f197b336342eebed90ecb2f9d10194def971a0 Mon Sep 17 00:00:00 2001 From: Jeremy Nicklas Date: Fri, 30 Mar 2018 16:38:57 -0400 Subject: [PATCH] add shell path as script option Fixes #82 --- CHANGELOG.md | 2 ++ lib/ood_core/batch_connect/template.rb | 4 +++- lib/ood_core/job/adapters/lsf/helper.rb | 1 + lib/ood_core/job/adapters/pbspro.rb | 1 + lib/ood_core/job/adapters/slurm.rb | 9 ++++++++- lib/ood_core/job/adapters/torque.rb | 1 + lib/ood_core/job/script.rb | 16 ++++++++++++---- spec/job/adapters/lsf/helper_spec.rb | 4 ++++ spec/job/adapters/pbspro_spec.rb | 6 ++++++ spec/job/adapters/slurm_spec.rb | 6 ++++++ spec/job/adapters/torque_spec.rb | 6 ++++++ spec/job/script_spec.rb | 8 ++++++++ 12 files changed, 58 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 097109b58..a5884224e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Basic multi-cluster support for LSF by specifying name of cluster for -m argument. [#24](https://github.com/OSC/ood_core/issues/24) +- Added `OodCore::Job::Script#shell_path` as an option to all adapters. + [#82](https://github.com/OSC/ood_core/issues/82) ## [0.2.1] - 2018-01-26 ### Changed diff --git a/lib/ood_core/batch_connect/template.rb b/lib/ood_core/batch_connect/template.rb index b6cfd662b..1ca515156 100644 --- a/lib/ood_core/batch_connect/template.rb +++ b/lib/ood_core/batch_connect/template.rb @@ -58,7 +58,9 @@ def initialize(context = {}) # @return [String] rendered template def to_s <<-EOT.gsub(/^ {10}/, '') - #!/bin/bash + # + # Generated with ood_core v#{VERSION} + # #{script_wrapper} EOT diff --git a/lib/ood_core/job/adapters/lsf/helper.rb b/lib/ood_core/job/adapters/lsf/helper.rb index f6d302ee9..7555b0250 100644 --- a/lib/ood_core/job/adapters/lsf/helper.rb +++ b/lib/ood_core/job/adapters/lsf/helper.rb @@ -88,6 +88,7 @@ def batch_submit_args(script, after: [], afterok: [], afternotok: [], afterany: args += (script.rerunnable ? ["-r"] : ["-rn"]) unless script.rerunnable.nil? args += ["-b", script.start_time.localtime.strftime("%Y:%m:%d:%H:%M")] unless script.start_time.nil? args += ["-W", (script.wall_time / 60).to_i] unless script.wall_time.nil? + args += ["-L", script.shell_path.to_s] unless script.shell_path.nil? # input and output files args += ["-i", script.input_path] unless script.input_path.nil? diff --git a/lib/ood_core/job/adapters/pbspro.rb b/lib/ood_core/job/adapters/pbspro.rb index d27697203..2f120c327 100644 --- a/lib/ood_core/job/adapters/pbspro.rb +++ b/lib/ood_core/job/adapters/pbspro.rb @@ -227,6 +227,7 @@ def submit(script, after: [], afterok: [], afternotok: [], afterany: []) args += ["-m", "e"] end args += ["-N", script.job_name] unless script.job_name.nil? + args += ["-S", script.shell_path] unless script.shell_path.nil? # ignore input_path (not defined in PBS Pro) args += ["-o", script.output_path] unless script.output_path.nil? args += ["-e", script.error_path] unless script.error_path.nil? diff --git a/lib/ood_core/job/adapters/slurm.rb b/lib/ood_core/job/adapters/slurm.rb index 8fd278ac0..9e96c64bf 100644 --- a/lib/ood_core/job/adapters/slurm.rb +++ b/lib/ood_core/job/adapters/slurm.rb @@ -298,8 +298,15 @@ def submit(script, after: [], afterok: [], afternotok: [], afterany: []) # Set native options args += script.native if script.native + # Set content + content = if script.shell_path.nil? + script.content + else + "#!#{script.shell_path}\n#{script.content}" + end + # Submit job - @slurm.submit_string(script.content, args: args, env: env) + @slurm.submit_string(content, args: args, env: env) rescue Batch::Error => e raise JobAdapterError, e.message end diff --git a/lib/ood_core/job/adapters/torque.rb b/lib/ood_core/job/adapters/torque.rb index 1fe83cfc1..fb90163b5 100644 --- a/lib/ood_core/job/adapters/torque.rb +++ b/lib/ood_core/job/adapters/torque.rb @@ -83,6 +83,7 @@ def submit(script, after: [], afterok: [], afternotok: [], afterany: []) mail_points += 'e' if script.email_on_terminated headers.merge!(Mail_Points: mail_points) unless mail_points.empty? headers.merge!(Job_Name: script.job_name) unless script.job_name.nil? + headers.merge!(Shell_Path_List: script.shell_path) unless script.shell_path.nil? # ignore input_path (not defined in Torque) headers.merge!(Output_Path: script.output_path) unless script.output_path.nil? headers.merge!(Error_Path: script.error_path) unless script.error_path.nil? diff --git a/lib/ood_core/job/script.rb b/lib/ood_core/job/script.rb index be8b0f113..21eeb1147 100644 --- a/lib/ood_core/job/script.rb +++ b/lib/ood_core/job/script.rb @@ -54,6 +54,10 @@ class Script # @return [String, nil] name of job attr_reader :job_name + # Path to file specifying the login shell of the job + # @return [Pathname, nil] file path specifying login shell + attr_reader :shell_path + # Path to file specifying the input stream of the job # @return [Pathname, nil] file path specifying input stream attr_reader :input_path @@ -106,6 +110,8 @@ class Script # @param email_on_started [Boolean, nil] whether email when job starts # @param email_on_terminated [Boolean, nil] whether email when job ends # @param job_name [#to_s, nil] name of job + # @param shell_path [#to_s, nil] file path specifying login shell + # @param error_path [#to_s, nil] file path specifying error stream # @param input_path [#to_s, nil] file path specifying input stream # @param output_path [#to_s, nil] file path specifying output stream # @param error_path [#to_s, nil] file path specifying error stream @@ -119,10 +125,10 @@ class Script def initialize(content:, args: nil, submit_as_hold: nil, rerunnable: nil, job_environment: nil, workdir: nil, email: nil, email_on_started: nil, email_on_terminated: nil, - job_name: nil, input_path: nil, output_path: nil, - error_path: nil, reservation_id: nil, queue_name: nil, - priority: nil, start_time: nil, wall_time: nil, - accounting_id: nil, native: nil, **_) + job_name: nil, shell_path: nil, input_path: nil, + output_path: nil, error_path: nil, reservation_id: nil, + queue_name: nil, priority: nil, start_time: nil, + wall_time: nil, accounting_id: nil, native: nil, **_) @content = content.to_s @submit_as_hold = submit_as_hold @@ -135,6 +141,7 @@ def initialize(content:, args: nil, submit_as_hold: nil, rerunnable: nil, @workdir = workdir && Pathname.new(workdir.to_s) @email = email && Array.wrap(email).map(&:to_s) @job_name = job_name && job_name.to_s + @shell_path = shell_path && Pathname.new(shell_path.to_s) @input_path = input_path && Pathname.new(input_path.to_s) @output_path = output_path && Pathname.new(output_path.to_s) @error_path = error_path && Pathname.new(error_path.to_s) @@ -161,6 +168,7 @@ def to_h email_on_started: email_on_started, email_on_terminated: email_on_terminated, job_name: job_name, + shell_path: shell_path, input_path: input_path, output_path: output_path, error_path: error_path, diff --git a/spec/job/adapters/lsf/helper_spec.rb b/spec/job/adapters/lsf/helper_spec.rb index 99365e3fd..ba7ca6b8e 100644 --- a/spec/job/adapters/lsf/helper_spec.rb +++ b/spec/job/adapters/lsf/helper_spec.rb @@ -146,6 +146,10 @@ def args_for(attrs = {}) expect(args_for(job_name: "my_job")).to eq({args: ["-J", "my_job"], env: {}}) end + it "with :shell_path" do + expect(args_for(shell_path: "/path/to/shell")).to eq({args: ["-L", "/path/to/shell"], env: {}}) + end + it "with :workdir" do expect(args_for(workdir: "/path/to/workdir")).to eq({args: ["-cwd", "/path/to/workdir"], env: {}}) end diff --git a/spec/job/adapters/pbspro_spec.rb b/spec/job/adapters/pbspro_spec.rb index 664f922b4..a76097b28 100644 --- a/spec/job/adapters/pbspro_spec.rb +++ b/spec/job/adapters/pbspro_spec.rb @@ -151,6 +151,12 @@ def build_script(opts = {}) it { expect(pbspro).to have_received(:submit_string).with(content, args: ["-N", "my_job", "-j", "oe"], chdir: nil) } end + context "with :shell_path" do + before { adapter.submit(build_script(shell_path: "/path/to/shell")) } + + it { expect(pbspro).to have_received(:submit_string).with(content, args: ["-S", Pathname.new("/path/to/shell"), "-j", "oe"], chdir: nil) } + end + context "with :input_path" do before { adapter.submit(build_script(input_path: "/path/to/input")) } diff --git a/spec/job/adapters/slurm_spec.rb b/spec/job/adapters/slurm_spec.rb index 0138acdb7..b5bd350a4 100644 --- a/spec/job/adapters/slurm_spec.rb +++ b/spec/job/adapters/slurm_spec.rb @@ -149,6 +149,12 @@ def build_script(opts = {}) it { expect(slurm).to have_received(:submit_string).with(content, args: ["-J", "my_job"], env: {}) } end + context "with :shell_path" do + before { adapter.submit(build_script(shell_path: "/path/to/shell")) } + + it { expect(slurm).to have_received(:submit_string).with("#!/path/to/shell\n#{content}", args: [], env: {}) } + end + context "with :input_path" do before { adapter.submit(build_script(input_path: "/path/to/input")) } diff --git a/spec/job/adapters/torque_spec.rb b/spec/job/adapters/torque_spec.rb index 4ec8b725e..8b81fc992 100644 --- a/spec/job/adapters/torque_spec.rb +++ b/spec/job/adapters/torque_spec.rb @@ -150,6 +150,12 @@ def build_script(opts = {}) it { expect(pbs).to have_received(:submit_string).with(content, queue: nil, headers: {Join_Path: "oe", Job_Name: "my_job"}, resources: {}, envvars: {}) } end + context "with :shell_path" do + before { adapter.submit(build_script(shell_path: "/path/to/shell")) } + + it { expect(pbs).to have_received(:submit_string).with(content, queue: nil, headers: {Join_Path: "oe", Shell_Path_List: Pathname.new("/path/to/shell")}, resources: {}, envvars: {}) } + end + context "with :input_path" do before { adapter.submit(build_script(input_path: "/path/to/input")) } diff --git a/spec/job/script_spec.rb b/spec/job/script_spec.rb index 2a0ab5f26..4ebb6d1ba 100644 --- a/spec/job/script_spec.rb +++ b/spec/job/script_spec.rb @@ -25,6 +25,7 @@ def build_script(opts = {}) it { is_expected.to respond_to(:email_on_started) } it { is_expected.to respond_to(:email_on_terminated) } it { is_expected.to respond_to(:job_name) } + it { is_expected.to respond_to(:shell_path) } it { is_expected.to respond_to(:input_path) } it { is_expected.to respond_to(:output_path) } it { is_expected.to respond_to(:error_path) } @@ -115,6 +116,12 @@ def build_script(opts = {}) it { is_expected.to eq("my_job") } end + describe "#shell_path" do + subject { build_script(shell_path: double(to_s: "/path/to/shell")).shell_path } + + it { is_expected.to eq(Pathname.new("/path/to/shell")) } + end + describe "#input_path" do subject { build_script(input_path: double(to_s: "/path/to/input")).input_path } @@ -189,6 +196,7 @@ def build_script(opts = {}) it { is_expected.to have_key(:email_on_started) } it { is_expected.to have_key(:email_on_terminated) } it { is_expected.to have_key(:job_name) } + it { is_expected.to have_key(:shell_path) } it { is_expected.to have_key(:input_path) } it { is_expected.to have_key(:output_path) } it { is_expected.to have_key(:error_path) }