submit.yml.erb

<%-
  cores = num_cores.to_i

  if cores == 0 && cluster == "pitzer"
    # little optimization for pitzer nodes. They want the whole node, if they chose 'any',
    # it can be scheduled on p18 or p20 nodes. If not, they'll get the constraint below.
    base_slurm_args = ["--nodes", "1", "--exclusive"]
  elsif cores == 0
    # full node on owens
    cores = 28
    base_slurm_args = ["--nodes", "1", "--ntasks-per-node", "28"]
  else
    base_slurm_args = ["--nodes", "1", "--ntasks-per-node", "#{cores}"]
  end

  slurm_args = case node_type
              when "gpu-40core"
                base_slurm_args + ["--constraint", "40core"]
              when "gpu-48core"
                base_slurm_args + ["--constraint", "48core"]
              when "any-40core"
                base_slurm_args + ["--constraint", "40core"]
              when "any-48core"
                base_slurm_args + ["--constraint", "48core"]
              when "hugemem"
                base_slurm_args + ["--partition", "hugemem"]
              when "largemem"
                base_slurm_args + ["--partition", "largemem"]
              when "debug"
                base_slurm_args += ["--partition", "debug"]
              else
                base_slurm_args
              end

-%>
<% if cluster =~ /owens|pitzer|ascend|cardinal/ -%>
---
batch_connect:
  template: "basic"
  conn_params:
    - jupyter_api
script:
<% if node_type =~ /gpu/ -%>
  gpus_per_node: 1
<% end -%>
  native:
    <%- slurm_args.each do |arg| %>
    - "<%= arg %>"
    <%- end %>
<% elsif cluster =~ /kubernetes/
   if node_type =~ /owens/
     compute_cluster = "owens"
     apps_path = "/usr/local"
     # Memory per core with hyperthreading enabled
     memory_mb = num_cores.to_i * 2200
   elsif node_type =~ /pitzer/
     compute_cluster = "pitzer"
     apps_path = "/apps"
     # Memory per core with hyperthreading enabled
     memory_mb = num_cores.to_i * 4000
   end
   mounts = {
     'home'    => OodSupport::User.new.home,
     'support' => OodSupport::User.new('support').home,
     'project' => '/fs/project',
     'scratch' => '/fs/scratch',
     'ess'     => '/fs/ess',
   }
-%>
---
script:
  wall_time: "<%= bc_num_hours.to_i * 3600 %>"
  <%- if node_type =~ /gpu/ -%>
  gpus_per_node: 1
  <%- end -%>
  native:
    container:
      name: "jupyter"
      image: "docker-registry.osc.edu/ondemand/ondemand-base-rhel7:0.10.0"
      image_pull_policy: "IfNotPresent"
      command: ["/bin/bash","-l","<%= staged_root %>/job_script_content.sh"]
      restart_policy: 'OnFailure'
      env:
        NB_UID: "<%= Etc.getpwnam(ENV['USER']).uid %>"
        NB_USER: "<%= ENV['USER'] %>"
        NB_GID: "<%= Etc.getpwnam(ENV['USER']).gid %>"
        CLUSTER: "<%= compute_cluster %>"
        KUBECONFIG: "/dev/null"
      labels:
        osc.edu/cluster: "<%= compute_cluster %>"
      port: "8080"
      cpu: "<%= num_cores %>"
      memory: "<%= memory_mb %>Mi"
    mounts:
    <%- mounts.each_pair do |name, mount| -%>
      - type: host
        name: <%= name %>
        host_type: Directory
        path: <%= mount %>
        destination_path: <%= mount %>
    <%- end -%>
      - type: host
        name: munge-socket
        host_type: Socket
        path: /var/run/munge/munge.socket.2
        destination_path: /var/run/munge/munge.socket.2
      - type: host
        name: slurm-conf
        host_type: Directory
        path: /etc/slurm
        destination_path: /etc/slurm
      - type: host
        name: sssd-pipes
        host_type: Directory
        path: /var/lib/sss/pipes
        destination_path: /var/lib/sss/pipes
      - type: host
        name: sssd-conf
        host_type: Directory
        path: /etc/sssd
        destination_path: /etc/sssd
      - type: host
        name: nsswitch
        host_type: File
        path: /etc/nsswitch.conf
        destination_path: /etc/nsswitch.conf
      - type: host
        name: lmod-init
        host_type: File
        path: /apps/<%= compute_cluster %>/lmod/lmod.sh
        destination_path: /etc/profile.d/lmod.sh
      - type: host
        name: intel
        host_type: Directory
        path: /nfsroot/<%= compute_cluster %>/opt/intel
        destination_path: /opt/intel
      - type: host
        name: apps
        host_type: Directory
        path: /apps/<%= compute_cluster %>
        destination_path: <%= apps_path %>
    node_selector:
      node-role.kubernetes.io/ondemand: ''
<% end -%>