compute_build_base_img.yml

---

- hosts: localhost

  vars:
    compute_base_image: "Featured-RockyLinux8"
    sec_group_global: "{{ ansible_facts.hostname }}-ssh-global"
    sec_group_internal: "{{ ansible_facts.hostname }}-internal"
    compute_base_size: "m3.tiny"
    network_name: "{{ ansible_facts.hostname }}-elastic-net"
    JS_ssh_keyname: "{{ ansible_facts.hostname }}-slurm-key"
    openstack_cloud: "openstack"

  vars_files:
    - clouds.yaml

  tasks:

  - name: build compute base instance
    os_server:
      timeout: 300
      state: present
      name: "compute-{{ ansible_facts.hostname }}-base-instance"
      cloud: "{{ openstack_cloud }}"
      image: "{{ compute_base_image }}"
      key_name: "{{ JS_ssh_keyname }}"
      security_groups: "{{ sec_group_global }},{{ sec_group_internal }}"
      flavor: "{{ compute_base_size }}"
      meta: { compute: "base" }
      auto_ip: "no"
      user_data: |
        #cloud-config
        packages: []
        package_update: false
        package_upgrade: false
        package_reboot_if_required: false
        final_message: "Boot completed in $UPTIME seconds"
      network: "{{ network_name }}"
      wait: yes
    register: "os_host"

  - debug:
      var: os_host

  - name: add compute instance to inventory
    add_host:
      name: "{{ os_host['openstack']['name'] }}"
      groups: "compute-base"
      ansible_host: "{{ os_host.openstack.private_v4 }}"

  - name: pause for ssh to come up
    pause:
      seconds: 90


- hosts: compute-base

  vars:
    compute_base_package_list:
      - "python3-libselinux"
      - "telnet"
      - "bind-utils"
      - "vim"
      - "openmpi4-gnu9-ohpc"
      - "ohpc-slurm-client"
      - "lmod-ohpc"
      - "ceph-common"
    packages_to_remove:
      - "environment-modules"
      - "containerd.io.x86_64"
      - "docker-ce.x86_64"
      - "docker-ce-cli.x86_64"
      - "docker-ce-rootless-extras.x86_64"
      - "Lmod"

  tasks:

  - name: Get the headnode private IP
    local_action:
      module: shell source /etc/slurm/openrc.sh && openstack server show $(hostname -s) | grep addresses | awk  -F'|' '{print $3}' | awk  -F'=' '{print $2}' | awk  -F',' '{print $1}'
    register: headnode_private_ip
    become: False # for running as slurm, since no sudo on localhost

  - name: Get the slurmctld uid
    local_action:
      module: shell getent passwd slurm | awk -F':' '{print $3}'
    register: headnode_slurm_uid
    become: False # for running as slurm, since no sudo on localhost

  - name: turn off the firewall
    service:
      name: firewalld
      state: stopped
      enabled: no

  - name: Add OpenHPC 2.0 repo
    dnf:
      name: "http://repos.openhpc.community/OpenHPC/2/CentOS_8/x86_64/ohpc-release-2-1.el8.x86_64.rpm"
      state: present
      lock_timeout: 900
      disable_gpg_check: yes


  - name: Enable CentOS PowerTools repo
    command: dnf config-manager --set-enabled powertools

  - name: Disable docker-ce repo
    command: dnf config-manager --set-disabled docker-ce-stable

  - name: remove env-modules and docker packages
    dnf:
      name: "{{ packages_to_remove }}"
      state: absent
      lock_timeout: 300

  # There is an issue in removing Lmod in early call. Seems like we need to run it twice
  - name: remove Lmod packages
    dnf:
      name: Lmod
      state: absent
      lock_timeout: 300

  - name: install basic packages
    dnf:
      name: "{{ compute_base_package_list }}"
      state: present
      lock_timeout: 300

  - name: fix slurm user uid
    user:
      name: slurm
      uid: "{{ headnode_slurm_uid.stdout}}"
      shell: "/sbin/nologin"
      home: "/etc/slurm"

  - name: create slurm spool directories
    file:
      path: /var/spool/slurm/ctld
      state: directory
      owner: slurm
      group: slurm
      mode: 0755
      recurse: yes

  - name: change ownership of slurm files
    file:
      path: "{{ item }}"
      owner: slurm
      group: slurm
    with_items:
      - "/var/spool/slurm"
      - "/var/spool/slurm/ctld"
#      - "/var/log/slurm_jobacct.log"

  - name: disable selinux
    selinux: state=permissive policy=targeted

 # - name: allow use_nfs_home_dirs
 #   seboolean: name=use_nfs_home_dirs state=yes persistent=yes

  - name: import /home on compute nodes
    lineinfile:
      dest: /etc/fstab
      line:  "{{ headnode_private_ip.stdout }}:/home  /home  nfs  defaults,nfsvers=4.0 0 0"
      state: present

  - name: ensure /opt/ohpc/pub exists
    file: path=/opt/ohpc/pub state=directory mode=777 recurse=yes

  - name: import /opt/ohpc/pub on compute nodes
    lineinfile:
      dest: /etc/fstab
      line:  "{{ headnode_private_ip.stdout }}:/opt/ohpc/pub  /opt/ohpc/pub  nfs  defaults,nfsvers=4.0 0 0"
      state: present

  - name: ensure /export exists
    file: path=/export state=directory mode=777

  - name: import /export on compute nodes
    lineinfile:
      dest: /etc/fstab
      line:  "{{ headnode_private_ip.stdout }}:/export  /export  nfs  defaults,nfsvers=4.0 0 0"
      state: present

  - name: fix sda1 mount in fstab
    lineinfile:
      dest: /etc/fstab
      regex: "/                       xfs     defaults"
      line: "/dev/sda1           /                       xfs     defaults  0 0"
      state: present

  - name: add local users to compute node
    script: /tmp/add_users.sh
    ignore_errors: True

  - name: copy munge key from headnode
    synchronize:
      mode: push
      src: /etc/munge/munge.key
      dest: /etc/munge/munge.key
      set_remote_user: no
      use_ssh_args: yes

  - name: fix perms on munge key
    file: 
      path: /etc/munge/munge.key
      owner: munge
      group: munge
      mode: 0600
 
  - name: copy slurm.conf from headnode
    synchronize:
      mode: push
      src: /etc/slurm/slurm.conf
      dest: /etc/slurm/slurm.conf
      set_remote_user: no
      use_ssh_args: yes
 
  - name: copy slurm_prolog.sh from headnode
    synchronize:
      mode: push
      src: /usr/local/sbin/slurm_prolog.sh
      dest: /usr/local/sbin/slurm_prolog.sh
      set_remote_user: no
      use_ssh_args: yes
 
  - name: enable munge
    service: name=munge.service enabled=yes 
 
  - name: enable slurmd
    service: name=slurmd enabled=yes

#cat /etc/systemd/system/multi-user.target.wants/slurmd.service
#[Unit]
#Description=Slurm node daemon
#After=network.target munge.service #CHANGING TO: network-online.target
#ConditionPathExists=/etc/slurm/slurm.conf
#
#[Service]
#Type=forking
#EnvironmentFile=-/etc/sysconfig/slurmd
#ExecStart=/usr/sbin/slurmd $SLURMD_OPTIONS
#ExecReload=/bin/kill -HUP $MAINPID
#PIDFile=/var/run/slurmd.pid
#KillMode=process
#LimitNOFILE=51200
#LimitMEMLOCK=infinity
#LimitSTACK=infinity
#Delegate=yes
#
#
#[Install]
#WantedBy=multi-user.target

  - name: change slurmd service "After" to sshd and remote filesystems
    command: sed -i 's/network.target/sshd.service remote-fs.target/' /usr/lib/systemd/system/slurmd.service

  - name: add slurmd service "Requires" of sshd and remote filesystems
    command: sed -i '/After=network/aRequires=sshd.service remote-fs.target' /usr/lib/systemd/system/slurmd.service

#  - name: mount -a on compute nodes
#    command: "mount -a"

- hosts: localhost

  vars_files:
    - clouds.yaml

  tasks:

  - name: create compute instance snapshot
    command: ./compute_take_snapshot.sh
      
# os_server no longer handles instance state correctly
#  - name: remove compute instance
#    os_server:
#      timeout: 200
#      state: absent
#      name: "compute-{{ inventory_hostname_short }}-base-instance"
#      cloud: "tacc"