opx_smoketest.yaml

##### Ansible Playbook for Smoketest on Dell OPX images
##### (c) 2017 Dell Technologies

# Prerequisites: 
#  Install Ansible as described in http://docs.ansible.com/ansible/intro_installation.html
#  Prepare an OPX device, installing the desired target image and bringing up eth0 (Ansible will SSH to this interface)
#  Update the inventory file with IP address and SSH login credentials.
#  If running smoketest on a hardware platform, consider that the case SmokeTest_Show_Transceivers requires at least one QSFP/SFP+. If not available, just comment out this case (with #)
#  This playbook has been verified on Ansible v2.3.

# Usage:
#  Execute as follows: ansible-playbook opx_smoketest.yaml -i inventory.yaml --extra-vars "dut=baseHW"(dut variable is mentioned in opx_smoketest.yaml file under hosts). This line assumes that the playbook and inventory files are on the directory where you execute andible-playbook). Insetad of baseHW, write the device name defined in the inventory file. More information about inventory configuration in http://docs.ansible.com/ansible/intro_inventory.html.
#  In case of failure, Ansible aborts execution immediately. You can login and troubleshoot.
#  By default, Ansible will check host keys (Read 'Host key Checking' in http://docs.ansible.com/ansible/intro_getting_started.html). If you don't want to configure the key in your system, disable it as follows: 1) enter ' bash '  shell. 2) Disable host key checking with ' export ANSIBLE_HOST_KEY_CHECKING=False '. In order to configure the key, just ssh to the switch from your system, and ssh should ask to save the host key.
#  More information about configuring the ansible playbook in http://docs.ansible.com/ansible/playbooks.html

# Supported platforms:
#  Dell network device, either hardware or virtual running OPX software.
#  This playbook will test via SSH connection.



#-----------------------------------------------------------------------------------------
- hosts: localhost # This is not part of the test, just print the timestamp as a reference
  tasks:
    - name: INFO - Get timestamp 
      set_fact:
        test_started: "{{ ansible_date_time.date}} {{ ansible_date_time.time}}"
    - name: INFO - Test START 
      debug: 
        msg: "{{ test_started }}"
#-----------------------------------------------------------------------------------------



- hosts: "{{ dut }}"

  vars:
    test_int:       e101-001-0
    test_int1:      e101-002-0
    test_route1:    100.0.0.1
    test_route2:    20.0.0.0
    test_mac   :    00:11:22:33:44:55
    test_vlan  :    100
    test_prefix:    24
    test_cps_prefix: 32
    test_route2hex: 14000000
    test_via:       100.0.0.2
    test_mgmtint:   eth0



  tasks:


    - name: SEND - ifconfig {{test_mgmtint}}
      command: /sbin/ifconfig {{test_mgmtint}}
      become: true
      register: result_geteth0
      changed_when: False
    - name: INFO - Eth0 status
      debug:
        msg: "{{result_geteth0.stdout_lines}}"
    - name: SEND - route -n
      command: /sbin/route -n 
      become: true
      register: result_getroute
      changed_when: False
    - name: INFO - Route status 
      debug:
        msg: "{{result_getroute.stdout_lines}}"

    - name: SEND - systemctl -l status opx-cps
      command: /bin/systemctl -l status opx-cps
      become: true
      register: result_getcpsd
      changed_when: False
    - name: INFO - CPSD status
      debug:
        msg: "{{result_getcpsd.stdout_lines}}"
    - name: VERIFY - Check if CPSD is running
      assert:
        that: "'active (running)' in result_getcpsd.stdout_lines[2]"
        msg:  "Expected active (running) status for CPSD."

    - name: SEND - systemctl -l status redis-server
      command: /bin/systemctl -l status redis-server
      become: true
      register: result_getredis
      changed_when: False
    - name: INFO - REDIS status
      debug:
        msg: "{{result_getredis.stdout_lines}}"
    - name: VERIFY - Check if REDIS is running
      assert:
        that: "'active (running)' in result_getredis.stdout"
        msg:  "Expected active (running) status for REDISD."

    - name: SEND - systemctl -l status opx-pas
      command: /bin/systemctl -l status opx-pas
      become: true
      register: result_getpasd
      changed_when: False
    - name: INFO - PASD status
      debug:
        msg: "{{result_getpasd.stdout_lines}}"
    - name: VERIFY - Check if PASD is running
      assert:
        that: "'active (running)' in result_getpasd.stdout_lines[2]"
        msg:  "Expected active (running) status for PASD."


    - name: SEND - systemctl -l status opx-nas
      command: /bin/systemctl -l status opx-nas
      become: true
      register: result_getnasd
      changed_when: False
    - name: INFO - NASD status
      debug:
        msg: "{{result_getnasd.stdout_lines}}"
    - name: VERIFY - Check if NASD is running 
      assert:
        that: "'active (running)' in result_getnasd.stdout_lines[2]"
        msg:  "Expected active (running) status for NASD."

    - name: SEND - ifconfig -a | grep BROADCAST -c
      command: bash -c "ifconfig -a | grep BROADCAST -c"
      become: true
      register: result_getports
      changed_when: False
    - name: INFO - Number of ports present 
      debug:
        msg: "{{result_getports.stdout_lines}}"
    - name:  VERIFY - Check if Ports are present
      assert: 
        that: "0 <= {{result_getports.stdout_lines[0]|int}}"
        msg:  "Expected to see at least 1 port"

    - block:
      - name: SEND - cps_get_oid.py base-pas/entity. Check the PAS entity returning object. Failure here means CPS get on PAS not working. Debug below.
        command: /usr/bin/cps_get_oid.py -qua observed base-pas/entity
        become: true
        register: result_pas_getcps
        changed_when: False
      - name: INFO - If PAS not returning any object or keys means PAS service not working. Check debugs below
        debug:
          msg: "{{result_pas_getcps.stdout_lines}}"
      - name: VERIFY - Check if the PAS object or key exists. If result_pas_getcps does not have key or object,check the debug below to make sure PAS is running.
        assert:
          that: "'base-pas/entity' in result_pas_getcps.stdout"
          msg: "Key or object doesn't exists"
      rescue:
      - name: DEBUG - Collecting data because the Test Failed on last VERIFY step
        command: systemctl status opx-pas
        become: true
        register: result_debugStatus_pas_svc
        changed_when: False
      - name: INFO - Debug info collected
        debug:
          msg: "{{result_debugStatus_pas_svc.stdout_lines}}"
      - name: FAIL - Test Failed on last VERIFY step. Exit here, after collecting debug data above
        fail:
        when: "Key or Object not in cps get"

    - block:
      - name: SEND - cps_get_oid.py base-pas/chassis. Check the PAS entity returning object. Failure here means CPS get on PAS not working. Debug below.
        command:  /usr/bin/cps_get_oid.py -qua observed base-pas/chassis
        become: true
        register: result_pas_getcps
        changed_when: False
      - name: INFO - If PAS not returning any object or keys means PAS service not working. Check debugs below
        debug:
          msg: "{{result_pas_getcps.stdout_lines}}"
      - name: VERIFY - Check if the PAS object or key exists. If result_pas_getcps does not have key or object,check the debug below to make sure PAS is running.
        assert:
          that: "'base-pas/chassis/base_mac_addresses = ''' in result_pas_getcps.stdout"
          msg: "Key or object doesn't exists"
      rescue:
      - name: DEBUG - Collecting data because the Test Failed on last VERIFY step
        command: systemctl status opx-pas
        become: true
        register: result_debugStatus_pas_svc
        changed_when: False
      - name: INFO - Debug info collected
        debug:
          msg: "{{result_debugStatus_pas_svc.stdout_lines}}"
      - name: FAIL - Test Failed on last VERIFY step. Exit here, after collecting debug data above
        fail:
        when: "Key or Object not in cps get"

    - name: SEND - systemctl -l status opx-acl-init
      shell: /bin/systemctl status opx-acl-init 
      become: true
      ignore_errors: yes
      register: result_getacl
      changed_when: False
    - name: INFO - ACL INIT status
      debug:
        msg: "{{result_getacl.stdout_lines}}"
    - name: VERIFY - Check if ACL INIT is loaded and exited successfully
      assert:
        that: "'Loaded: loaded' in result_getacl.stdout"
        msg:  "Expected LOADED status for ACL INIT SERVICE."

    - name: SEND - systemctl -l status opx-qos-init
      command: /bin/systemctl -l status opx-qos-init
      become: true
      register: result_getqos
      changed_when: False
    - name: INFO - QoS INIT status
      debug:
        msg: "{{result_getqos.stdout_lines}}"
    - name: VERIFY - Check if QoS INIT is loaded and exited successfully
      assert:
        that: "'status=0/SUCCESS' in result_getqos.stdout"
        msg:  "Expected SUCCESS status for QoS INIT SERVICE."

    - name: SEND - Config static route in kernel, ignoring errors if the route was previously configured (ip link set dev {{test_int}} up; ip addr add {{test_route1}}/{{test_prefix}} dev {{test_int}}; sysctl -w net.ipv4.ip_forward=1; ip route add {{test_route2}}/{{test_prefix}} scope global nexthop via {{test_via}})
      command: bash -c "ip link set dev {{test_int}} up; ip addr add {{test_route1}}/{{test_prefix}} dev {{test_int}}; sysctl -w net.ipv4.ip_forward=1; ip route add {{test_route2}}/{{test_prefix}} scope global nexthop via {{test_via}}"
      become: true
      ignore_errors: yes
      register: result_krouteconfigout
      changed_when: False
    - name: INFO - Any messages printed during route configuration are shown here
      debug:
        msg: "{{result_krouteconfigout.stdout_lines}}"
    - name: SEND - ip route show {{test_route2}}/{{test_prefix}}
      command: ip route show {{test_route2}}/{{test_prefix}}
      register: result_getkroute
      changed_when: False
    - name: INFO - Route in kernel
      debug:
        msg: "{{result_getkroute.stdout_lines}}"
    - name: SEND - route -n
      command: /sbin/route -n
      become: true
      register: result_getroute2
      changed_when: False
    - name: INFO - Route status
      debug:
        msg: "{{result_getroute2.stdout_lines}}"
    - name: VERIFY - Check if kernel route is '{{test_route2}}/{{test_prefix}} via {{test_via}} dev {{test_int}}' (ip route show {{test_route2}}/{{test_prefix}}) [SmokeTest_Forwarding]
      assert:
        that: "'{{test_route2}}/{{test_prefix}} via {{test_via}} dev {{test_int}}' in result_getkroute.stdout_lines[0]"
        msg:  "'{{test_route2}}/{{test_prefix}} via {{test_via}} dev {{test_int}}' is not set."

    - block:
      - name: SEND - cps_get_oid.py base-route/obj/entry base-route/obj/entry/af=2 . A failure here could mean that the command didn't return any result (either bug or misconfig). Errors here are ignored, check the output next.
        command:  /usr/bin/cps_get_oid.py base-route/obj/entry base-route/obj/entry/af=2 base-route/obj/entry/route-prefix={{test_route2hex}} base-route/obj/entry/prefix-len={{test_prefix}} base-route/obj/entry/nh-list/ifname={{test_int}}
        become: true
        register: result_getcps1
        changed_when: False
      - debug:
          var: result_getcps1
      - name: INFO - NAS routes (if empty, then configuration was not applied or there is some bug)
        debug:
          msg: "{{result_getcps1}}"
      - name: VERIFY - Check if the route 0x{{test_route2hex}} exists. If result_getcps1.stdout_lines does not have an element, then the error is because the result of the previous command was empty. It could be a bug.
        assert:
          that: "'base-route/obj/entry/route-prefix = {{test_route2hex}}' in result_getcps1.stdout"
          msg: "Route 0x{{test_route2hex}} not found"
      rescue:
      - name: DEBUG - Collecting data because the Test Failed on last VERIFY step
        command: bash -c "ip route show"
        register: result_debug_cpsfail
        changed_when: False
      - name: INFO - Debug info collected
        debug:
          msg: "{{result_debug_cpsfail}}"
      - name: FAIL - Test Failed on last VERIFY step. Exit here, after collecting debug data above
        fail:
        when: "{{test_route2hex}} not in result_getcps1.stdout_lines"

    - name: SEND -Configure Static route through CPS ADD
      command: bash -c "ip link set dev {{test_int}} up; /usr/bin/opx-config-route create --dst {{test_route2}}/{{test_cps_prefix}} --nh_addr {{test_via}}"
      become: true
      register: result_cpsaddroute
      changed_when: False
    - name: INFO - Any messages printed during route configuration are shown here
      debug:
        msg: "{{result_cpsaddroute.stdout_lines}}"
    - name: VERIFY- Check whether the CPS get succeeded
      assert:
        that: "'Success' in result_cpsaddroute.stdout"
        msg: CPS set failed here
    - name: SEND - ip route show {{test_route2}}/{{test_cps_prefix}}
      command: ip route show {{test_route2}}/{{test_cps_prefix}}
      register: result_getkroute
      changed_when: False
    - name: VERIFY - the route is present in kernel
      assert:
        that: "'{{test_route2}}' in result_getkroute.stdout"
        msg: "'{{test_route2}}' is not programmed in kernel"
    - name: SEND -CPS GET from NAS
      command:  /usr/bin/cps_get_oid.py base-route/obj/entry base-route/obj/entry/af=2 base-route/obj/entry/route-prefix={{test_route2hex}} base-route/obj/entry/prefix-len={{test_cps_prefix}} base-route/obj/entry/nh-list/ifname={{test_via}}
      become: true
      register: result_getcps1
      changed_when: False
    - name: INFO - NAS routes (if empty, then configuration was not applied or there is some bug)
      debug:
        msg: "{{result_getcps1.stdout_lines}}"
    - name: VERIFY - Check if the route 0x{{test_route2hex}} exists. If result_getcps1.stdout_lines does not have an element, then the error is because the result of the previous command was empty. It could be a bug.
      assert:
        that: "'base-route/obj/entry/route-prefix = {{test_route2hex}}' in result_getcps1.stdout"
        msg: "Route 0x{{test_route2hex}} not found"

    - name: SEND -Unconfigure Static route through CPS Delete
      command: bash -c "/usr/bin/opx-config-route delete --dst {{test_route2}}/{{test_cps_prefix}}"
      become: true
      register: result_cpsdelroute
      changed_when: False
    - name: INFO - Any messages printed during route configuration are shown here
      debug:
        msg: "{{result_cpsdelroute.stdout_lines}}"
    - name: VERIFY- Check whether the CPS deleting command succeeded
      assert:
        that: "'Success' in result_cpsaddroute.stdout"
        msg: CPS delete failed here
    - name: SEND - ip route show {{test_route2}}/{{test_cps_prefix}}
      command: ip route show {{test_route2}}/{{test_cps_prefix}}
      register: result_getkroute
      changed_when: False
    - name: VERIFY - the route is not present in kernel
      assert:
        that: "'' in result_getkroute.stdout"
        msg: "'{{test_route2}}' is not deleted from kernel"
    - name: SEND -CPS GET from NAS
      command:  /usr/bin/cps_get_oid.py base-route/obj/entry base-route/obj/entry/af=2 base-route/obj/entry/route-prefix={{test_route2hex}} base-route/obj/entry/prefix-len={{test_cps_prefix}}
      register: result_getcps1
      become: true
      changed_when: False
      ignore_errors: yes
    - name: INFO - NAS routes should be empty
      debug:
        msg: "{{result_getcps1.stdout_lines}}"
    - name: VERIFY - Check if the route 0x{{test_route2hex}} does not exists. If result_getcps1.stdout_lines does have an element, then the error is CPS deletion is not working 
      assert:
        that: "'' in result_getcps1.stdout"
        msg: "Route 0x{{test_route2hex}} is found in NAS. CPS deletion failed"

    - name: SEND - Unconfig ip address in kernel
      command: bash -c "sysctl -w net.ipv4.ip_forward=1; ip addr flush dev {{test_int}}; ip addr flush dev {{test_int1}}; ip link set dev {{test_int}} down; ip link set dev {{test_int1}} down"
      become: true
      register: result_getkrouteunconfig
    - name: INFO - Issues during kernel route unconfig, if any, will be shown here
      debug:
          msg: "{{result_getkrouteunconfig.stdout_lines}}"

    - name: SEND -Configure VLAN and add untagged ports using CPS set
      command: bash -c "ip link set dev {{test_int}} up; ip link set dev {{test_int1}} up; /usr/bin/opx-config-vlan create --id {{test_vlan}} --ports {{test_int}},{{test_int1}} --type data"
      become: true
      register: result_cpsvlanadd
      failed_when: result_cpsvlanadd.rc >= 2
      changed_when: False
    - name: INFO - Any messages printed during VLAN configuration are shown here
      debug:
        msg: "{{result_cpsvlanadd.stdout_lines}}"
    - name: Show
      command: bash -c "/usr/bin/opx-show-vlan --id {{test_vlan}}"
      become: true
      register: show_cps
    - name: TC8 - Verify if the VLAN contains the untagged ports {{test_int}} and {{test_int1}} in the CPS data
      assert:
        that: "'Untagged ports' in show_cps.stdout"
        msg: CPS is not updated - untagged ports {{test_int}} and {{test_int1}} are not present in the VLAN {{test_vlan}}

    - name: SEND -Delete one of the untagged port in vlan {{test_vlan}} using CPS set
      command: bash -c "ip link set dev {{test_int}} up; ip link set dev {{test_int1}} up; /usr/bin/opx-config-vlan remove --id {{test_vlan}} --ports {{test_int1}}"
      become: true
      register: result_cpsvlanupdate
      changed_when: False
    - name: INFO - Any messages printed during VLAN configuration are shown here
      debug:
        msg: "{{result_cpsvlanupdate.stdout_lines}}"
    - name: SEND -CPS GET from NAS, check if {{test_int1}} removed from vlan id {{test_vlan}}
      command:  /usr/bin/opx-show-vlan
      become: true
      register: result_showvlan
    - debug:
        var: result_showvlan
    - name: VERIFY - Check if {{test_int1}} removed from br{{test_vlan}}
      assert:
        that: "'{{test_int1}}' not in result_showvlan.stdout"
        msg:  "Failed to remove {{test_int1}} from VLAN {{test_vlan}}"
    - name: SEND -Delete the vlan {{test_vlan}} using CPS set
      command: bash -c "ip link set dev {{test_int}} up; ip link set dev {{test_int1}} up; sudo opx-config-vlan set --id {{test_vlan}} --ports none --taggedports none; /usr/bin/opx-config-vlan delete --id {{test_vlan}}"
      become: true
      register: result_cpsvlandel
      changed_when: False
    - name: INFO - Any messages printed during VLAN deletion are shown here
      debug:
        msg: "{{result_cpsvlandel.stdout_lines}}"
    - name: SEND -CPS GET from NAS
      command:  /usr/bin/opx-show-vlan
      become: true
      register: result_getvlancps
      changed_when: False
      ignore_errors: true
    - name: INFO- Dumping VLAN entry using CPS Get
      debug:
        msg: "{{result_getvlancps.stdout_lines}}"
    - name: VERIFY - Check if the vlan {{test_vlan}} exists. If the vlan  does not exists, then CPS get passed.
      assert:
        that: "'' in result_getvlancps.stdout"
        msg: "VLAN {{test_vlan}} found in NAS"

    - block:
      - name: SEND - systemctl is-system-running
        command: systemctl is-system-running
        become: true
        register: result_getservicestat
        changed_when: False
        ignore_errors: yes
      - name: INFO - Systemctl service status
        debug:
          msg: "{{result_getservicestat.stdout_lines}}"
      - name: VERIFY - Check Systemctl service status [SmokeTest_Systemctl]
        assert:
          that: "'running' or 'degraded' in result_getservicestat.stdout_lines[0]"
          msg:  "Expected 'running' status."
      rescue:
      - name: DEBUG - Collecting data because the Test Failed on last VERIFY step
        command: systemctl list-units --type=service --state=failed
        become: true
        register: result_debugStatus_svc
        changed_when: False
      - name: INFO - Debug info collected
        debug:
          msg: "{{result_debugStatus_svc.stdout_lines}}"
      - name: FAIL - Test Failed on last VERIFY step. Exit here, after collecting debug data above
        fail:
        when: "'failed' in result_debugStatus_svc.stdout_lines"

    - name: Remove the cloud-init service from AWS
      become: true
      apt:
        name: cloud-init
        state: absent
        purge: yes

    - name: Reset the failed units in systemctl
      command: systemctl reset-failed
      become: true
      register: result_systemctl_failed_units
      changed_when: False

    - name: SEND - systemctl --all --state=failed list-units
      command: systemctl --all --state=failed list-units
      become: true
      register: result_getservicesfailstat
      changed_when: False
    - name: INFO - Failed Systemctl
      debug:
        msg: "{{result_getservicesfailstat.stdout_lines}}"
    - name:   VERIFY - Check for failed Systemctl units [SmokeTest_Systemctl]
      assert: 
        that: "'0 loaded units listed.' == result_getservicesfailstat.stdout_lines[0]"
        msg:  "One or more units failed."

    - name: VERIFY - Check if  Aptlist Installed command execution is successfull (apt list --installed) [SmokeTest_Aptlist_Installed]
      command: apt list --installed 
      become: true
      register: result_getaptlistcmd
      changed_when: False
    - name: INFO - Aptlist Installed 
      debug:
        msg: "{{result_getaptlistcmd.stdout_lines}}"

#    - block:
#      - name: SEND - opx-show-env
#        command: /usr/bin/opx-show-env
#        become: true
#        register: result_getenv
#        changed_when: False
#      - name: INFO - List of all env like power supply, fans status
#        debug:
#          msg: "{{result_getenv.stdout_lines}}"
#      - name: VERIFY - Show env command is working
#        assert:
#          that: "'Temperature sensors' in result_getenv.stdout"
#          msg:  "Error when executing command."


    - block:
      - name: SEND - opx-show-transceiver
        command: /usr/bin/opx-show-transceiver
        become: true
        register: result_gettransceivers
        changed_when: False
      - name: INFO - List of all tranceivers inserted to the ports
        debug:
          msg: "{{result_gettransceivers.stdout_lines}}"
      - name: VERIFY - Show transceivers command is working [SmokeTest_Show_Transceivers] 
        assert:
          that: "'Present:            yes' in result_gettransceivers.stdout"
          msg:  "Error when executing command."

#-----------------------------------------------------------------------------------------
- hosts: localhost # This is not part of the test, just print the timestamp as a reference
  tasks:
    - name: INFO - Test END
      debug:
        msg: 
          - "Started: {{ test_started }}"
          - "Ended:   {{ ansible_date_time.date}} {{ ansible_date_time.time}}"
#-----------------------------------------------------------------------------------------