openstack-crowbar: Add pci passthough option

Add Jenkins parameter 'want_pci_passthrough' to allow running tests to validate
pci passthough.

To test pci passthrough locally with mkcloud, simply just export
'want_pci_passthrouh=1' and append 'rebootcloud testpcipassthru' to 'plain'
step, e.g. your_mkcloud.sh plain rebootcloud testpcipassthru

The rebootcloud step will ensure intel_iommu=on kernel param enabled.

The testpcipassthru step will prepare the environment and spin up a nested vm
(L2) on compute (L1) with a virtio disk passing from compute vm (L1). Noted
that this is not passing any device from the host (L0) itself.

There will be another PR to construct job definition for new ci.

1. add environment prep and verify function in qa_crowbarsetup.sh
2. create legacy ci job template
3. introduce testpcipassthru step in mkcloud

jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml

@@ -0,0 +1,28 @@
+- job-template:
+    name: 'cloud-mkcloud{version}-job-pcipassthru-{arch}'
+    node: cloud-trigger
+    disabled: '{obj:disabled}'
+
+    triggers:
+      - timed: 'H 20 * * *'
+
+    logrotate:
+      numToKeep: -1
+      daysToKeep: 7
+
+    builders:
+      - trigger-builds:
+        - project: openstack-mkcloud
+          condition: SUCCESS
+          block: true
+          current-parameters: true
+          predefined-parameters: |
+            TESTHEAD=1
+            cloudsource=develcloud{version}
+            libvirt_type=kvm
+            nodenumber=2
+            mkcloudtarget=all testpcipassthru
+            label={label}
+            custom_settings='export want_pci_passthrough=1'
+            job_name=cloud-mkcloud{version}-job-pcipassthru-{arch}
+

scripts/jenkins/cloud/ansible/deploy-crowbar.yml

@@ -47,6 +47,13 @@
             - reboot_after_deploy
             - not update_after_deploy
 
+        - include_role:
+            name: crowbar_setup
+          vars:
+            qa_crowbarsetup_cmd: onadmin_verify_pci_passthru
+          when:
+            - want_pci_passthrough|int == 1
+
       rescue:
         - include_role:
             name: rocketchat_notify

scripts/mkcloud

@@ -1029,6 +1029,14 @@ function rebootcrowbar
     return $?
 }
 
+# Need to run rebootcloud step to enable iommu (e.g., plain rebootcloud
+# testpcipassthru)
+function testpcipassthru
+{
+    onadmin verify_pci_passthru
+    return $?
+}
+
 function rebootcloud
 {
     # reboot compute nodes
@@ -1455,6 +1463,8 @@ Optional
         If set, does not use the --insecure flag in openstack CLI commands.
     want_monasca_tsdb (Cloud9+ only)
         Allows setting time-series DB used for Monasca [influxdb|cassandra].
+    want_pci_passthrough (default=0)
+        Deploy node with an extra disk to test pci passthrough. Cloud9+ only.
     want_ipv6 (Currently in development)
         Prepare crowbar to use a full IPv6 single stack control plane. Enabling will also set
         the firmware_type to UEFI as it's required to netboot nodes over IPv6.
@@ -1619,7 +1629,7 @@ allcmds="$step_aliases _test_setuphost \
     lonelynode_nfs_server setupironicnodes\
     restoreadminfromsnapshot createcloudsnapshot restorecloudfromsnapshot \
     cct steps batch setup_aliases onadmin onhost devsetup plain_with_upgrade_test \
-    testpreupgrade testpostupgrade deployexternalceph"
+    testpreupgrade testpostupgrade deployexternalceph testpcipassthru"
 wantedcmds=$@
 
 function expand_steps

scripts/qa_crowbarsetup.sh

@@ -57,10 +57,12 @@ fi
 
 # global variables that are set within this script
 novacontroller=
+novacompute_kvm=
 horizonserver=
 horizonservice=
 manila_service_vm_uuid=
 manila_tenant_vm_ip=
+pci_passthru_vm_name="pci-passthru-instance"
 clusternodesdrbd=
 clusternodesdata=
 clusternodesnetwork=
@@ -3315,6 +3317,43 @@ function onadmin_proposal
     done
 
     set_dashboard_alias
+
+    # prepare env to test pci_passthrough if condition met
+    prepare_pci_passthru_env
+}
+
+function prepare_pci_passthru_env
+{
+    if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then
+        # Place nova configuration to verify pci_passthru
+        get_novacontroller
+        safely oncontroller create_pci_passthru_conf
+        append_pci_passthru_filter
+        get_novacompute_kvm
+        safely oncompute create_pci_passthru_conf
+        safely oncompute load_vfio_module_onboot
+        safely oncompute bind_vfio_module_onboot
+        # TODO: need support for amd param
+        safely oncompute add_intel_iommu_param
+    fi
+}
+
+function onadmin_verify_pci_passthru
+{
+    # spin up vm and pass device from compute host to vm to verify pci
+    # passthru functionality.
+    local cirros_image_name=cirros-0.4.0-x86_64-disk.img
+    local cirros_image_url=$imageserver_url/$arch/openstack/$cirros_image_name
+    get_novacontroller
+    safely oncontroller retrieve_image $cirros_image_name $cirros_image_url
+    safely oncontroller upload_image_to_glance "cirros-0.4.0-x86_64-disk" $cirros_image_name
+    safely oncontroller setup_pci_passthru_vm "cirros-0.4.0-x86_64-disk"
+
+    # Get instance name (e.g.,instance-00000001)
+    pci_passthru_instance_name=$(ssh "$novacontroller" "source /root/.openrc;
+        openstack server show -c OS-EXT-SRV-ATTR:instance_name -f value $pci_passthru_vm_name")
+    get_novacompute_kvm
+    safely oncompute parse_vm_xml "$pci_passthru_instance_name"
 }
 
 function set_node_alias
@@ -3417,6 +3456,15 @@ function get_novacontroller
     novacontroller=`resolve_element_to_hostname "$element"`
 }
 
+function get_novacompute_kvm
+{
+    local element=`crowbar nova proposal show default | \
+        rubyjsonparse "
+                    puts j['deployment']['nova']\
+                        ['elements']['nova-compute-kvm']"`
+    novacompute_kvm=`resolve_element_to_hostname "$element"`
+}
+
 function get_horizon
 {
     local element=`crowbar horizon proposal show default | \
@@ -3860,6 +3908,98 @@ function oncontroller_heat_image_setup()
     fi
 }
 
+function oncompute_parse_vm_xml
+{
+    local instance_name=$1
+    echo $instance_name
+    virsh list --all | grep $instance_name
+    [ $? != 0 ] && complain 53 "failed to find pci-passthru-instance"
+
+    # verify node xml to see if it has pci passthru section which has the
+    # specific address passing from compute host
+    virsh dumpxml $instance_name | grep -E "<hostdev" -A 7 | grep "0x1d"
+    [ $? != 0 ] && complain 63 "failed to find pci-passthru block in node xml"
+
+    return 0
+}
+
+function oncontroller_retrieve_image
+{
+    local cirros_image_name=$1
+    local cirros_image_url=$2
+
+    if [ -n "${localreposdir_target}" ] ; then
+        local_image_path=$localreposdir_target/images/$arch/openstack/$cirros_image_name
+    fi
+
+    if [ -n "${local_image_path}" -a -f "${local_image_path}" ] ; then
+        cirros_image_name=$local_image_path
+    else
+        local ret=$(wget -N --progress=dot:mega "$cirros_image_url" 2>&1 >/dev/null)
+        if [[ $ret =~ "200 OK" ]]; then
+            echo $ret
+        elif [[ $ret =~ "Not Found" ]]; then
+            complain 73 "cirros image not found: $ret"
+        else
+            complain 74 "failed to retrieve cirros image: $ret"
+        fi
+    fi
+}
+
+function oncontroller_upload_image_to_glance
+{
+    local image_name=$1
+    local image_path=$2
+    shift ; shift
+    local image_params=$@
+
+    if [ -z "$image_params" ]; then
+        image_params="--disk-format qcow2 --property hypervisor_type=kvm"
+    fi
+
+    . .openrc
+
+    # using list subcommand because show requires an ID
+    if ! openstack image list --format value -c Name | grep -q "$image_name"; then
+        openstack image create --file $image_path \
+            $image_params --container-format bare --public \
+            $image_name
+    fi
+}
+
+function oncontroller_setup_pci_passthru_vm
+{
+    local image_name=$1
+    local sec_group="pci-passthru"
+
+    . .openrc
+
+    # create flavor with pci-passthru property so that the vm using this flavor
+    # will aquire the pci device passing from the host (compute)
+    openstack flavor create --id 200 --ram 512 --ephemeral 0 \
+        --vcpus 1 --property "pci_passthrough:alias"="a1:1" \
+        pci-passthru-flavor
+
+    if iscloudver 9plus && ! openstack security group show $sec_group 2>/dev/null ; then
+        openstack security group create --description "$sec_group description" $sec_group
+        openstack security group rule create --protocol icmp $sec_group
+        openstack security group rule create --protocol tcp --dst-port 22 $sec_group
+    fi
+
+    fixed_net_id=`neutron net-show fixed -f value -c id`
+    timeout 10m openstack server create --flavor 200 \
+        --image $image_name \
+        --security-group $sec_group \
+        --nic net-id=$fixed_net_id $pci_passthru_vm_name
+
+    [ $? != 0 ] && complain 43 "nova boot pci-passthru-instance failed"
+
+    # Check status of the vm
+    wait_for 60 1 "openstack server show -c status -f value pci-passthru-instance | grep '^ACTIVE$'" \
+        "pci passthru VM booted and is in ACTIVE state" \
+        "echo \"ERROR: pci passthru VM is not in ACTIVE state.\""
+}
+
 function oncontroller_manila_generic_driver_setup()
 {
     if [[ $wantxenpv ]] ; then
@@ -4478,6 +4618,107 @@ function oncontroller
     run_on "$novacontroller" "oncontroller_$func $@"
 }
 
+function oncompute
+{
+    local func=$1 ; shift
+    run_on "$novacompute_kvm" "oncompute_$func $@"
+}
+
+function oncompute_add_intel_iommu_param
+{
+    cat /etc/default/grub | grep iommu >/dev/null 2>&1
+    if [ $? != 0 ]; then
+        # enabled intel_iommu in kernel param
+        sed -in "s/\(^GRUB_CMDLINE_LINUX_DEFAULT=.*\)\"$/\1 intel_iommu=on\"/g" /etc/default/grub
+        update-bootloader
+    fi
+}
+
+function oncompute_load_vfio_module_onboot
+{
+    if [ ! -e "/etc/modules-load.d/pci-passthru.conf" ]; then
+        # vfio-pci module is not loaded by default, however, we need it to bind to
+        # our candidate device to verify pci passthru
+        cat > /etc/modules-load.d/pci-passthru.conf <<EOF
+vfio_pci
+EOF
+        modprobe vfio-pci
+    fi
+}
+
+function oncompute_bind_vfio_module_onboot
+{
+    if [ ! -e "/etc/udev/rules.d/99-pci-passthru.rules" ]; then
+            # We need to bind vfio-pci to our candidate device to verify pci passthru
+            cat > /etc/udev/rules.d/99-pci-passthru.rules <<EOF
+KERNELS=="0000:00:1d.0", SUBSYSTEMS=="pci", ATTRS{device}=="0x1001", ATTRS{vendor}=="0x1af4", \
+RUN+="/bin/bash -c 'echo  0000:00:1d.0 > /sys/bus/pci/devices/0000:00:1d.0/driver/unbind'", \
+RUN+="/bin/bash -c 'echo 1af4 1001 > /sys/bus/pci/drivers/vfio-pci/new_id'"
+EOF
+    fi
+}
+
+function oncompute_create_pci_passthru_conf
+{
+    if [ ! -e "/etc/nova/nova.conf.d/200-nova-pci-passthru.conf" ]; then
+        # drop new configuration file under /etc/nova/nova.conf.d/ on compute
+        # [pci]
+        # passthrough_whitelist = { "address": "0000:41:00.0" }
+        # alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
+        cat > /etc/nova/nova.conf.d/200-nova-pci-passthru.conf <<EOF
+[pci]
+passthrough_whitelist = { "address": "0000:00:1d.0" }
+alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
+EOF
+        # Make the changes effective
+        systemctl restart openstack-nova-compute.service
+    fi
+}
+
+function oncontroller_create_pci_passthru_conf
+{
+    if [ ! -e "/etc/nova/nova.conf.d/200-nova-pci-passthru.conf" ]; then
+        # drop new configuration file under /etc/nova/nova.conf.d/ on controller
+        # [pci]
+        # alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
+        cat > /etc/nova/nova.conf.d/200-nova-pci-passthru.conf <<EOF
+[pci]
+alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
+EOF
+        # Make the changes effective
+        systemctl restart openstack-nova-api.service
+    fi
+}
+
+function append_pci_passthru_filter
+{
+    set -x
+    prepare_proposal_for_modification nova
+    local pfile=$(get_proposal_filename nova default)
+    local nova_conf="/etc/nova/nova.conf.d/100-nova.conf"
+    local enabled_filters=
+    enabled_filters=$(ssh "$novacontroller" "grep enabled_filters $nova_conf | cut -d' ' -f3")
+
+    echo $enabled_filters | grep "PciPassthroughFilter" >/dev/null 2>&1
+    if [ $? != 0 ]; then
+        enabled_filters="${enabled_filters},PciPassthroughFilter"
+
+        # append PciPassthroughFilter to enabled_filters
+        proposal_set_value nova default \
+            "['attributes']['nova']['scheduler']['enabled_filters']" \
+            "'$enabled_filters'"
+
+        crowbar nova proposal --file=$pfile edit default ||\
+            complain 88 "'crowbar nova proposal --file=$pfile edit default' failed with exit code: $?"
+
+        crowbar_proposal_commit nova
+
+        # Make the changes effective right away
+        run_on "$novacontroller" 'sed -in "s/\(enabled_filters = .*\)$/\1,PciPassthroughFilter/g"' $nova_conf
+        run_on "$novacontroller" "systemctl restart openstack-nova-scheduler.service"
+    fi
+}
+
 function install_suse_ca
 {
     # trust build key - workaround https://bugzilla.opensuse.org/show_bug.cgi?id=935020