From b09e7c5bf6366adfe2be64076cd11e49b3fb9206 Mon Sep 17 00:00:00 2001 From: wukent Date: Sat, 13 Jul 2019 19:35:57 +0000 Subject: [PATCH] openstack-crowbar: Add pci passthough option Add Jenkins parameter 'want_pci_passthrough' to allow running tests to validate pci passthough. To test pci passthrough locally with mkcloud, simply just export 'want_pci_passthrouh=1' and append 'rebootcloud testpcipassthru' to 'plain' step, e.g. your_mkcloud.sh plain rebootcloud testpcipassthru The rebootcloud step will ensure intel_iommu=on kernel param enabled. The testpcipassthru step will prepare the environment and spin up a nested vm (L2) on compute (L1) with a virtio disk passing from compute vm (L1). Noted that this is not passing any device from the host (L0) itself. There will be another PR to construct job definition and other prep works for new ci. 1. add environment prep and verify function in qa_crowbarsetup.sh 2. create legacy ci job template 3. introduce testpcipassthru step in mkcloud --- ...cloud-mkcloud-job-pcipassthru-template.yml | 28 ++ scripts/mkcloud | 12 +- scripts/qa_crowbarsetup.sh | 314 +++++++++++++++++- 3 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml diff --git a/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml new file mode 100644 index 0000000000..d903f50589 --- /dev/null +++ b/jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml @@ -0,0 +1,28 @@ +- job-template: + name: 'cloud-mkcloud{version}-job-pcipassthru-{arch}' + node: cloud-trigger + disabled: '{obj:disabled}' + + triggers: + - timed: 'H 20 * * *' + + logrotate: + numToKeep: -1 + daysToKeep: 7 + + builders: + - trigger-builds: + - project: openstack-mkcloud + condition: SUCCESS + block: true + current-parameters: true + predefined-parameters: | + TESTHEAD=1 + cloudsource=develcloud{version} + libvirt_type=kvm + nodenumber=2 + mkcloudtarget=plain rebootcloud testpcipassthru + label={label} + custom_settings='export want_pci_passthrough=1' + job_name=cloud-mkcloud{version}-job-pcipassthru-{arch} + diff --git a/scripts/mkcloud b/scripts/mkcloud index 77008bd516..6e95c0a8c5 100755 --- a/scripts/mkcloud +++ b/scripts/mkcloud @@ -1027,6 +1027,14 @@ function rebootcrowbar return $? } +# Need to run rebootcloud step to enable iommu (e.g., plain rebootcloud +# testpcipassthru) +function testpcipassthru +{ + onadmin verify_pci_passthru + return $? +} + function rebootcloud { # reboot compute nodes @@ -1453,6 +1461,8 @@ Optional If set, does not use the --insecure flag in openstack CLI commands. want_monasca_tsdb (Cloud9+ only) Allows setting time-series DB used for Monasca [influxdb|cassandra]. + want_pci_passthrough (default=0) + Deploy node with an extra disk to test pci passthrough. Cloud9+ only. want_ipv6 (Currently in development) Prepare crowbar to use a full IPv6 single stack control plane. Enabling will also set the firmware_type to UEFI as it's required to netboot nodes over IPv6. @@ -1617,7 +1627,7 @@ allcmds="$step_aliases _test_setuphost \ lonelynode_nfs_server setupironicnodes\ restoreadminfromsnapshot createcloudsnapshot restorecloudfromsnapshot \ cct steps batch setup_aliases onadmin onhost devsetup plain_with_upgrade_test \ - testpreupgrade testpostupgrade deployexternalceph" + testpreupgrade testpostupgrade deployexternalceph testpcipassthru" wantedcmds=$@ function expand_steps diff --git a/scripts/qa_crowbarsetup.sh b/scripts/qa_crowbarsetup.sh index 71a4c8a3f5..289f82c3fd 100644 --- a/scripts/qa_crowbarsetup.sh +++ b/scripts/qa_crowbarsetup.sh @@ -57,10 +57,12 @@ fi # global variables that are set within this script novacontroller= +novacompute_kvm= horizonserver= horizonservice= manila_service_vm_uuid= manila_tenant_vm_ip= +pci_passthru_vm_name="pci-passthru-instance" clusternodesdrbd= clusternodesdata= clusternodesnetwork= @@ -3332,6 +3334,50 @@ function onadmin_proposal done set_dashboard_alias + + # prepare env to test pci_passthrough if condition met + prepare_pci_passthru_env +} + +function prepare_pci_passthru_env +{ + if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then + # Place nova configuration to verify pci_passthru + get_novacontroller + safely oncontroller create_pci_passthru_conf + append_pci_passthru_filter + get_novacompute_kvm + safely oncompute create_pci_passthru_conf + safely oncompute load_vfio_module_onboot + safely oncompute bind_vfio_module_onboot + # TODO: need support for amd param + safely oncompute add_intel_iommu_param + fi +} + +function onadmin_verify_pci_passthru +{ + if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then + # spin up vm and pass device from compute host to vm to verify pci + # passthru functionality. + local cirros_image_name=cirros-0.4.0-x86_64-disk.img + local cirros_image_url=$imageserver_url/$arch/openstack/$cirros_image_name + get_novacontroller + safely oncontroller retrieve_image $cirros_image_name $cirros_image_url + safely oncontroller upload_image_to_glance "cirros-0.4.0-x86_64-disk" $cirros_image_name + safely oncontroller setup_pci_passthru_vm "cirros-0.4.0-x86_64-disk" + + # Get instance name (e.g.,instance-00000001) + pci_passthru_instance_name=$(ssh "$novacontroller" "source /root/.openrc; + openstack server show -c OS-EXT-SRV-ATTR:instance_name -f value $pci_passthru_vm_name") + get_novacompute_kvm + # configure vm to bypass the boot up failure due to the passthru device + safely oncompute unload_pci_device $pci_passthru_instance_name + # wait until vm is pingable before re-attaching the device + safely oncontroller waitforinstance $pci_passthru_vm_name + safely oncompute reload_pci_device $pci_passthru_instance_name + safely oncompute parse_vm_xml $pci_passthru_instance_name + fi } function set_node_alias @@ -3434,6 +3480,15 @@ function get_novacontroller novacontroller=`resolve_element_to_hostname "$element"` } +function get_novacompute_kvm +{ + local element=`crowbar nova proposal show default | \ + rubyjsonparse " + puts j['deployment']['nova']\ + ['elements']['nova-compute-kvm']"` + novacompute_kvm=`resolve_element_to_hostname "$element"` +} + function get_horizon { local element=`crowbar horizon proposal show default | \ @@ -3875,6 +3930,140 @@ function oncontroller_heat_image_setup() fi } +function oncompute_parse_vm_xml +{ + local instance_name=$1 + echo $instance_name + virsh list --all | grep $instance_name + [ $? != 0 ] && complain 53 "failed to find pci-passthru-instance" + + # verify node xml to see if it has pci passthru section which has the + # specific address passing from compute host + virsh dumpxml $instance_name | grep -E "&1 >/dev/null) + if [[ $ret =~ "200 OK" ]]; then + echo $ret + elif [[ $ret =~ "Not Found" ]]; then + complain 73 "$image_name image not found: $ret" + else + complain 74 "failed to retrieve $image_name image: $ret" + fi + fi +} + +function oncontroller_upload_image_to_glance +{ + local image_name=$1 + local image_path=$2 + shift ; shift + local image_params=$@ + + if [ -z "$image_params" ]; then + image_params="--disk-format qcow2 --property hypervisor_type=kvm" + fi + + . .openrc + + # using list subcommand because show requires an ID + if ! openstack image list --format value -c Name | grep -q "$image_name"; then + openstack image create --file $image_path \ + $image_params --container-format bare --public \ + $image_name + fi +} + +function oncontroller_setup_pci_passthru_vm +{ + local image_name=$1 + local sec_group="pci-passthru" + + . .openrc + + # create flavor with pci-passthru property so that the vm using this flavor + # will aquire the pci device passing from the host (compute) + openstack flavor create --id 200 --ram 512 --ephemeral 0 \ + --vcpus 1 --property "pci_passthrough:alias"="a1:1" \ + pci-passthru-flavor + + if iscloudver 9plus && ! openstack security group show $sec_group 2>/dev/null ; then + openstack security group create --description "$sec_group description" $sec_group + openstack security group rule create --protocol icmp $sec_group + openstack security group rule create --protocol tcp --dst-port 22 $sec_group + fi + + fixed_net_id=`neutron net-show fixed -f value -c id` + timeout 10m openstack server create --flavor 200 \ + --image $image_name \ + --security-group $sec_group \ + --nic net-id=$fixed_net_id $pci_passthru_vm_name + + [ $? != 0 ] && complain 43 "nova boot $pci_passthru_vm_name failed" + + # Check status of the vm + wait_for 60 1 "openstack server show -c status -f value $pci_passthru_vm_name | grep '^ACTIVE$'" \ + "pci passthru VM booted and is in ACTIVE state" \ + "echo \"ERROR: pci passthru VM is not in ACTIVE state.\"" +} + +function oncompute_unload_pci_device +{ + local vm_domain=$1 + cat > pcip_device.xml < + +
+ + +EOF + if [[ -n $(virsh dumpxml $vm_domain|grep hostdev) ]]; then + # avoid nova to revert vm state for the change on pci device + systemctl stop openstack-nova-compute + + virsh destroy $vm_domain + + virsh detach-device $vm_domain --file pcip_device.xml --current + [[ -n $(virsh dumpxml $vm_domain|grep hostdev) ]] && \ + complain 64 "failed to detach pci device" + fi + virsh start $vm_domain +} + +function oncompute_reload_pci_device +{ + local vm_domain=$1 + if [ ! -f "pcip_device.xml" ]; then + cat > pcip_device.xml < + +
+ + +EOF + fi + virsh attach-device $vm_domain --file pcip_device.xml --current + [[ $? != 0 ]] && complain 65 "failed to attach pci device" + + systemctl start openstack-nova-compute + # $vm_domain shall be pingable from now on +} + function oncontroller_manila_generic_driver_setup() { if [[ $wantxenpv ]] ; then @@ -4489,6 +4678,120 @@ function oncontroller run_on "$novacontroller" "oncontroller_$func $@" } +function oncompute +{ + local func=$1 ; shift + cd /root + scp -r $SCRIPTS_DIR $mkcconf "$novacompute_kvm:" + ssh "$novacompute_kvm" "export deployswift=$deployswift ; export deployceph=$deployceph ; + export tempestoptions=\"$tempestoptions\" ; + export cephmons=\"$cephmons\" ; export cephosds=\"$cephosds\" ; + export cephmons_names=\"$cephmons_names\" ; + export cephradosgws=\"$cephradosgws\" ; export wantcephtestsuite=\"$wantcephtestsuite\" ; + export wantradosgwtest=\"$wantradosgwtest\" ; export cloudsource=\"$cloudsource\" ; + export libvirt_type=\"$libvirt_type\" ; + export cloud=$cloud ; export TESTHEAD=$TESTHEAD ; + export is_oncontroller=yes ; export want_ceph_ssl=$want_ceph_ssl ; + export want_all_ssl=$want_all_ssl ; + . ./$(basename $SCRIPTS_DIR)/qa_crowbarsetup.sh ; onadmin_set_source_variables ; + oncompute_$func $@" +} + +function oncompute_add_intel_iommu_param +{ + cat /etc/default/grub | grep iommu >/dev/null 2>&1 + if [ $? != 0 ]; then + # enabled intel_iommu in kernel param + sed -in "s/\(^GRUB_CMDLINE_LINUX_DEFAULT=.*\)\"$/\1 intel_iommu=on\"/g" /etc/default/grub + update-bootloader + fi +} + +function oncompute_load_vfio_module_onboot +{ + if [ ! -e "/etc/modules-load.d/pci-passthru.conf" ]; then + # vfio-pci module is not loaded by default, however, we need it to bind to + # our candidate device to verify pci passthru + cat > /etc/modules-load.d/pci-passthru.conf < /etc/udev/rules.d/99-pci-passthru.rules < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf < /etc/nova/nova.conf.d/200-nova-pci-passthru.conf </dev/null 2>&1 + if [ $? != 0 ]; then + enabled_filters="${enabled_filters},PciPassthroughFilter" + + # append PciPassthroughFilter to enabled_filters + proposal_set_value nova default \ + "['attributes']['nova']['scheduler']['enabled_filters']" \ + "'$enabled_filters'" + + crowbar nova proposal --file=$pfile edit default ||\ + complain 88 "'crowbar nova proposal --file=$pfile edit default' failed with exit code: $?" + + crowbar_proposal_commit nova + + # Make the changes effective right away + run_on "$novacontroller" 'sed -in "s/\(enabled_filters = .*\)$/\1,PciPassthroughFilter/g"' $nova_conf + run_on "$novacontroller" "systemctl restart openstack-nova-scheduler.service" + fi +} + function install_suse_ca { # trust build key - workaround https://bugzilla.opensuse.org/show_bug.cgi?id=935020 @@ -5261,11 +5564,12 @@ function onadmin_rebootcloud exit $ret } -# make sure that testvm and manila-service VMs are up and reachable -# if VMs were shutdown, VMs are started +# make sure that testvm and manila-service VMs or a given VM are up and +# reachable. If VMs were shutdown, VMs are started # adds a floating IP to VM function oncontroller_waitforinstance { + local vm_name=$1 . .openrc safely openstack server list if [[ -n $(openstack server show testvm -f value -c id) ]]; then @@ -5275,6 +5579,12 @@ function oncontroller_waitforinstance [[ $floatingip ]] || complain 12 "no IP found for instance" wait_for 100 1 "ping -q -c 1 -w 1 $floatingip >/dev/null" "testvm to boot up" fi + if [[ -n "$vm_name" ]] && [[ -n $(openstack server show $vm_name -f value -c id) ]]; then + local floatingip=$(addfloatingip $vm_name) + [[ $floatingip ]] || complain 12 "no IP found for instance" + wait_for 1000 1 "ping -q -c 1 -w 1 $floatingip >/dev/null" "$vm_name booted and ping returned" + wait_for 500 1 "netcat -z $floatingip 22" "ssh daemon on $vm_name is accessible" + fi export OS_TENANT_NAME='manila-service' export OS_PROJECT_NAME=$OS_TENANT_NAME if [[ -n $(openstack server show manila-service -f value -c id) ]]; then