Skip to content

Commit

Permalink
openstack-crowbar: Add pci passthough option
Browse files Browse the repository at this point in the history 
Add Jenkins parameter 'want_pci_passthrough' to allow running tests to validate
pci passthough.

To test pci passthrough locally with mkcloud, simply just export
'want_pci_passthrouh=1' and append 'rebootcloud testpcipassthru' to 'plain'
step, e.g. your_mkcloud.sh plain rebootcloud testpcipassthru

The rebootcloud step will ensure intel_iommu=on kernel param enabled.

The testpcipassthru step will prepare the environment and spin up a nested vm
(L2) on compute (L1) with a virtio disk passing from compute vm (L1). Noted
that this is not passing any device from the host (L0) itself.

There will be another PR to construct job definition and other prep works for
new ci.

1. add environment prep and verify function in qa_crowbarsetup.sh
2. create legacy ci job template
3. introduce testpcipassthru step in mkcloud
  • Loading branch information
@kwu83tw
kwu83tw committed Aug 9, 2019
1 parent a98aeb9 commit c9c7f4e
Show file tree
Hide file tree
Showing 3 changed files with 315 additions and 1 deletion.
28 changes: 28 additions & 0 deletions jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
- job-template:
name: 'cloud-mkcloud{version}-job-pcipassthru-{arch}'
node: cloud-trigger
disabled: '{obj:disabled}'

triggers:
- timed: 'H 20 * * *'

logrotate:
numToKeep: -1
daysToKeep: 7

builders:
- trigger-builds:
- project: openstack-mkcloud
condition: SUCCESS
block: true
current-parameters: true
predefined-parameters: |
TESTHEAD=1
cloudsource=develcloud{version}
libvirt_type=kvm
nodenumber=2
mkcloudtarget=all testpcipassthru
label={label}
custom_settings='export want_pci_passthrough=1'
job_name=cloud-mkcloud{version}-job-pcipassthru-{arch}
12 changes: 11 additions & 1 deletion scripts/mkcloud
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1029,6 +1029,14 @@ function rebootcrowbar
return $?
}

# Need to run rebootcloud step to enable iommu (e.g., plain rebootcloud
# testpcipassthru)
function testpcipassthru
{
onadmin verify_pci_passthru
return $?
}

function rebootcloud
{
# reboot compute nodes
Expand Down Expand Up @@ -1455,6 +1463,8 @@ Optional
If set, does not use the --insecure flag in openstack CLI commands.
want_monasca_tsdb (Cloud9+ only)
Allows setting time-series DB used for Monasca [influxdb|cassandra].
want_pci_passthrough (default=0)
Deploy node with an extra disk to test pci passthrough. Cloud9+ only.
want_ipv6 (Currently in development)
Prepare crowbar to use a full IPv6 single stack control plane. Enabling will also set
the firmware_type to UEFI as it's required to netboot nodes over IPv6.
Expand Down Expand Up @@ -1619,7 +1629,7 @@ allcmds="$step_aliases _test_setuphost \
lonelynode_nfs_server setupironicnodes\
restoreadminfromsnapshot createcloudsnapshot restorecloudfromsnapshot \
cct steps batch setup_aliases onadmin onhost devsetup plain_with_upgrade_test \
testpreupgrade testpostupgrade deployexternalceph"
testpreupgrade testpostupgrade deployexternalceph testpcipassthru"
wantedcmds=$@

function expand_steps
Expand Down
276 changes: 276 additions & 0 deletions scripts/qa_crowbarsetup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,12 @@ fi

# global variables that are set within this script
novacontroller=
novacompute_kvm=
horizonserver=
horizonservice=
manila_service_vm_uuid=
manila_tenant_vm_ip=
pci_passthru_vm_name="pci-passthru-instance"
clusternodesdrbd=
clusternodesdata=
clusternodesnetwork=
Expand Down Expand Up @@ -3315,6 +3317,45 @@ function onadmin_proposal
done

set_dashboard_alias

# prepare env to test pci_passthrough if condition met
prepare_pci_passthru_env
}

function prepare_pci_passthru_env
{
if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then
# Place nova configuration to verify pci_passthru
get_novacontroller
safely oncontroller create_pci_passthru_conf
append_pci_passthru_filter
get_novacompute_kvm
safely oncompute create_pci_passthru_conf
safely oncompute load_vfio_module_onboot
safely oncompute bind_vfio_module_onboot
# TODO: need support for amd param
safely oncompute add_intel_iommu_param
fi
}

function onadmin_verify_pci_passthru
{
# spin up vm and pass device from compute host to vm to verify pci
# passthru functionality.
local cirros_image_name=cirros-0.4.0-x86_64-disk.img
local cirros_image_url=$imageserver_url/$arch/openstack/$cirros_image_name
get_novacontroller
safely oncontroller retrieve_image $cirros_image_name $cirros_image_url
safely oncontroller upload_image_to_glance "cirros-0.4.0-x86_64-disk" $cirros_image_name
safely oncontroller setup_pci_passthru_vm "cirros-0.4.0-x86_64-disk"

# Get instance name (e.g.,instance-00000001)
pci_passthru_instance_name=$(ssh "$novacontroller" "source /root/.openrc;
openstack server show -c OS-EXT-SRV-ATTR:instance_name -f value $pci_passthru_vm_name")
get_novacompute_kvm
safely oncompute unload_pci_device "$pci_passthru_instance_name"
safely oncompute reload_pci_device "$pci_passthru_instance_name"
safely oncompute parse_vm_xml "$pci_passthru_instance_name"
}

function set_node_alias
Expand Down Expand Up @@ -3417,6 +3458,15 @@ function get_novacontroller
novacontroller=`resolve_element_to_hostname "$element"`
}

function get_novacompute_kvm
{
local element=`crowbar nova proposal show default | \
rubyjsonparse "
puts j['deployment']['nova']\
['elements']['nova-compute-kvm']"`
novacompute_kvm=`resolve_element_to_hostname "$element"`
}

function get_horizon
{
local element=`crowbar horizon proposal show default | \
Expand Down Expand Up @@ -3860,6 +3910,131 @@ function oncontroller_heat_image_setup()
fi
}
function oncompute_parse_vm_xml
{
local instance_name=$1
echo $instance_name
virsh list --all | grep $instance_name
[ $? != 0 ] && complain 53 "failed to find pci-passthru-instance"
# verify node xml to see if it has pci passthru section which has the
# specific address passing from compute host
virsh dumpxml $instance_name | grep -E "<hostdev" -A 7 | grep "0x1d"
[ $? != 0 ] && complain 63 "failed to find pci-passthru block in node xml"
return 0
}
function oncontroller_retrieve_image
{
local image_name=$1
local image_url=$2
if [ -n "${localreposdir_target}" ] ; then
local_image_path=$localreposdir_target/images/$arch/openstack/$image_name
fi
if [ -n "${local_image_path}" -a -f "${local_image_path}" ] ; then
image_name=$local_image_path
else
local ret=$(wget -N --progress=dot:mega "$image_url" 2>&1 >/dev/null)
if [[ $ret =~ "200 OK" ]]; then
echo $ret
elif [[ $ret =~ "Not Found" ]]; then
complain 73 "$image_name image not found: $ret"
else
complain 74 "failed to retrieve $image_name image: $ret"
fi
fi
}
function oncontroller_upload_image_to_glance
{
local image_name=$1
local image_path=$2
shift ; shift
local image_params=$@
if [ -z "$image_params" ]; then
image_params="--disk-format qcow2 --property hypervisor_type=kvm"
fi
. .openrc
# using list subcommand because show requires an ID
if ! openstack image list --format value -c Name | grep -q "$image_name"; then
openstack image create --file $image_path \
$image_params --container-format bare --public \
$image_name
fi
}
function oncontroller_setup_pci_passthru_vm
{
local image_name=$1
local sec_group="pci-passthru"
. .openrc
# create flavor with pci-passthru property so that the vm using this flavor
# will aquire the pci device passing from the host (compute)
openstack flavor create --id 200 --ram 512 --ephemeral 0 \
--vcpus 1 --property "pci_passthrough:alias"="a1:1" \
pci-passthru-flavor
if iscloudver 9plus && ! openstack security group show $sec_group 2>/dev/null ; then
openstack security group create --description "$sec_group description" $sec_group
openstack security group rule create --protocol icmp $sec_group
openstack security group rule create --protocol tcp --dst-port 22 $sec_group
fi
fixed_net_id=`neutron net-show fixed -f value -c id`
timeout 10m openstack server create --flavor 200 \
--image $image_name \
--security-group $sec_group \
--nic net-id=$fixed_net_id $pci_passthru_vm_name
[ $? != 0 ] && complain 43 "nova boot $pci_passthru_vm_name failed"
# Check status of the vm
wait_for 60 1 "openstack server show -c status -f value $pci_passthru_vm_name | grep '^ACTIVE$'" \
"pci passthru VM booted and is in ACTIVE state" \
"echo \"ERROR: pci passthru VM is not in ACTIVE state.\""
local floatingip=$(addfloatingip $pci_passthru_vm_name)
[ -z "$floatingip" ] && complain 44 "adding a floating ip to $pci_passthru_vm_name VM failed"
}
function oncompute_unload_pci_device
{
local vm_domain=$1
local pci_nodedev=$(virsh nodedev-list --cap pci|grep 00_1d)
if [ -z "$pci_nodedev" ] ; then
pci_nodedev="pci_0000_00_1d_0"
fi
virt-xml $vm_domain --remove-device --update --hostdev $pci_nodedev
# avoid nova to revert vm state for the change on pci device
systemctl stop openstack-nova-compute
# make change effect
virsh destroy $vm_domain
virsh start $vm_domain
}
function oncompute_reload_pci_device
{
local vm_domain=$1
local pci_nodedev=$(virsh nodedev-list --cap pci|grep 00_1d)
if [ -z "$pci_nodedev" ] ; then
pci_nodedev="pci_0000_00_1d_0"
fi
virt-xml $vm_domain --add-device --update --hostdev $pci_nodedev
systemctl start openstack-nova-compute
# $vm_domain shall be pingable from now on
}
function oncontroller_manila_generic_driver_setup()
{
if [[ $wantxenpv ]] ; then
Expand Down Expand Up @@ -4478,6 +4653,107 @@ function oncontroller
run_on "$novacontroller" "oncontroller_$func $@"
}
function oncompute
{
local func=$1 ; shift
run_on "$novacompute_kvm" "oncompute_$func $@"
}
function oncompute_add_intel_iommu_param
{
cat /etc/default/grub | grep iommu >/dev/null 2>&1
if [ $? != 0 ]; then
# enabled intel_iommu in kernel param
sed -in "s/\(^GRUB_CMDLINE_LINUX_DEFAULT=.*\)\"$/\1 intel_iommu=on\"/g" /etc/default/grub
update-bootloader
fi
}
function oncompute_load_vfio_module_onboot
{
if [ ! -e "/etc/modules-load.d/pci-passthru.conf" ]; then
# vfio-pci module is not loaded by default, however, we need it to bind to
# our candidate device to verify pci passthru
cat > /etc/modules-load.d/pci-passthru.conf <<EOF
vfio_pci
EOF
modprobe vfio-pci
fi
}
function oncompute_bind_vfio_module_onboot
{
if [ ! -e "/etc/udev/rules.d/99-pci-passthru.rules" ]; then
# We need to bind vfio-pci to our candidate device to verify pci passthru
cat > /etc/udev/rules.d/99-pci-passthru.rules <<EOF
KERNELS=="0000:00:1d.0", SUBSYSTEMS=="pci", ATTRS{device}=="0x1001", ATTRS{vendor}=="0x1af4", \
RUN+="/bin/bash -c 'echo 0000:00:1d.0 > /sys/bus/pci/devices/0000:00:1d.0/driver/unbind'", \
RUN+="/bin/bash -c 'echo 1af4 1001 > /sys/bus/pci/drivers/vfio-pci/new_id'"
EOF
fi
}
function oncompute_create_pci_passthru_conf
{
if [ ! -e "/etc/nova/nova.conf.d/200-nova-pci-passthru.conf" ]; then
# drop new configuration file under /etc/nova/nova.conf.d/ on compute
# [pci]
# passthrough_whitelist = { "address": "0000:41:00.0" }
# alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
cat > /etc/nova/nova.conf.d/200-nova-pci-passthru.conf <<EOF
[pci]
passthrough_whitelist = { "address": "0000:00:1d.0" }
alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
EOF
# Make the changes effective
systemctl restart openstack-nova-compute.service
fi
}
function oncontroller_create_pci_passthru_conf
{
if [ ! -e "/etc/nova/nova.conf.d/200-nova-pci-passthru.conf" ]; then
# drop new configuration file under /etc/nova/nova.conf.d/ on controller
# [pci]
# alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
cat > /etc/nova/nova.conf.d/200-nova-pci-passthru.conf <<EOF
[pci]
alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
EOF
# Make the changes effective
systemctl restart openstack-nova-api.service
fi
}
function append_pci_passthru_filter
{
set -x
prepare_proposal_for_modification nova
local pfile=$(get_proposal_filename nova default)
local nova_conf="/etc/nova/nova.conf.d/100-nova.conf"
local enabled_filters=
enabled_filters=$(ssh "$novacontroller" "grep enabled_filters $nova_conf | cut -d' ' -f3")
echo $enabled_filters | grep "PciPassthroughFilter" >/dev/null 2>&1
if [ $? != 0 ]; then
enabled_filters="${enabled_filters},PciPassthroughFilter"
# append PciPassthroughFilter to enabled_filters
proposal_set_value nova default \
"['attributes']['nova']['scheduler']['enabled_filters']" \
"'$enabled_filters'"
crowbar nova proposal --file=$pfile edit default ||\
complain 88 "'crowbar nova proposal --file=$pfile edit default' failed with exit code: $?"
crowbar_proposal_commit nova
# Make the changes effective right away
run_on "$novacontroller" 'sed -in "s/\(enabled_filters = .*\)$/\1,PciPassthroughFilter/g"' $nova_conf
run_on "$novacontroller" "systemctl restart openstack-nova-scheduler.service"
fi
}
function install_suse_ca
{
# trust build key - workaround https://bugzilla.opensuse.org/show_bug.cgi?id=935020
Expand Down

0 comments on commit c9c7f4e

Please sign in to comment.