Split run.sh into separate scripts (setup.sh, run.sh, cleanup.sh) to enable multi-device testing, and prepare for VFIO selftests automatically detecting which devices to use for testing by storing device metadata on the filesystem.
- setup.sh takes one or more BDFs as arguments and sets up each device. Metadata about each device is stored on the filesystem in the directory:
${TMPDIR:-/tmp}/vfio-selftests-devices
Within this directory is a directory for each BDF, and then files in those directories that cleanup.sh uses to cleanup the device.
- run.sh runs a selftest by passing it the BDFs of all set up devices.
- cleanup.sh takes zero or more BDFs as arguments and cleans up each device. If no BDFs are provided, it cleans up all devices.
This split enables multi-device testing by allowing multiple BDFs to be set up and passed into tests:
For example:
$ tools/testing/selftests/vfio/scripts/setup.sh <BDF1> <BDF2> $ tools/testing/selftests/vfio/scripts/setup.sh <BDF3> $ tools/testing/selftests/vfio/scripts/run.sh echo <BDF1> <BDF2> <BDF3> $ tools/testing/selftests/vfio/scripts/cleanup.sh
In the future, VFIO selftests can automatically detect set up devices by inspecting ${TMPDIR:-/tmp}/vfio-selftests-devices. This will avoid the need for the run.sh script.
Reviewed-by: Alex Mastro amastro@fb.com Tested-by: Alex Mastro amastro@fb.com Reviewed-by: Raghavendra Rao Ananta rananta@google.com Signed-off-by: David Matlack dmatlack@google.com --- tools/testing/selftests/vfio/Makefile | 3 + .../testing/selftests/vfio/scripts/cleanup.sh | 41 +++++++ tools/testing/selftests/vfio/scripts/lib.sh | 42 +++++++ tools/testing/selftests/vfio/scripts/run.sh | 105 +----------------- tools/testing/selftests/vfio/scripts/setup.sh | 48 ++++++++ 5 files changed, 140 insertions(+), 99 deletions(-) create mode 100755 tools/testing/selftests/vfio/scripts/cleanup.sh create mode 100755 tools/testing/selftests/vfio/scripts/lib.sh create mode 100755 tools/testing/selftests/vfio/scripts/setup.sh
diff --git a/tools/testing/selftests/vfio/Makefile b/tools/testing/selftests/vfio/Makefile index ad7fa4071c20..a29f99395206 100644 --- a/tools/testing/selftests/vfio/Makefile +++ b/tools/testing/selftests/vfio/Makefile @@ -4,7 +4,10 @@ TEST_GEN_PROGS += vfio_iommufd_setup_test TEST_GEN_PROGS += vfio_pci_device_test TEST_GEN_PROGS += vfio_pci_driver_test
+TEST_FILES += scripts/cleanup.sh +TEST_FILES += scripts/lib.sh TEST_FILES += scripts/run.sh +TEST_FILES += scripts/setup.sh
include ../lib.mk include lib/libvfio.mk diff --git a/tools/testing/selftests/vfio/scripts/cleanup.sh b/tools/testing/selftests/vfio/scripts/cleanup.sh new file mode 100755 index 000000000000..69c922d8aafb --- /dev/null +++ b/tools/testing/selftests/vfio/scripts/cleanup.sh @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0-or-later + +source $(dirname -- "${BASH_SOURCE[0]}")/lib.sh + +function cleanup_devices() { + local device_bdf + local device_dir + + for device_bdf in "$@"; do + device_dir=${DEVICES_DIR}/${device_bdf} + + if [ -f ${device_dir}/vfio-pci ]; then + unbind ${device_bdf} vfio-pci + fi + + if [ -f ${device_dir}/driver_override ]; then + clear_driver_override ${device_bdf} + fi + + if [ -f ${device_dir}/driver ]; then + bind ${device_bdf} $(cat ${device_dir}/driver) + fi + + if [ -f ${device_dir}/sriov_numvfs ]; then + set_sriov_numvfs ${device_bdf} $(cat ${device_dir}/sriov_numvfs) + fi + + rm -rf ${device_dir} + done +} + +function main() { + if [ $# = 0 ]; then + cleanup_devices $(ls ${DEVICES_DIR}) + rmdir ${DEVICES_DIR} + else + cleanup_devices "$@" + fi +} + +main "$@" diff --git a/tools/testing/selftests/vfio/scripts/lib.sh b/tools/testing/selftests/vfio/scripts/lib.sh new file mode 100755 index 000000000000..9f05f29c7b86 --- /dev/null +++ b/tools/testing/selftests/vfio/scripts/lib.sh @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0-or-later + +readonly DEVICES_DIR="${TMPDIR:-/tmp}/vfio-selftests-devices" + +function write_to() { + # Unfortunately set -x does not show redirects so use echo to manually + # tell the user what commands are being run. + echo "+ echo "${2}" > ${1}" + echo "${2}" > ${1} +} + +function get_driver() { + if [ -L /sys/bus/pci/devices/${1}/driver ]; then + basename $(readlink -m /sys/bus/pci/devices/${1}/driver) + fi +} + +function bind() { + write_to /sys/bus/pci/drivers/${2}/bind ${1} +} + +function unbind() { + write_to /sys/bus/pci/drivers/${2}/unbind ${1} +} + +function set_sriov_numvfs() { + write_to /sys/bus/pci/devices/${1}/sriov_numvfs ${2} +} + +function get_sriov_numvfs() { + if [ -f /sys/bus/pci/devices/${1}/sriov_numvfs ]; then + cat /sys/bus/pci/devices/${1}/sriov_numvfs + fi +} + +function set_driver_override() { + write_to /sys/bus/pci/devices/${1}/driver_override ${2} +} + +function clear_driver_override() { + set_driver_override ${1} "" +} diff --git a/tools/testing/selftests/vfio/scripts/run.sh b/tools/testing/selftests/vfio/scripts/run.sh index 0476b6d7adc3..91fd38f9f6f6 100755 --- a/tools/testing/selftests/vfio/scripts/run.sh +++ b/tools/testing/selftests/vfio/scripts/run.sh @@ -1,109 +1,16 @@ # SPDX-License-Identifier: GPL-2.0-or-later
-# Global variables initialized in main() and then used during cleanup() when -# the script exits. -declare DEVICE_BDF -declare NEW_DRIVER -declare OLD_DRIVER -declare OLD_NUMVFS -declare DRIVER_OVERRIDE - -function write_to() { - # Unfortunately set -x does not show redirects so use echo to manually - # tell the user what commands are being run. - echo "+ echo "${2}" > ${1}" - echo "${2}" > ${1} -} - -function bind() { - write_to /sys/bus/pci/drivers/${2}/bind ${1} -} - -function unbind() { - write_to /sys/bus/pci/drivers/${2}/unbind ${1} -} - -function set_sriov_numvfs() { - write_to /sys/bus/pci/devices/${1}/sriov_numvfs ${2} -} - -function set_driver_override() { - write_to /sys/bus/pci/devices/${1}/driver_override ${2} -} - -function clear_driver_override() { - set_driver_override ${1} "" -} - -function cleanup() { - if [ "${NEW_DRIVER}" ]; then unbind ${DEVICE_BDF} ${NEW_DRIVER} ; fi - if [ "${DRIVER_OVERRIDE}" ]; then clear_driver_override ${DEVICE_BDF} ; fi - if [ "${OLD_DRIVER}" ]; then bind ${DEVICE_BDF} ${OLD_DRIVER} ; fi - if [ "${OLD_NUMVFS}" ]; then set_sriov_numvfs ${DEVICE_BDF} ${OLD_NUMVFS} ; fi -} - -function usage() { - echo "usage: $0 [-d segment:bus:device.function] [-s] [-h] [cmd ...]" >&2 - echo >&2 - echo " -d: The BDF of the device to use for the test (required)" >&2 - echo " -h: Show this help message" >&2 - echo " -s: Drop into a shell rather than running a command" >&2 - echo >&2 - echo " cmd: The command to run and arguments to pass to it." >&2 - echo " Required when not using -s. The SBDF will be " >&2 - echo " appended to the argument list." >&2 - exit 1 -} +source $(dirname -- "${BASH_SOURCE[0]}")/lib.sh
function main() { - local shell - - while getopts "d:hs" opt; do - case $opt in - d) DEVICE_BDF="$OPTARG" ;; - s) shell=true ;; - *) usage ;; - esac - done - - # Shift past all optional arguments. - shift $((OPTIND - 1)) - - # Check that the user passed in the command to run. - [ ! "${shell}" ] && [ $# = 0 ] && usage - - # Check that the user passed in a BDF. - [ "${DEVICE_BDF}" ] || usage - - trap cleanup EXIT - set -e + local device_bdfs=$(ls ${DEVICES_DIR})
- test -d /sys/bus/pci/devices/${DEVICE_BDF} - - if [ -f /sys/bus/pci/devices/${DEVICE_BDF}/sriov_numvfs ]; then - OLD_NUMVFS=$(cat /sys/bus/pci/devices/${DEVICE_BDF}/sriov_numvfs) - set_sriov_numvfs ${DEVICE_BDF} 0 - fi - - if [ -L /sys/bus/pci/devices/${DEVICE_BDF}/driver ]; then - OLD_DRIVER=$(basename $(readlink -m /sys/bus/pci/devices/${DEVICE_BDF}/driver)) - unbind ${DEVICE_BDF} ${OLD_DRIVER} + if [ -z "${device_bdfs}" ]; then + echo "No devices found, skipping." + exit 4 fi
- set_driver_override ${DEVICE_BDF} vfio-pci - DRIVER_OVERRIDE=true - - bind ${DEVICE_BDF} vfio-pci - NEW_DRIVER=vfio-pci - - echo - if [ "${shell}" ]; then - echo "Dropping into ${SHELL} with VFIO_SELFTESTS_BDF=${DEVICE_BDF}" - VFIO_SELFTESTS_BDF=${DEVICE_BDF} ${SHELL} - else - "$@" ${DEVICE_BDF} - fi - echo + "$@" ${device_bdfs} }
main "$@" diff --git a/tools/testing/selftests/vfio/scripts/setup.sh b/tools/testing/selftests/vfio/scripts/setup.sh new file mode 100755 index 000000000000..49a499e51cbe --- /dev/null +++ b/tools/testing/selftests/vfio/scripts/setup.sh @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +set -e + +source $(dirname -- "${BASH_SOURCE[0]}")/lib.sh + +function main() { + local device_bdf + local device_dir + local numvfs + local driver + + if [ $# = 0 ]; then + echo "usage: $0 segment:bus:device.function ..." >&2 + exit 1 + fi + + for device_bdf in "$@"; do + test -d /sys/bus/pci/devices/${device_bdf} + + device_dir=${DEVICES_DIR}/${device_bdf} + if [ -d "${device_dir}" ]; then + echo "${device_bdf} has already been set up, exiting." + exit 0 + fi + + mkdir -p ${device_dir} + + numvfs=$(get_sriov_numvfs ${device_bdf}) + if [ "${numvfs}" ]; then + set_sriov_numvfs ${device_bdf} 0 + echo ${numvfs} > ${device_dir}/sriov_numvfs + fi + + driver=$(get_driver ${device_bdf}) + if [ "${driver}" ]; then + unbind ${device_bdf} ${driver} + echo ${driver} > ${device_dir}/driver + fi + + set_driver_override ${device_bdf} vfio-pci + touch ${device_dir}/driver_override + + bind ${device_bdf} vfio-pci + touch ${device_dir}/vfio-pci + done +} + +main "$@"