#!/usr/bin/env expect
############################################################################
# Purpose:  Test gres.conf-specified and system-detected GPU device merging
#           logic
############################################################################
# Copyright (C) 2018 SchedMD LLC
# Written by Michael Hinton
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set exit_code		0
set test_prog		"test$test_id.prog"
set cfgdir		"test${test_id}_configs"
set cfgpath		"[pwd]/$cfgdir"
set nodename_base	"tux"
set nodename		"${nodename_base}0"
set nodename_diff	"${nodename_base}1"
set test_ulong_prog	"test$test_id.ulong"
set file_in		"test$test_id.bash"
set file_out		"test$test_id.out"
# Assume unsigned long is 64 bits unless overwritten
set ulong_bits  64
set dup_err		"error: gpu duplicate device file name"
set cpu_set_err		"fatal: _set_cpu_set_bitstr: cpu_set_bitstr is empty"
set mismatch_err	"error: This GPU specified in \\\[slurm\\\|gres\\\].conf has mismatching Cores or Links"
set no_name_err		"error: Invalid GRES data, no type name"
set parse_err		"error: Parse error in file"
set links_err		"error: gres.conf: Ignoring invalid Link"
set conf_mismatch_err	"WARNING: A line in gres.conf for GRES $re_word_str has $number more configured than expected in slurm.conf"
set slurm_conf_type_err	"GRES in slurm.conf have a type while others do not"
set gres_conf_type_err	"fatal: gres.conf for $re_word_str, some records have \"Type=\" specification while others do not"
set gres_conf_file_err	"fatal: gres.conf for $re_word_str, some records have \"File\" specification while others do not"

set debug_mode 0
# set debug_mode 1

if {[test_cray]} {
#	Many Cray-specific changes required in slurm.conf test files
	skip "This test can not run on a Cray system"
}

# Run the test under valgrind if any arguments are given
set use_valgrind 0
set test_minor_offset 0
set test_minor_in ""

if {[lindex $argv 0] == "v" || [lindex $argv 0] == "valgrind"} {
	set use_valgrind 1
	set test_minor_offset 1
	log_info "Running tests under valgrind!"
}

if {[lindex $argv $test_minor_offset] != ""} {
	set test_minor_in [lindex $argv $test_minor_offset]
	log_info "Running test $test_id.$test_minor_in "
}

proc mkdir {dir} {
	exec mkdir -p $dir
}

proc touch_file {file} {
	exec touch $file
}

proc generate_file {text filepath} {
	exec echo $text > $filepath
}

# Create or overwrite the slurm.conf file used by the test runner program
proc create_slurm_conf {} {
	global test_id cfgpath
	# Contain slurm.conf data to this file
	# This is all we need to trigger loading the GRES GPU plugin
	set slurm_conf "
	# This file was autogenerated by test$test_id
	ControlMachine=test_machine
	ClusterName=test_cluster
	GresTypes=gpu,mps,nic,mic,tmpdisk
	"

	# NOTE: There are no node definitions - node name and node gres info
	# are passed into the test program directly, not through this slurm.conf

	generate_file $slurm_conf $cfgpath/slurm.conf
}

mkdir $cfgpath
create_slurm_conf
# Set up dummy device files for testing. They just need to exist
set dev "$cfgpath/nvidia"
set dev0 "${dev}0"
set dev1 "${dev}1"
set dev2 "${dev}2"
set dev3 "${dev}3"
set dev4 "${dev}4"
set dev5 "${dev}5"
set dev6 "${dev}6"
set dev7 "${dev}7"
set dev8 "${dev}8"
touch_file $dev0
touch_file $dev1
touch_file $dev2
touch_file $dev3
touch_file $dev4
touch_file $dev5
touch_file $dev6
touch_file $dev7
touch_file $dev8


# Build test program
if {$use_valgrind} {
	compile_against_libslurm ${test_prog} 1 "-DUSING_VALGRIND"
} else {
	compile_against_libslurm ${test_prog} 1
}
fail_on_error "Cannot compile test program"

# Build program to determine size of unsigned long in C
set build_cmd "$bin_cc ${test_ulong_prog}.c -o ${test_ulong_prog}"
log_debug "$build_cmd"
eval exec $build_cmd

spawn ./$test_ulong_prog
expect {
	-re "($number)" {
		set ulong_bits $expect_out(1,string)
		exp_continue
	}
	timeout {
		fail "No response from $test_ulong_prog"
	}
	eof {
		wait
	}
}

log_debug "Sizeof(unsigned long) = $ulong_bits"


#
# Tests if $test_prog actual output matches expected output
# NOTE: The output strings are sorted so the order of lines is irrelevant
#
# test_minor -		The minor test number (39.18.X). If global variable
# 			"$test_minor_in" is set, then run only that test.
# slurm_conf_gres -     This is the `Gres=` value that would be in
# 			slurm.conf for this node (it's easier to pass it into
# 			the program directly than to regenerate a slurm.conf
# 			file with different node definitions).
# gres_conf -		The gres.conf to use.
# fake_gpus_conf -	The fake_gpus.conf to use. This file tells Slurm to
# 			pretend to have devices detected by system, in effect
# 			emulating the result of AutoDetect=nvml.
# output_expected -	The expected "GRES_PARSABLE" output.
# err_msgs (optional) -	The error message regex to look for. Use | for multiple.
# 			If using with $output_expected, it will only reliably
# 			match if $err_msgs appear BEFORE lines with
# 			GRES_PARSABLE.
# errs_expected (optional) - The number of times you expect to see $err_msgs in
# 			the output. Defaults to 0. If this number isn't hit,
# 			then the test will fail.
#
set failed_tests [list]
proc check_configuration {test_minor slurm_conf_gres gres_conf fake_gpus_conf output_expected {err_msgs ""} {errs_expected 0} } {
	global cfgpath nodename test_prog re_word_str exit_code test_minor_in
	global test_id failed_tests use_valgrind bin_sort file_in file_out
	global bin_cat debug_mode

	# Skip all tests not specified by the tester; force string comparison
	if {$test_minor_in != "" && [string compare "$test_minor" "$test_minor_in"] != 0} {
		return
	}

	set failed 0
	log_info "====test$test_id.$test_minor===="

	generate_file $gres_conf $cfgpath/gres.conf
	generate_file $fake_gpus_conf $cfgpath/fake_gpus.conf

	# We don't want to sort when debugging - we want to see output in order
	if {$debug_mode} {
		set output_method "$bin_cat $file_out"
	} else {
		set output_method "$bin_sort <./$file_out"
	}

	if {$use_valgrind} {
		make_bash_script $file_in "
valgrind --tool=memcheck --error-limit=no --leak-check=full --show-reachable=yes --max-stackframe=16777216 --num-callers=20 --child-silent-after-fork=yes --track-origins=yes ./$test_prog \"$cfgpath\" \"$nodename\" \"$slurm_conf_gres\" >./$file_out 2>&1
$output_method
rm -rf ./$file_out"
	} else {
		make_bash_script $file_in "
./$test_prog \"$cfgpath\" \"$nodename\" \"$slurm_conf_gres\" >./$file_out 2>&1
$output_method
rm -rf ./$file_out"
	}

	##
	### Parse the `GRES_PARSABLE` output
	##
	set sorted_actual ""

	if {!$debug_mode} {
		log_user 0
	}
	spawn ./$file_in
	expect {
		-re "(GRES_PARSABLE$re_word_str)" {
			if {$sorted_actual == ""} {
				set sorted_actual "\n"
			}
			# Append string
			set sorted_actual "$sorted_actual$expect_out(1,string)\n"
			exp_continue
		}
		timeout {
			log_error "No response from $test_prog"
			set failed 1
		}
		eof {
			wait
		}
	}

	##
	### Run through output again to parse the errors (if needed)
	##
	# Do this in a separate expect loop to prevent the two unrelated regexes
	# colliding with each other.
	# When expect matches something in a chunk of text, if exp_continue is
	# called, it throws away everything up to and including that text and
	# then continues searching. This means multiple regexes in the same
	# expect loop can skip each other. So avoid this.
	set errs_actual 0
	set ignore_err_msgs 0
	# Doing a regex on an empty string stalls expect. So don't do it.
	if {$err_msgs != ""} {
		# We for sure don't want to print this out
		log_user 0
		spawn ./$file_in
		expect {
			-re "($err_msgs)" {
				log_debug "Found '$err_msgs'"
				if {$ignore_err_msgs == 0} {
					incr errs_actual
				}
				exp_continue
			}
			timeout {
				log_error "No response from $test_prog"
				set failed 1
			}
			eof {
				wait
			}
		}
	}

	set sorted_expected ""
	make_bash_script $file_in "/bin/echo \"$output_expected\" | $bin_sort"
	spawn ./$file_in
	expect {
		-re "(GRES_PARSABLE$re_word_str)" {
			if {$sorted_expected == ""} {
				set sorted_expected "\n"
			}
			# Append string
			set sorted_expected "$sorted_expected$expect_out(1,string)\n"
			exp_continue
		}
		timeout {
			log_error "No response from $test_prog"
			set failed 1
		}
		eof {
			wait
		}
	}
	file delete ./$file_in
	log_user 1

	# Check the output with what is expected
	log_debug "Actual (sorted): $sorted_actual"
	if {$sorted_actual != $sorted_expected} {
		log_debug "Expected (sorted): $sorted_expected"
		log_error "Expected output does not match actual output for sub-test $test_minor"
		set failed 1
	} else {
		log_info "(This is OK) Expected output does match actual output for sub-test $test_minor"
	}

	#  Check for expected errors, if applicable
	if {$err_msgs != "bogus"} {
		set response "Errors: Expected $errs_expected, found $errs_actual for error regex '$err_msgs' in sub-test $test_minor"
		if {$errs_actual == $errs_expected} {
			set response "(This is OK) ${response}"
			log_info $response
		} else {
			log_error $response
			set failed 1
		}
	}

	if {$failed == 1} {
		set exit_code 1
		lappend failed_tests $test_minor
	}
}

################################################################################
# NOTES:
################################################################################
# fake_gpus.conf is of the following format, with each line representing one
# GPU device:
#
#     type|cpu_count|cpu_range|links|device_files
#
# Optionally, cpu_range allows special keywords starting in `~` to aid in
# testing the conversion from an NVML cpu affinity array to Slurm bitstring.
#
# The GRES_PARSABLE output is of the following format:
#
# GRES_PARSABLE[name](device_count):type|cpu_count|cpu_range|links|device_files
#
# To better decipher the tests, temporarily increase the log verbosity in
# test39.18.prog.c. E.g.:
#
#     opts.stderr_level = LOG_LEVEL_DEBUG2;
#
# Note that there is a known issue with Expect where the regex parsing fails
# when the output is somewhat large. So to avoid random errors, each test output
# should be kept concise.
#
################################################################################



# ##############################################################################
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# ##############################################################################

# ##############################################################################
# # Test 0.2 - Type-less gpu specification in slurm.conf and empty gres.conf
# # NOTE the device numbers being out of order
# ##############################################################################

set slurm_conf_gres "gpu:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
tesla|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}4|
"
check_configuration 0.2 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.4 - Test empty and null identifiers in fake_gpus.conf
# ##############################################################################

set slurm_conf_gres "gpu:8"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|(null)|(null)|(null)
tesla|4|(null)|(null)|${dev}1
tesla|4|(null)|0-1|(null)
tesla|4|(null)|0-1|${dev}3
tesla|4|0-1|(null)|(null)
tesla|4|0-1|(null)|${dev}2
tesla|4|0-1|0-1|(null)
tesla|4|0-1|0-1|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|0-1|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|0-1|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|0-1|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|0-1|${dev}4|
"
check_configuration 0.4 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output



# ##############################################################################
# The following tests are NVML-specific.
# They will only execute if HAVE_NVML = 1 in config.h.
# In fake_gpus_conf, a CPU range starting with `~` will trigger the GPU plugin's
# gpu_p_test_cpu_conv(). In order for this to exercise gpu/nvml-specific code,
# explicitly specify `AutoDetect=nvml` in the ad-hoc gres.conf. If not,
# gpu/generic will be used and CPU ranges will be set to null, failing the tests
# ##############################################################################

if {[have_nvml] == 1} {
	if {$test_minor_in == ""} {
		log_info "Running NVML-specific tests:"
	}

# ##############################################################################
# # Test 0.6 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       Convert device CPU affinity of 20 ulongs with all bits set
# ##############################################################################

set arr_len 20
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.6 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.8 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       Convert device CPU affinity of 1 ulong with all bits set
# ##############################################################################

set arr_len 1
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.8 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.10 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity of 3 ulongs with all bits set
# ##############################################################################

set arr_len 3
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.10 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.12 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for max # of CPUs supported (2^15)
# ##############################################################################

set max_cpus [expr pow(2, 15)]
set arr_len [expr int($max_cpus/$ulong_bits)]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~max|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.12 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.14 - Test conversion of NVML-style CPU affinity array to Slurm bitstr
# 		Convert device CPU affinity for all zeros (expect this to fatal)
# ##############################################################################

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|0|~zero|(null)|${dev}1
"
set expected_output ""
set err_msgs $cpu_set_err
set expected_errs 1
check_configuration 0.14 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 0.16 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for CPU 0 set out of 16 total CPUs
# ##############################################################################

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|16|~one|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|16|0|(null)|${dev}1|
"
check_configuration 0.16 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.18 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for CPUs 0-1 set out of 4 total CPUs
# ##############################################################################

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|~three|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}1|
"
check_configuration 0.18 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.20 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for CPUs 8-15 set out of 16 CPUs
# ##############################################################################

set slurm_conf_gres "gpu:1"
set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|16|~half|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|16|8-15|(null)|${dev}1|
"
check_configuration 0.20 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# End NVML-specific tests
# ##############################################################################

} else {
	log_warn "====Omitting NVML-specific tests===="
}

# ##############################################################################
# # Test 0.22 - Test that when no GRES type is specified in slurm.conf,
#               the GRES type is NOT set to what the system detects, but rather
#               is set to NULL.
# ##############################################################################

set slurm_conf_gres "gpu:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}4|
"
check_configuration 0.22 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.24 - Test that the GRES type specified in slurm.conf overrides the
#               GRES type specified in the system, if it's a substring.
# ##############################################################################

set slurm_conf_gres "gpu:v100:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):v100|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):v100|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):v100|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):v100|4|2-3|(null)|${dev}4|
"
check_configuration 0.24 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 0.26 - Test that the GRES type specified in slurm.conf doesn't need
#               to match the case of the system-detected GRES type, and that it
#               can also match the first part.
# ##############################################################################

set slurm_conf_gres "gpu:TESLA_V100:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|2-3|(null)|${dev}4|
"
check_configuration 0.26 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 0.28 - Test that the GRES type specified in slurm.conf can match the
#               last part of the system-detected GRES type
# ##############################################################################

set slurm_conf_gres "gpu:pcie-16gb:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|2-3|(null)|${dev}4|
"
check_configuration 0.28 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.30 - Test that the GRES type specified in slurm.conf can match the
#               full system-detected GRES type
# ##############################################################################

set slurm_conf_gres "gpu:tesla_v100-pcie-16gb:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4|
"
check_configuration 0.30 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.32 - Test that GRES types work as expected when one GRES type is a
#               substring of another GRES type (e.g. k20m + k20m1). See bug 7345
# ##############################################################################

set slurm_conf_gres "gpu:k20m:1,gpu:k20m1:1,gpu:v100:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_k20m1|4|0-1|(null)|${dev}1
tesla_k20m|4|0-1|(null)|${dev}2
tesla_v100-sxm2-32gb|4|0-1|(null)|${dev}3
"
set expected_output "
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):k20m1|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):v100|4|0-1|(null)|${dev}3|
"
check_configuration 0.32 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.34 - Test that only the first system device that matches a conf gres
#               is used, and that the gres type is truncated.
# ##############################################################################

set slurm_conf_gres "gpu:p100:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
nvidia-p100|4|0-1|(null)|${dev}2
nvidia-p100|4|0-1|(null)|${dev}6
nvidia-p100|4|0-1|(null)|${dev}3
nvidia-p100|4|0-1|(null)|${dev}4
nvidia-p100|4|0-1|(null)|${dev}1
nvidia-p100|4|0-1|(null)|${dev}5
"
set expected_output "
GRES_PARSABLE\[gpu\](1):p100|4|0-1|(null)|${dev}1|
"
check_configuration 0.34 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.36 - Test that the shorter GRES types don't match against the longer
#               GRES types.
# ##############################################################################

set slurm_conf_gres "gpu:aaaa:1,gpu:a:1,gpu:aa:1,gpu:aaa:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
aa|4|0-1|(null)|${dev}2
aaa|4|0-1|(null)|${dev}1
a|4|0-1|(null)|${dev}3
aaaa|4|0-1|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):aa|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):a|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):aaaa|4|0-1|(null)|${dev}4|
"

check_configuration 0.36 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.38 - Test that the shorter GRES types don't match against the longer
#               GRES types prematurely.
# ##############################################################################

set slurm_conf_gres "gpu:k20:2,gpu:k20m:4"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
k20|4|0-1|(null)|${dev}2
k20m|4|0-1|(null)|${dev}1
k20|4|0-1|(null)|${dev}3
k20m|4|0-1|(null)|${dev}4
k20|4|0-1|(null)|${dev}5
k20m|4|0-1|(null)|${dev}7
k20|4|0-1|(null)|${dev}6
k20m|4|0-1|(null)|${dev}8
"
set expected_output "
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}7|
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}8|
"

check_configuration 0.38 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.39 - Show that shorter types will match against longer types if the
# 		longer types aren't used up, and that the longer type will be
# 		truncated accordingly.
# ##############################################################################

set slurm_conf_gres "gpu:k20:2,gpu:k20m:2"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
k20m|4|0-1|(null)|${dev}1
k20m|4|0-1|(null)|${dev}2
k20m|4|0-1|(null)|${dev}3
k20|4|0-1|(null)|${dev}4
k20|4|0-1|(null)|${dev}5
"
set expected_output "
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}4|
"

check_configuration 0.39 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.40 - Test GRES types that are the same length
# 		Note: In a real system, the controller would set the node to
# 		drain, since the node reports one less aaa GPU than the
# 		controller is expecting.
# ##############################################################################

set slurm_conf_gres "gpu:aaa:3,gpu:bbb:2,gpu:ccc:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
test-bbb-test|4|0-1|(null)|${dev}3
test-aaa-test|4|0-1|(null)|${dev}2
test-ccc-test|4|0-1|(null)|${dev}5
test-aaa-test|4|0-1|(null)|${dev}1
test-bbb-test|4|0-1|(null)|${dev}4
test-ccc-test|4|0-1|(null)|${dev}6
"
set expected_output "
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):bbb|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):bbb|4|0-1|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):ccc|4|0-1|(null)|${dev}5|
"

check_configuration 0.40 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 0.42 - Test multiple identical GRES type specifications
# ##############################################################################

set slurm_conf_gres "gpu:aaa:1,gpu:aaa:1,gpu:aaa:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
test-aaa-test|4|0-1|(null)|${dev}4
test-aaa-test|4|0-1|(null)|${dev}3
test-aaa-test|4|0-1|(null)|${dev}2
test-aaa-test|4|0-1|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}3|
"

check_configuration 0.42 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# Test 0.52 - Test slurm.conf, gres.conf, and detected device interaction
#
# The node returns a list of only 3 GPUs, not 4 (since gpu:special is not
# detected on the system). So the controller should set the node to drain.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|
"

check_configuration 0.52 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# Test 0.53 - Test slurm.conf, gres.conf, and detected device interaction
#
# gpu:special is added on via gres.conf. The final GPU list count matches what
# is expected in slurm.conf. No errors.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|
GRES_PARSABLE\[gpu\](1):special|4|(null)|(null)|${dev}5|
"

check_configuration 0.53 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# Test 0.54 - Test slurm.conf, gres.conf, and detected device interaction
#
# The gres conf record for nvidia1 *does* match a system GPU with the same Type
# and File. However, Cores is mismatched, so an error is printed in slurmd and
# the system device is omitted from the final GPU list. The total GPU count is 3
# instead of 4, so the controller sets the node to drain.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|
GRES_PARSABLE\[gpu\](1):special|4|(null)|(null)|${dev}5|
"

check_configuration 0.54 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# Test 0.55 - Test slurm.conf, gres.conf, and detected device interaction
#
# nvidia[0-2] matches exactly what is found on the system, so no problem there.
# tesla + nvidia3 does not match any type and file combo found in the system
# GPUs, so this is assumed to be an “extra” GPU. However, it is not added, since
# there are already 3 teslas found as outlined in slurm.conf.
# The total GPU count is 5 instead of 4, which is fine (i.e. won't set node to
# drain).
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[0-2\] Cores=0-1
Name=gpu Type=tesla File=${dev}3 Cores=0
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|
GRES_PARSABLE\[gpu\](1):special|4|(null)|(null)|${dev}5|
"
set err_msgs $conf_mismatch_err
set expected_errs 1

check_configuration 0.55 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# Test 0.56 - Test slurm.conf, gres.conf, and detected device interaction
#
# nvidia[0-2] are found on the system, but nvidia3 is not. However, nvidia3 is
# assumed to be an extra GPU (like gpu:special), so it’s ok.
# The total GPU count is 5, so there are no errors or warnings.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4,gpu:special:1"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[0-3\] Cores=0-1
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):special|4|(null)|(null)|${dev}5|
"

check_configuration 0.56 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.57 - Test that an empty slurm.conf yields no devices
# ##############################################################################

set slurm_conf_gres ""
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[0-3\] Cores=0-1
Name=gpu Type=special File=${dev}5
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output ""

set err_msgs $conf_mismatch_err
set expected_errs 2

check_configuration 0.57 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# Test 0.58 - Test that a non-GPU GRES doesn't need an explicit entry in
# gres.conf. Also test that GPUs are rejected unless they have a File
# specification in gres.conf when AutoDetect is turned off.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3,gpu:special:1,tmpdisk:100"
# Nothing in gres.conf
set gres_conf ""
# Effectively "turn off" AutoDetect
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[tmpdisk\](100):(null)|4|(null)|(null)|(null)|
"
check_configuration 0.58 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# Test 0.59 - Test over-specified gres.conf compared to slurm.conf
#
# This tests that a larger gres.conf line is truncated down to match what
# slurm.conf specifies. This also tests that user errors are printed whenever
# more GRES are found in gres.conf than specified in slurm.conf.
# ##############################################################################

set slurm_conf_gres "gpu:tesla:2,gpu:gtx:1"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[0-4\]
Name=gpu Type=tesla File=${dev}5
Name=gpu Type=special File=${dev}6
Name=gpu Type=gtx File=${dev}7
Name=nic Count=100
Name=tmpdisk Count=10G
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|-1|${dev}0
tesla|4|0-1|-1|${dev}1
tesla|4|0-1|-1|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|-1|${dev}1|
GRES_PARSABLE\[gpu\](1):gtx|4|(null)|(null)|${dev}7|
"

set err_msgs $conf_mismatch_err
set expected_errs 5

check_configuration 0.59 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs


# ##############################################################################
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# ##############################################################################



# ##############################################################################
# # Test 1.2 - Test that all MPS is distributed across multiple GPU types
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,gpu:1080:1,gpu:gtx560:1,mps:300"
set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
1080ti|4|0-1|(null)|${dev}2
gtx560|4|0-1|(null)|${dev}0
"
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0-1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):1080|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[mps\](100):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[mps\](100):1080|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[mps\](100):gtx560|4|0-1|(null)|${dev}0|
"

check_configuration 1.2 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 1.4 - Test that errors are emitted and configured devices omitted when
#	       Cores or Links mismatch with the corresponding system devices
# ##############################################################################

set slurm_conf_gres "gpu:ti:3,gpu:gtx:3"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=ti File=${dev}0 COREs=0
Name=gpu Type=ti File=${dev}\[1-2\] COREs=0-1
Name=gpu Type=gtx File=${dev}\[3-5\] COREs=0-1 Links=0,0,0,0,0,0
"

set fake_gpus_conf "
# This file was autogenerated by test$test_id
1080ti|4|0-1|(null)|${dev}0
1080ti|4|0-1|(null)|${dev}1
1080ti|4|0-1|(null)|${dev}2
gtx560|4|0-1|0,0,0,1,0,0|${dev}3
gtx560|4|0-1|0,0,0,0,1,0|${dev}4
gtx560|4|0-1|0,0,0,0,0,0|${dev}5
"

set expected_output "
GRES_PARSABLE\[gpu\](1):ti|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):ti|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):gtx|4|0-1|0,0,0,0,0,0|${dev}5|
"

set err_msgs $mismatch_err
set expected_errs 3
check_configuration 1.4 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs


# ##############################################################################
# # Test 1.6 - Test that "extra" GPUs are still used when not found on system
# ##############################################################################

set slurm_conf_gres "gpu:1080:3"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=1080 File=${dev}\[0-2\] COREs=0-1
"

set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):1080|4|0-1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):1080|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):1080|4|0-1|(null)|${dev}2|
"

check_configuration 1.6 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 1.8 - Test that separate "extra" GPUs in gres.conf with different Cores
# 	       and Links are properly accounted under the typeless slurm.conf
# 	       gres specification.
# ##############################################################################

set slurm_conf_gres "gpu:5"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu File=${dev}\[0-1\] Cores=0,1
Name=gpu File=${dev}\[2-3\] Cores=0,1 Links=-1
Name=gpu File=${dev}4 Cores=0
"

set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):(null)|4|0,1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0,1|-1|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|0,1|-1|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|0|(null)|${dev}4|
"

check_configuration 1.8 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 1.10 - Test that slurm.conf doesn't allow a mix of Type and no Type
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,gpu:1"
set gres_conf ""
set fake_gpus_conf ""
set expected_output ""
set err_msgs $slurm_conf_type_err
set expected_errs 1

check_configuration 1.10 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 1.12 - Test that gres.conf doesn't allow a mix of Type and no Type
# ##############################################################################

set slurm_conf_gres "gpu:tesla:2,gpu:v100:2"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu            File=${dev}\[0-1\]
Name=gpu Type=v100  File=${dev}\[2-3\]
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $gres_conf_type_err
set expected_errs 1

check_configuration 1.12 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 1.14 - Test that gres.conf doesn't allow a mix of File and no File
# ##############################################################################

set slurm_conf_gres "gpu:tesla:2,gpu:v100:2"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla  File=${dev}\[0-1\]
Name=gpu Type=v100
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $gres_conf_file_err
set expected_errs 1

check_configuration 1.14 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs


# ##############################################################################
# ------------------------------------------------------------------------------
# Test gres.conf parsing
# ------------------------------------------------------------------------------
# ##############################################################################


# ##############################################################################
# # Test 2.2 - Test gres/gpu plus gres/mps with count
# # NOTE the device numbers being out of order
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,gpu:gtx560:1,mps:200"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}1
Name=mps Count=100 File=${dev}0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2.2 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.4 - Test gres/mps with count and no file names
# # NOTE the device numbers being out of order
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,gpu:gtx560:1,mps:210"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=mps Count=210
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](105):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](105):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2.4 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.6 - Test using only a subset of the system devices found
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3
Name=gpu Type=tesla File=${dev}1 Cores=0-1
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
tesla|4|0-1|(null)|${dev}2
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 2.6 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.8 - Test disjoint sets of conf and system devices
# ##############################################################################

set slurm_conf_gres "gpu:tesla:6"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=2-3
Name=gpu Type=tesla File=${dev}2 Cores=2-3
Name=gpu Type=tesla File=${dev}3 Cores=2-3
Name=gpu Type=tesla File=${dev}4 Cores=2-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}5
tesla|4|0-1|(null)|${dev}6
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}6|
"
check_configuration 2.8 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.10 - Different links, different records
# ##############################################################################

# Devices 0-2 are all doubly linked to each other
# Device 5 is singly linked to 3-4
# Devices 7-8 are doubly linked to each other
set slurm_conf_gres "gpu:tesla:8"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}0 Cores=0-3 Links=-1,2,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=2,-1,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=2,2,-1,0,0,0,0,0
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,0,0,-1,0,1,0,0
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,0,0,0,-1,1,0,0
Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,0,0,1,1,-1,0,0
Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,0,0,0,0,0,-1,2
Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,0,0,0,0,0,2,-1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|-1,2,2,0,0,0,0,0|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,-1,2,0,0,0,0,0|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,2,-1,0,0,0,0,0|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,-1,0,1,0,0|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,-1,1,0,0|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,1,1,-1,0,0|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,-1,2|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,2,-1|${dev}7|
"
check_configuration 2.10 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.12 - Empty everything
# ##############################################################################

set slurm_conf_gres ""
set gres_conf ""
set fake_gpus_conf ""
set expected_output ""
check_configuration 2.12 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.14 - Empty system devices
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 2.14 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 2.16 - Test non-GPU GRESs with types
# ##############################################################################

set slurm_conf_gres "gpu:tesla:1,tmpdisk:disky:1,nic:nikki:1,mic:mickey:1"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu     Type=tesla File=${dev}1 Cores=0-3
Name=tmpdisk Type=disky File=${dev}2 Cores=0-3
Name=nic     Type=nikki File=${dev}3 Cores=0-3
Name=mic     Type=mickey File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[tmpdisk\](1):disky|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[nic\](1):nikki|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mic\](1):mickey|4|0-3|(null)|${dev}4|
"
check_configuration 2.16 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.17 - Test non-GPU GRESs without types
# ##############################################################################

set slurm_conf_gres "gpu:1,tmpdisk:1,nic:1,mic:1"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu     File=${dev}1 Cores=0-3
Name=tmpdisk File=${dev}2 Cores=0-3
Name=nic     File=${dev}3 Cores=0-3
Name=mic     File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[tmpdisk\](1):(null)|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[nic\](1):(null)|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mic\](1):(null)|4|0-3|(null)|${dev}4|
"
check_configuration 2.17 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.18 - Test NodeName when Name isn't specified
# ##############################################################################

set slurm_conf_gres "gpu:tesla:2"
set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=gpu     Type=tesla File=${dev}1 Cores=0-3
NodeName=$nodename              Type=tesla File=${dev}2 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
"

set err_msgs $no_name_err
set expected_errs 1
check_configuration 2.18 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.20 - Test that empty Name does not parse and produces error
# ##############################################################################

set slurm_conf_gres ""
set gres_conf "
# This file was autogenerated by test$test_id
Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $parse_err
set expected_errs 1
check_configuration 2.20 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.22 - Ensure no malloc error for large count with non-GPU GRES
#               See bug 6014
# ##############################################################################

set slurm_conf_gres "tmpdisk:10g"
set gres_conf "Name=tmpdisk count=10g"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[tmpdisk\](10737418240):(null)|4|(null)|(null)|(null)|
"
check_configuration 2.22 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.23 - Ensure no errors for large count with non-GPU GRES *and* with
# 		Type and Cores specified in gres.conf.
# ##############################################################################

set slurm_conf_gres "tmpdisk:tempy:10g"
set gres_conf "Name=tmpdisk Type=tempy Count=10g Cores=0-1"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[tmpdisk\](10737418240):tempy|4|0-1|(null)|(null)|
"
check_configuration 2.23 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.24 - Different types, different records
# ##############################################################################

set slurm_conf_gres "gpu:a:2,gpu:b:2"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=a File=${dev}1 Cores=0-3
Name=gpu Type=a File=${dev}2 Cores=0-3
Name=gpu Type=b File=${dev}5 Cores=0-3
Name=gpu Type=b File=${dev}6 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}6|
"
check_configuration 2.24 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.28 - Test alternating device/cpu ranges
# ##############################################################################

set slurm_conf_gres "gpu:tesla_a:4,gpu:tesla_b:4"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla_a File=${dev}5       Cores=0-3
Name=gpu Type=tesla_b File=${dev}6       Cores=0-3
Name=gpu Type=tesla_a File=${dev}7       Cores=0-3
Name=gpu Type=tesla_b File=${dev}8       Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}7|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}8|
"
check_configuration 2.28 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.30 - Test duplicate devices in gres.conf
# ##############################################################################

set slurm_conf_gres "gpu:tesla:6"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $dup_err
set expected_errs 1
check_configuration 2.30 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.32 - Test increasing duplicate devices in gres.conf
# ##############################################################################

set slurm_conf_gres "gpu:tesla:10"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1       Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $dup_err
set expected_errs 1
check_configuration 2.32 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.34 - Test decreasing duplicate devices in gres.conf
# ##############################################################################

set slurm_conf_gres "gpu:tesla:10"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $dup_err
set expected_errs 1
check_configuration 2.34 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.36 - Test duplicate devices
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}1 Cores=0-1
Name=gpu Type=tesla File=${dev}1 Cores=0-2
Name=gpu Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs $dup_err
set expected_errs 1
check_configuration 2.36 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.38 - Test overlapping Core ranges
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}2 Cores=0-1
Name=gpu Type=tesla File=${dev}3 Cores=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-2|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 2.38 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 2.40 - Work with null links
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=\"\"
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=null
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0|${dev}4|
"
set err_msgs $links_err
set expected_errs 1
check_configuration 2.40 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.42 - Show link ranges do NOT work - only commas
# ##############################################################################

set slurm_conf_gres "gpu:tesla:4"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0-1
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,-1
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,-1,2
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1,2|${dev}4|
"
check_configuration 2.42 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.46 - Type not specified
# ##############################################################################

set slurm_conf_gres "gpu:4"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu File=${dev}1 Cores=0-3
Name=gpu File=${dev}2 Cores=0-3
Name=gpu File=${dev}3 Cores=0-3
Name=gpu File=${dev}4 Cores=0-3

"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}4|
"
check_configuration 2.46 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.48 - Cores not specified or null
# ##############################################################################

set slurm_conf_gres "gpu:tesla:5"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1
Name=gpu Type=tesla File=${dev}2
Name=gpu Type=tesla File=${dev}3 Cores=\"\"
Name=gpu Type=tesla File=${dev}4 Cores=null
Name=gpu Type=tesla File=${dev}5 Cores=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4||(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|null|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}5|
"
check_configuration 2.48 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output


# TODO: What other tests should we add?
# Make sure current gres.conf files still work as expected
# Test to make sure CPU affinity is correct? Machine vs abstract?
# Invalid CPU counts
# Invalid cpu range

# ##############################################################################
# # Test 2.50 - Test examples in gres.conf docs
# ##############################################################################

set slurm_conf_gres "gpu:gtx560:1,gpu:tesla:1,mps:200"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}0
Name=mps Count=100 File=${dev}1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2.50 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.52 - Test examples in gres.conf docs
# ##############################################################################

set slurm_conf_gres "gpu:tesla:3"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[0-1\] COREs=0,1
# NOTE: nvidia2 device is out of service
# Name=gpu Type=tesla File=${dev}\[2-3\] COREs=2,3
Name=gpu Type=tesla File=${dev}3 COREs=2,3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}3|
"
check_configuration 2.52 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.54 - Test examples in gres.conf docs
# #		Note: ${nodename_base}0 == $nodename, so only node 0's GRES
# #		definitions will apply to this test
# ##############################################################################

set slurm_conf_gres "gpu:4"
set gres_conf "
# This file was autogenerated by test$test_id
## Explicitly specify devices on nodes ${nodename_base}0-${nodename_base}15
# NodeName=${nodename_base}\[0-15\] Name=gpu File=${dev}\[0-3\]
# NOTE: ${nodename_base}3 ${dev}1 device is out of service
NodeName=${nodename_base}\[0-2\] Name=gpu File=${dev}\[0-3\]
NodeName=${nodename_base}3 Name=gpu File=${dev}\[0,2-3\]
NodeName=${nodename_base}\[4-15\] Name=gpu File=${dev}\[0-3\]
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}3|
"
check_configuration 2.54 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.56 - Test gres/mps with count on only some devices
# # NOTE the device numbers being out of order, one GPU without a gres/mps and
# #      a gres/mps with a device file not valid for any configured GPU
# ##############################################################################

set slurm_conf_gres "gpu:gtx560:1,gpu:tesla:2,mps:900"
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}3 COREs=2,3
Name=mps Count=200 File=${dev}1
Name=mps Count=300 File=${dev}3
Name=mps Count=400 File=${dev}8
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](200):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](300):tesla|4|2,3|(null)|${dev}3|
"
check_configuration 2.56 $slurm_conf_gres $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Tests for expected failures
# ##############################################################################



# ##############################################################################
# # Cleanup and finish
# ##############################################################################

if {$exit_code != 0} {
	set message "Tests $test_id.\["
	set length [llength $failed_tests]
	set i 0
	foreach test_minor $failed_tests {
		set i [expr $i+1]
		append message "$test_minor"
		if {$i < $length} {
			append message ","
		}
	}
	append message "\] failed"
	if {$test_minor_in == ""} {
		append message ". To run an individual sub-test, do `expect test$test_id <sub-test>`"
	}
	log_error $message
}

# Delete generated files
file delete $cfgpath/gres.conf
file delete $cfgpath/fake_gpus.conf
file delete $cfgpath/slurm.conf
file delete $test_prog
file delete $test_ulong_prog
file delete $dev0
file delete $dev1
file delete $dev2
file delete $dev3
file delete $dev4
file delete $dev5
file delete $dev6
file delete $dev7
file delete $dev8

# Delete generated folders
file delete $cfgpath

if {$exit_code != 0} {
	fail "Test failed due to previous errors (\$exit_code = $exit_code)"
}