#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          Test sacctmgr update job [condition] set newwckey="newkey"
############################################################################
# Copyright (C) 2019 SchedMD LLC
# Written by Albert Gil <albert.gil@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals
source ./globals_accounting

set exit_code     0

# globals
set access_err    0
set timeout 120

set sql_in_clus1  "$test_name-clus1-in.sql"
set sql_rem_clus1 "$test_name-clus1-rem.sql"
set sql_in_clus2  "$test_name-clus2-in.sql"
set sql_rem_clus2 "$test_name-clus2-rem.sql"

set cluster1      [format "%s%s" $test_name "clus1"]
set cluster2      [format "%s%s" $test_name "clus2"]
set account1      [format "%s%s" $test_name "acct1"]
set account2      [format "%s%s" $test_name "acct2"]
set user1         [get_my_user_name]
set user2         "root"
set wckey1        [format "%s%s" $test_name "wckey1"]
set wckey2        [format "%s%s" $test_name "wckey2"]
set wckey_new     [format "%s%s" $test_name "wckeynew"]

set clusters      [format "%s,%s" $cluster1 $cluster2]
set accounts      [format "%s,%s" $account1 $account2]
set users         [format "%s,%s" $user1    $user2]
set wckeys        [format "%s,%s" $wckey1   $wckey2]

# 8 jobs to test combinations of 2 accounts, 2 users and 2 wckeys
# same job ids will be used for the 2 clusters
set jobid01       "1001"
set jobid02       "1002"
set jobid03       "1003"
set jobid04       "1004"
set jobid05       "1005"
set jobid06       "1006"
set jobid07       "1007"
set jobid08       "1008"

###############################################################################
#
# Proc: Loads the remove sql entries and remove the added users, accounts and
#       clusters.
#
###############################################################################
proc cleanup {} {
	global exit_code bin_rm
	global sql_in_clus1 sql_in_clus2
	global sql_rem_clus1 sql_rem_clus2
	global clusters accounts users

	incr exit_code [archive_load $sql_rem_clus1]
	incr exit_code [archive_load $sql_rem_clus2]
	remove_user    $clusters $accounts $users
	remove_acct    "" $accounts
	remove_cluster $clusters

	exec $bin_rm $sql_in_clus1 $sql_in_clus2 $sql_rem_clus1 $sql_rem_clus2
}


###############################################################################
#
# Proc: Creates the sql files to load 8 test jobs, and to remove them.
#       Each job is a unique combination of 2 accounts, 2 users and 2 wckeys
#
# Input: cluster: cluster name to use
#        sql_in : filename to save the creation file
#        sql_rem: filename to save the remove file
#
# TODO: Error handling.
###############################################################################
proc create_sql {cluster sql_in sql_rem} {

	global bin_rm
	global exit_code test_name
	global account1 account2
	global wckey1 wckey2
	global user1acct1 user1acct2
	global user2acct1 user2acct2
	global user1wckey1 user1wckey2
	global user2wckey1 user2wckey2
	global jobid01 jobid02 jobid03 jobid04 jobid05 jobid06 jobid07 jobid08

	set node_list     [format "%s%s" $cluster "\[0-1\]"]
	set job_name      "$test_name\_job"
	set cluster_cpus  4

	set uid [get_my_uid]
	set gid [get_my_gid]

	# Mon Dec 31 23:00:00 2007
	set start_date "Mon Dec 31 23:00:00 2007"
	set period_start [exec date -d $start_date +%s]

	# Thu Jan 31 23:59:59 2008
	set end_date "Thu Jan 31 23:59:59 2008"
	set period_end [exec date -d $end_date +%s]

	# DON'T MESS WITH THIS UNLESS YOU REALLY UNDERSTAND WHAT YOU ARE DOING!!!!!
	# THIS COULD SERIOUSLY MESS UP YOUR DATABASE IF YOU ALTER THIS INCORRECTLY
	# JUST A FRIENDLY REMINDER ;)

	# create the in sql file
	exec $bin_rm -f $sql_in
	set file [open $sql_in "w"]

	# put in the cluster for back in the day before accounting was made here for us we are using 'Tue Jan  1 00:00:00 2008' = 1199174400 as the start
	puts $file "insert into cluster_event_table (node_name, cluster, tres, period_start, period_end, reason, cluster_nodes) values"
	puts $file "('', '$cluster', '1=$cluster_cpus', $period_start, $period_end, 'Cluster processor count', '$node_list' )"
	puts $file "on duplicate key update period_start=VALUES(period_start), period_end=VALUES(period_end);"

	# add jobs
	puts $file "insert into job_table (jobid, associd, wckey, wckeyid, uid, gid, `partition`, blockid, cluster, account, eligible, submit, start, end, suspended, name, track_steps, state, comp_code, priority, req_cpus, tres_alloc, nodelist, kill_requid, qos, deleted) values"
	puts $file "('$jobid01', '$user1acct1', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid02', '$user1acct1', '$wckey2', '$user1wckey2', '$uid', '$gid', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid03', '$user2acct1', '$wckey1', '$user2wckey1',    '0',    '0', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid04', '$user2acct1', '$wckey2', '$user2wckey2',    '0',    '0', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"

	puts $file "('$jobid05', '$user1acct2', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid06', '$user1acct2', '$wckey2', '$user1wckey2', '$uid', '$gid', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid07', '$user2acct2', '$wckey1', '$user2wckey1',    '0',    '0', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid08', '$user2acct2', '$wckey2', '$user2wckey2',    '0',    '0', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "on duplicate key update id=LAST_INSERT_ID(id), eligible=VALUES(eligible), submit=VALUES(submit), start=VALUES(start), end=VALUES(end), associd=VALUES(associd), tres_alloc=VALUES(tres_alloc), wckey=VALUES(wckey), wckeyid=VALUES(wckeyid);";
	close $file

	# create the remove sql file
	exec $bin_rm -f $sql_rem
	set file [open $sql_rem "w"]

	puts $file [format "%s%s%s" "truncate table \"" $cluster "_event_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_job_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_step_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_day_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_hour_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_month_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_day_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_hour_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_month_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_day_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_hour_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_month_table\";"]
	close $file
}


###############################################################################
#
# Proc: Runs the desired "sacctmgr update job" command and checkd the expected
#       output.
#
# Returns: 0 if result was the expected, non-zero otherwise.
#
# Input: sacctmgr_args: the "where" args to the update job command.
#        expected_jobs: the list of job ids expected to be updated.
#        sql_rem: filename to save the remove file
#
###############################################################################
proc test_job_update {sacctmgr_args expected_jobs} {
	global sacct sacctmgr number
	global clusters wckey_new

	set matches_right 0
	set matches_wrong 0
	set rc 0

	set cmd_str "$sacctmgr -i update job set newwckey=$wckey_new where $sacctmgr_args"
	set my_pid [eval spawn [lindex $cmd_str 0] {*}[lrange $cmd_str 1 end]]
	expect {
		-re "($number) submitted at" {
			set jobid $expect_out(1,string)
			if {"$jobid" in $expected_jobs} {
				incr matches_right
			} else {
				incr matches_wrong
			}
			exp_continue
		}
		timeout {
			log_error "sacctmgr update job not responding"
			slow_kill $my_pid
			incr rc
			return $rc
		}
		eof {
			wait
		}
	}
	if {$matches_right != [llength $expected_jobs] || $matches_wrong != 0} {
		log_error "Expected job updates wrong $matches_right $matches_wrong"
		incr rc
		return $rc
	}

	set matches_right 0
	set matches_wrong 0
	set my_pid [eval spawn $sacct -a -n --cluster=$clusters --wckey=$wckey_new --start=2000-01-01 --end=now --format=jobid]
	expect {
		-re "($number)" {
			set jobid $expect_out(1,string)
			if {"$jobid" in $expected_jobs} {
				incr matches_right
			} else {
				incr matches_wrong
			}
			exp_continue
		}
		timeout {
			log_error "sacct not responding"
			slow_kill $my_pid
			incr rc
			return $rc
		}
		eof {
			wait
		}
	}
	if {$matches_right != [llength $expected_jobs] || $matches_wrong != 0} {
		log_error "Expected job updates wrong $matches_right $matches_wrong"
		incr rc
		return $rc
	}
	return $rc
}

proc _run_test { update_str expected_jobs } {
	global sql_rem_clus1 sql_rem_clus2 sql_in_clus1 sql_in_clus2
	global exit_code

	incr exit_code [archive_load $sql_rem_clus1]
	incr exit_code [archive_load $sql_rem_clus2]
	incr exit_code [archive_load $sql_in_clus1]
	incr exit_code [archive_load $sql_in_clus2]
	fail_on_error "Unable to load archive"

	incr exit_code [test_job_update $update_str $expected_jobs]
	fail_on_error "Update failed with: $update_str"
}

#
# Check configuration
# TODO: chack also slurm.conf!!
#
if { [test_account_storage] == 0 } {
	skip "This test can't be run without a usable AccountStorageType"
}

set wc_key_track [check_wckey_tracking_slurmdbd]
if { $wc_key_track == 0 } {
	skip "This test can't be run without TrackWCKey set in slurmdbd.conf"
}

if { [string compare [check_accounting_admin_level] "Administrator"] } {
	skip "This test can't be run without being an Accounting administrator.\nUse: sacctmgr mod user \$USER set admin=admin"
}

# Cluster Request
array set clus_req {}

# Account Request
array set acct_req {}
set acct_req(cluster) $clusters

# User Request
array set user_req {}
set user_req(cluster) $clusters
set user_req(account) $accounts
set user_req(wckey)   $wckeys

#
# Remove previous test clusters, accounts and users
#
remove_user    $clusters $accounts $users
remove_acct    "" $accounts
remove_cluster $clusters

# Add cluster
incr exit_code [add_cluster $cluster1 [array get clus_req]]
incr exit_code [add_cluster $cluster2 [array get clus_req]]
fail_on_error "Unable to add clusters"

# Add accounts
incr exit_code [add_acct "$account1,$account2" [array get acct_req]]
fail_on_error "Unable to add accounts"

# Add users
incr exit_code [add_user $users [array get user_req]]
fail_on_error "Unable to add users"

#
# Get the association ids
# TODO: Assuming cluster1 and cluster2 has the same assoc ids
#
set user1acct1 0
set user1acct2 0
set user2acct1 0
set user2acct2 0
set my_pid [eval spawn $sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format="User,account,id"]
expect {
	-re "There was a problem" {
		log_error "there was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user1.$account1.($number)." {
		set user1acct1 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr list associations not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}
set my_pid [eval spawn $sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format="User,account,id"]
expect {
	-re "There was a problem" {
		log_error "there was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user1.$account2.($number)." {
		set user1acct2 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr list associations not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}
set my_pid [eval spawn $sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format="User,account,id"]
expect {
	-re "There was a problem" {
		log_error "there was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user2.$account1.($number)." {
		set user2acct1 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr list associations not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}
set my_pid [eval spawn $sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format="User,account,id"]
expect {
	-re "There was a problem" {
		log_error "there was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user2.$account2.($number)." {
		set user2acct2 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr list associations not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {!$user1acct1 || !$user1acct2 || !$user2acct1 || !$user2acct2} {
	fail "Didn't get one of the user associations $user1acct1 $user1acct2 $user2acct1 $user2acct2."
}

#
# Get the wckeys ids
# TODO: Assuming cluster1 and cluster2 has the same wckeys ids
#
set user1wckey1 0
set user2wckey1 0
set user1wckey2 0
set user2wckey2 0
set my_pid [eval spawn $sacctmgr -n -p list wckeys users=$users cluster=$clusters format="user,wckey,id"]
expect {
	-re "There was a problem" {
		log_error "There was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user1.$wckey1.($number)." {
		set user1wckey1 $expect_out(1,string)
		exp_continue
	}
	-re "$user2.$wckey1.($number)." {
		set user2wckey1 $expect_out(1,string)
		exp_continue
	}
	-re "$user1.$wckey2.($number)." {
		set user1wckey2 $expect_out(1,string)
		exp_continue
	}
	-re "$user2.$wckey2.($number)." {
		set user2wckey2 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr wckeys not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {!$user1wckey1 || !$user2wckey1 || !$user1wckey2 || !$user2wckey2} {
	fail "Didn't get one of the user wckeys $user1wckey1 $user2wckey1 $user1wckey2 $user2wckey2."
}

#
# Create the SQL files
#
create_sql $cluster1 $sql_in_clus1 $sql_rem_clus1
create_sql $cluster2 $sql_in_clus2 $sql_rem_clus2

#
# Tests
#

# 1 cluster1, 1 user1
_run_test "cluster=$cluster1 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster1, 1 user2
_run_test "cluster=$cluster1 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster1, 2 users
_run_test "cluster=$cluster1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid05 $jobid06 $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster2, 1 user1
_run_test "cluster=$cluster2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster2, 1 user2
_run_test "cluster=$cluster2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster2, 2 users
_run_test "cluster=$cluster2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid05 $jobid06 $jobid03 $jobid04 $jobid07 $jobid08]


# 1 cluster1, 1 account1, 1 user1
_run_test "cluster=$cluster1 account=$account1 user=$user1" [list $jobid01 $jobid02]

# 1 cluster1, 1 account1, 1 user2
_run_test "cluster=$cluster1 account=$account1 user=$user2" [list $jobid03 $jobid04]

# 1 cluster1, 1 account1, 2 users
_run_test "cluster=$cluster1 account=$account1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster1, 1 account2, 1 user1
_run_test "cluster=$cluster1 account=$account2 user=$user1" [list $jobid05 $jobid06]

# 1 cluster1, 1 account2, 1 user2
_run_test "cluster=$cluster1 account=$account2 user=$user2" [list $jobid07 $jobid08]

# 1 cluster1, 1 account2, 2 users
_run_test "cluster=$cluster1 account=$account2 user=$user1,$user2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster1, 2 accounts, 1 user1
_run_test "cluster=$cluster1 account=$account1,$account2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster1, 2 accounts, 1 user2
_run_test "cluster=$cluster1 account=$account1,$account2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster1, 2 accounts, 2 users
_run_test "cluster=$cluster1 account=$account1,$account2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1 wckey1, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey1 user=$user1" [list $jobid01 $jobid05]

# 1 cluster1, 1 wckey1, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey1 user=$user2" [list $jobid03 $jobid07]

# 1 cluster1, 1 wckey1, 2 users
_run_test "cluster=$cluster1 wckey=$wckey1 user=$user1,$user2" [list $jobid01 $jobid05 $jobid03 $jobid07]

# 1 cluster1, 1 wckey2, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey2 user=$user1" [list $jobid02 $jobid06]

# 1 cluster1, 1 wckey2, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey2 user=$user2" [list $jobid04 $jobid08]

# 1 cluster1, 1 wckey2, 2 users
_run_test "cluster=$cluster1 wckey=$wckey2 user=$user1,$user2" [list $jobid02 $jobid06 $jobid04 $jobid08]

# 1 cluster1, 2 wckeys, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster1, 2 wckeys, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster1, 2 wckeys, 2 users
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1account1, 1 wckey1, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user1" [list $jobid01]

# 1 cluster1, 1account1, 1 wckey1, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user2" [list $jobid03]

# 1 cluster1, 1account1, 1 wckey1, 2 users
_run_test "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user1,$user2" [list $jobid01 $jobid03]

# 1 cluster1, 1account1, 1 wckey2, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user1" [list $jobid02]

# 1 cluster1, 1account1, 1 wckey2, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user2" [list $jobid04]

# 1 cluster1, 1account1, 1 wckey2, 2 users
_run_test "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user1,$user2" [list $jobid02 $jobid04]

# 1 cluster1, 1account1, 2 wckeys, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user1" [list $jobid01 $jobid02]

# 1 cluster1, 1account1, 2 wckeys, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user2" [list $jobid03 $jobid04]

# 1 cluster1, 1account1, 2 wckeys, 2 users
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04]


# 1 cluster1, 1 account1
_run_test "cluster=$cluster1 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster1, 1 account2
_run_test "cluster=$cluster1 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster1, 2 accounts
_run_test "cluster=$cluster1 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 1 account1
_run_test "cluster=$cluster2 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster2, 1 account2
_run_test "cluster=$cluster2 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 2 accounts
_run_test "cluster=$cluster2 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1 wckey1
_run_test "cluster=$cluster1 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster1, 1 wckey2
_run_test "cluster=$cluster1 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster1, 2 wckeys
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 1 wckey1
_run_test "cluster=$cluster2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster2, 1 wckey2
_run_test "cluster=$cluster2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster2, 2 wckeys
_run_test "cluster=$cluster2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1 account1, 1 wckey1
_run_test "cluster=$cluster1 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03]

# 1 cluster1, 1 account1, 1 wckey2
_run_test "cluster=$cluster1 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04]

# 1 cluster1, 1 account1, 2 wckeys
_run_test "cluster=$cluster1 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster1, 1 account2, 1 wckey1
_run_test "cluster=$cluster1 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07]

# 1 cluster1, 1 account2, 1 wckey2
_run_test "cluster=$cluster1 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08]

# 1 cluster1, 1 account2, 2 wckeys
_run_test "cluster=$cluster1 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster1, 2 accounts, 1 wckey1
_run_test "cluster=$cluster1 account=$account1,$account2 wckey=$wckey1" [list  $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster1, 2 accounts, 1 wckey2
_run_test "cluster=$cluster1 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster1, 2 accounts, 2 wckeys
_run_test "cluster=$cluster1 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster2, 1 account1, 1 wckey1
_run_test "cluster=$cluster2 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03]

# 1 cluster2, 1 account1, 1 wckey2
_run_test "cluster=$cluster2 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04]

# 1 cluster2, 1 account1, 2 wckeys
_run_test "cluster=$cluster2 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster2, 1 account2, 1 wckey1
_run_test "cluster=$cluster2 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07]

# 1 cluster2, 1 account2, 1 wckey2
_run_test "cluster=$cluster2 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08]

# 1 cluster2, 1 account2, 2 wckeys
_run_test "cluster=$cluster2 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 2 accounts, 1 wckey1
_run_test "cluster=$cluster2 account=$account1,$account2 wckey=$wckey1" [list  $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster2, 2 accounts, 1 wckey2
_run_test "cluster=$cluster2 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster2, 2 accounts, 2 wckeys
_run_test "cluster=$cluster2 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 2 clusters, 1 account1
_run_test "cluster=$cluster1,$cluster2 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid01 $jobid02 $jobid03 $jobid04]

# 2 clusters, 1 account2
_run_test "cluster=$cluster1,$cluster2 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08 $jobid05 $jobid06 $jobid07 $jobid08]

# 2 clusters, 2 accounts
_run_test "cluster=$cluster1,$cluster2 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08 $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 2 clusters, 1 wckey1
_run_test "cluster=$cluster1,$cluster2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07 $jobid01 $jobid03 $jobid05 $jobid07]

# 2 clusters, 1 wckey2
_run_test "cluster=$cluster1,$cluster2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08 $jobid02 $jobid04 $jobid06 $jobid08]

# 2 clusters, 2 wckeys
_run_test "cluster=$cluster1,$cluster2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08 $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 2 clusters, 1 account1, 1 wckey1
_run_test "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03 $jobid01 $jobid03]

# 2 clusters, 1 account1, 1 wckey2
_run_test "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04 $jobid02 $jobid04]

# 2 clusters, 1 account1, 2 wckeys
_run_test "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid02 $jobid04 $jobid02 $jobid04]

# 2 clusters, 1 account2, 1 wckey1
_run_test "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07 $jobid05 $jobid07]

# 2 clusters, 1 account2, 1 wckey2
_run_test "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08 $jobid06 $jobid08]

# 2 clusters, 1 account2, 2 wckeys
_run_test "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid07 $jobid05 $jobid07 $jobid06 $jobid08 $jobid06 $jobid08]

# 2 clusters, 2 accounts, 1 wckey1
_run_test "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid05 $jobid07 $jobid05 $jobid07]

# 2 clusters, 2 accounts, 1 wckey2
_run_test "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid02 $jobid04 $jobid06 $jobid08 $jobid06 $jobid08]

# 2 clusters, 2 accounts, 2 wckeys
_run_test "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid05 $jobid07 $jobid05 $jobid07 $jobid02 $jobid04 $jobid02 $jobid04 $jobid06 $jobid08 $jobid06 $jobid08]