#!/usr/bin/env expect
############################################################################
# Purpose: Test local and remote job dependencies
#
# Reqs:    1. Using slurmdbd accounting storage type and is up
#          2. fed_slurm_base is defined in globals.local - set to directory that
#          has access to each federation configure (fedc1, fedc2, fedc3).
#          Eg.
#          fedr/slurm/ (src)
#          fedr/fed1/bin
#          fedr/fed1/sbin
#          fedr/fed1/etc
#          fedr/fed1/...
#          fedr/fed2/...
#          fedr/fed3/...
#          3. controllers are up and running.
############################################################################
# Copyright (C) 2020 SchedMD LLC.
# Written by Marshall Garey <marshall@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

source ./globals
source ./globals_accounting
source ./globals_federation

set c1 $fedc1
set c2 $fedc2
set c3 $fedc3
set exit_code 0
set job_id1 0
set job_id2 0
set user_name [get_my_user_name]
set file_in_long "test$test_id\_long.in"
set file_in_short "test$test_id\_short.in"
set my_scancel "${fed_slurm_base}/$c1/bin/scancel"
set my_scontrol "${fed_slurm_base}/$c1/bin/scontrol"
set reason ""
set dependency ""
set fed_name "fed_test$test_id"
set all_clusters "$c1,$c2,$c3"

###############################################################################
# Functions
###############################################################################

proc cancel_federation_jobs { } {
	global user_name c1 c2 c3 my_scancel all_clusters test_id

	spawn $my_scancel -M$all_clusters --jobname test${test_id}_job
	expect {
		eof {
			wait
		}
	}
	sleep 5
}

proc cancel_job { job_id clusters } {
	global my_scancel

	spawn $my_scancel $job_id
	expect {
		eof {
			wait
		}
	}
	wait_for_fed_job $job_id "DONE" $clusters
}

proc cleanup { rc } {
	global bin_rm file_in_long file_in_short fed_name test_id

	delete_federations $fed_name
	cancel_federation_jobs
	exec $bin_rm -f $file_in_long
	exec $bin_rm -f $file_in_short
}

proc submit_job { options cdir file_in } {
	global bin_sleep sbatch number fed_slurm_base slow_kill test_id

	set job_id 0
	set my_sbatch "${fed_slurm_base}/$cdir/bin/sbatch"
	if {[test_federation_setup]} {
		set my_sbatch $sbatch
	}
	set command "$my_sbatch --job-name=test$test_id\_job -t1 \
		$options --output=/dev/null $file_in"
	set sbatch_pid [spawn {*}$command]
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		timeout {
			slow_kill $sbatch_pid
			fail "sbatch not responding"
		}
		eof {
			wait
		}
	}
	if { $job_id == 0 } {
		fail "Failed to submit job\n"
	}

	return $job_id
}

proc get_job_dependency { job_id } {
	global my_scontrol reason dependency re_word_str

	set reason ""
	set dependency ""
	set reason_match "\[a-zA-Z_\]+"
	# Possible dependency syntax:
	# <type>:<jobid>+<time>(state)
	# <type>:<jobid>(state)
	# singleton(unfulfilled)
	# Notes:
	# * Multiple dependencies are separated by a comma or a question mark
	# * The state for singleton will only ever be "unfulfilled"
	# * The state is either failed or unfulfilled. Fulfilled dependencies
	#   are cleared from the list
	# * When there are no dependencies, it will be this string: "(null)"

	# This regex takes care of handling dependencies separated by comma or
	# question mark.
	# ([a-zA-Z_]+:[0-9_*+]+\([a-zA-Z]+\)\?*\,*|singleton\(unfulfilled\)\?*\,*)+|\(null\)


#set depend_match "\[a-zA-Z_\]+:\[0-9_*+\]+\\($re_word_str\\)\|\\(null\\)\|singleton\\(unfulfilled\\)"

	set type "\[a-zA-Z_\]+"
	set jobid_time "\[0-9_*+\]+"
	set state "\\($re_word_str\\)\\"
	set delim ",*\\?*"
	set depend_regex "\($type:$jobid_time$state$delim\|singleton\\(unfulfilled\\)$delim\)+"
	set no_depend "\\(null\\)"
	set depend_match "$depend_regex\|$no_depend"

	log_user 0
	spawn $my_scontrol show job $job_id
	expect {
		-re "Reason=($reason_match) Dependency=($depend_match)" {
			set reason $expect_out(1,string)
			set dependency $expect_out(2,string)
			exp_continue
		}
		timeout {
			fail "scontrol not responding"
		}
		eof {
			wait
		}
	}
	log_user 1
	log_info "job $job_id; actual   reason: \"$reason\"; dependency: \"$dependency\""
	return $dependency
}

proc check_depend { job_id expected_reason expected_dependency } {
	global reason dependency

	get_job_dependency $job_id
	if { [string compare "$reason" "$expected_reason"] } {
		return 1
	}
	if { [string compare "$dependency" "$expected_dependency"] } {
		return 1
	}
	return 0
}

proc wait_for_depend { job_id expected_reason expected_dependency } {
	global reason dependency

	set error 0
	set my_delay 0
	# By default we test remote dependencies every 30 seconds.
	# Once the dependencies are cleared, we have to wait for the job to be
	# scheduled which could be another 30-60 seconds dependending on the
	# main scheduler and backfill scheduler timing.
	set max_delay 120
	set poll_interval 3

	if { [string compare $expected_reason "DependencyNeverSatisfied"] } {
		set want_never_satisfied 0
	} else {
		set want_never_satisfied 1
	}

	log_info "job $job_id; expected reason: \"$expected_reason\"; dependency: \"$expected_dependency\""

	while 1 {
		if { ![check_depend $job_id $expected_reason \
			$expected_dependency] } {
			return 0
		}

		if { (!$want_never_satisfied) && \
			!([string compare $reason \
			  "DependencyNeverSatisfied"]) } {
			log_error "Job dependency failed, but it shouldn't have."
			set error 1
		}
		if { $my_delay >= $max_delay } {
			log_info "delay $my_delay max $max_delay"
			log_error "Timeout waiting for dependency to change."
			set error 1
		}

		if { $error } {
			fail "Job $job_id actual: reason=\"$reason\"; dependency=\"$dependency\"; expected: reason=\"$expected_reason\"; dependency=\"expected_dependency\""
		}

		exec sleep $poll_interval
		set my_delay [expr $my_delay + $poll_interval]
	}
}

proc my_wait_for_fed_job { job_id state cluster } {
	set ret_cluster [wait_for_fed_job $job_id $state $cluster]
	if { [string compare $cluster $ret_cluster] } {
		fail "Job $job_id did not reach state $state on cluster $cluster"
	}
}

proc is_job_on_cluster { job_id cluster } {
	global my_scontrol

	log_user 0
	spawn $my_scontrol -M$cluster --local -o show job $job_id
	expect {
		-re "JobId=$job_id" {
			log_user 1
			log_info "Found job $job_id in cluster $cluster"
			return 1
		}
		-re "Invalid job id specified" {
			log_user 1
			log_info "Did not find job $job_id in cluster $cluster"
			return 0
		}
		timeout {
			log_user 1
			fail "scontrol not responding"
		}
		eof {
			wait
		}
	}
}

proc test_local_after { } {
	global c1 file_in_long bin_sleep
	log_info "#############################################################"
	log_info "# Test local after"
	log_info "#############################################################"

	# Local dependency succeeds
	log_info "after: test that local dependency succeeds:"
	set job_id1 [submit_job "-M$c1 --begin=now+5" $c1 $file_in_long]
	set job_id2 [submit_job "--depend=after:$job_id1 -M$c1" $c1 \
		$file_in_long]

	wait_for_depend $job_id2 "Dependency" "after:$job_id1\(unfulfilled\)"
	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id1 $c1
	cancel_job $job_id2 $c1

}

proc test_remote_after { } {
	global c1 c2 file_in_long bin_sleep

	log_info "#############################################################"
	log_info "# Test remote after"
	log_info "#############################################################"

	# Remote dependency succeeds
	log_info "after: test that remote dependency succeeds:"
	set job_id1 [submit_job "-M$c2 --begin=now+5" $c2 $file_in_long]
	set job_id2 [submit_job "--depend=after:$job_id1 -M$c1" $c1 \
		$file_in_long]

	wait_for_depend $job_id2 "Dependency" "after:$job_id1\(unfulfilled\)"
	my_wait_for_fed_job $job_id1 "RUNNING" $c2
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id1 $c2
	cancel_job $job_id2 $c1

	# Test after with a time attached. file_in_long sleeps for 60 seconds.
	log_info "after: test that a after+time works:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "--depend=after:$job_id1+1 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" "after:$job_id1+1(unfulfilled)"
	log_info "Check that job $job_id2 is still dependent after 45 seconds"
	exec $bin_sleep 45
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" "after:$job_id1+1(unfulfilled)"
	log_info "Wait for job $job_id2 dependency to be fulfilled"
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id2 $c1

	# After dependency never fails.
}

proc test_local_afterany { } {
	global c1 file_in_long

	log_info "#############################################################"
	log_info "# Test local afterany"
	log_info "#############################################################"

	# Local dependency succeeds
	log_info "afterany: test that local dependency succeeds:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "--depend=afterany:$job_id1 -M$c1" $c1 \
		$file_in_long]

	wait_for_depend $job_id2 "Dependency" "afterany:$job_id1\(unfulfilled\)"
	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	cancel_job $job_id1 $c1
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id2 $c1
}

proc test_remote_afterany { } {
	global c1 c2 file_in_long

	log_info "#############################################################"
	log_info "# Test remote afterany"
	log_info "#############################################################"

	# Remote dependency succeeds
	log_info "afterany: test that remote dependency succeeds:"
	set job_id1 [submit_job "-M$c2" $c2 $file_in_long]
	set job_id2 [submit_job "--depend=afterany:$job_id1 -M$c1" $c1 \
		$file_in_long]

	wait_for_depend $job_id2 "Dependency" "afterany:$job_id1\(unfulfilled\)"
	my_wait_for_fed_job $job_id1 "RUNNING" $c2
	cancel_job $job_id1 $c2
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id2 $c1

	# Test old syntax: --depend=jobid,jobid
	log_info "afterany: test old syntax: --depend=jobid\[,jobid,jobid...\]"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "--depend=$job_id1 -M$c1" $c1 \
		$file_in_long]
	set job_id3 [submit_job "--depend=$job_id1,$job_id2 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	my_wait_for_fed_job $job_id3 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" "afterany:$job_id1\(unfulfilled\)"
	wait_for_depend $job_id3 "Dependency" \
		"afterany:$job_id1\(unfulfilled\),afterany:$job_id2\(unfulfilled\)"
	cancel_federation_jobs
}

proc test_local_aftercorr { } {
	global c1 kill_invalid_depend file_in_long file_in_short

	log_info "#############################################################"
	log_info "# Test local aftercorr"
	log_info "#############################################################"

	# Local dependency succeeds
	log_info "aftercorr: test that local dependency succeeds:"
	set job_array1 [submit_job "-M$c1 --array=1-2" $c1 $file_in_short]
	set job_array2 [submit_job "--depend=aftercorr:$job_array1 -M$c1 \
		--array=1-2" $c1 $file_in_long]

	my_wait_for_fed_job "$job_array1\_1" "RUNNING" $c1
	my_wait_for_fed_job "$job_array1\_2" "RUNNING" $c1
	my_wait_for_fed_job "$job_array2\_1" "PENDING" $c1
	my_wait_for_fed_job "$job_array2\_2" "PENDING" $c1
	wait_for_depend "$job_array2\_1" "Dependency" \
		"aftercorr:$job_array1\_*(unfulfilled)"
	wait_for_depend "$job_array2\_2" "Dependency" \
		"aftercorr:$job_array1\_*(unfulfilled)"

	my_wait_for_fed_job "$job_array1\_1" "DONE" $c1
	my_wait_for_fed_job "$job_array1\_2" "DONE" $c1
	wait_for_depend "$job_array2\_1" "None" "(null)"
	wait_for_depend "$job_array2\_2" "None" "(null)"
	my_wait_for_fed_job "$job_array2\_1" "RUNNING" $c1
	my_wait_for_fed_job "$job_array2\_2" "RUNNING" $c1
	cancel_job $job_array2 $c1

	# Local dependency fails
	log_info "aftercorr: test that local dependency fails:"
	set job_array1 [submit_job "-M$c1 --array=1-2" $c1 $file_in_long]
	set job_array2 [submit_job "--depend=aftercorr:$job_array1 -M$c1 \
		--array=1-2" $c1 $file_in_long]

	my_wait_for_fed_job "$job_array1\_1" "RUNNING" $c1
	my_wait_for_fed_job "$job_array1\_2" "RUNNING" $c1
	my_wait_for_fed_job "$job_array2\_1" "PENDING" $c1
	my_wait_for_fed_job "$job_array2\_2" "PENDING" $c1
	wait_for_depend "$job_array2\_1" "Dependency" \
		"aftercorr:$job_array1\_*(unfulfilled)"
	wait_for_depend "$job_array2\_2" "Dependency" \
		"aftercorr:$job_array1\_*(unfulfilled)"

	cancel_job "$job_array1\_1" $c1
	wait_for_depend "$job_array2\_1" "DependencyNeverSatisfied" \
		"aftercorr:$job_array1\_*(failed)"
	cancel_job "$job_array1\_2" $c1
	wait_for_depend "$job_array2\_2" "DependencyNeverSatisfied" \
		"aftercorr:$job_array1\_*(failed)"
	if { !$kill_invalid_depend } {
		cancel_job "$job_array2" $c1
	}
}

proc test_remote_aftercorr { } {
	global c1 c2 kill_invalid_depend file_in_long file_in_short

	log_info "#############################################################"
	log_info "# Test remote aftercorr"
	log_info "#############################################################"

	# Remote dependency succeeds
	log_info "aftercorr: test that remote dependency succeeds:"
	set job_array1 [submit_job "-M$c2 --array=1-2" $c2 $file_in_short]
	set job_array2 [submit_job "--depend=aftercorr:$job_array1 -M$c1 \
		--array=1-2" $c1 $file_in_long]

	my_wait_for_fed_job "$job_array1\_1" "RUNNING" $c2
	my_wait_for_fed_job "$job_array1\_2" "RUNNING" $c2
	my_wait_for_fed_job "$job_array2\_1" "PENDING" $c1
	my_wait_for_fed_job "$job_array2\_2" "PENDING" $c1
	# The dependency on the remote side has _*, but the dependency locally
	# doesn't because it couldn't find the remote job.
	wait_for_depend "$job_array2\_1" "Dependency" \
		"aftercorr:$job_array1\(unfulfilled)"
	wait_for_depend "$job_array2\_2" "Dependency" \
		"aftercorr:$job_array1\(unfulfilled)"

	my_wait_for_fed_job "$job_array1\_1" "DONE" $c2
	my_wait_for_fed_job "$job_array1\_2" "DONE" $c2
	wait_for_depend "$job_array2\_1" "None" "(null)"
	wait_for_depend "$job_array2\_2" "None" "(null)"
	my_wait_for_fed_job "$job_array2\_1" "RUNNING" $c1
	my_wait_for_fed_job "$job_array2\_2" "RUNNING" $c1
	cancel_job $job_array2 $c1

	# Remote dependency fails
	log_info "aftercorr: test that remote dependency fails:"
	set job_array1 [submit_job "-M$c2 --array=1-2" $c2 $file_in_long]
	set job_array2 [submit_job "--depend=aftercorr:$job_array1 -M$c1 \
		--array=1-2" $c1 $file_in_long]

	my_wait_for_fed_job "$job_array1\_1" "RUNNING" $c2
	my_wait_for_fed_job "$job_array1\_2" "RUNNING" $c2
	my_wait_for_fed_job "$job_array2\_1" "PENDING" $c1
	my_wait_for_fed_job "$job_array2\_2" "PENDING" $c1
	wait_for_depend "$job_array2\_1" "Dependency" \
		"aftercorr:$job_array1\(unfulfilled)"
	wait_for_depend "$job_array2\_2" "Dependency" \
		"aftercorr:$job_array1\(unfulfilled)"

	cancel_job "$job_array1\_1" $c2
	wait_for_depend "$job_array2\_1" "DependencyNeverSatisfied" \
		"aftercorr:$job_array1\(failed)"
	cancel_job "$job_array1\_2" $c2
	wait_for_depend "$job_array2\_2" "DependencyNeverSatisfied" \
		"aftercorr:$job_array1\(failed)"
	if { !$kill_invalid_depend } {
		cancel_job "$job_array2" $c1
	}
}

proc test_local_afterok { } {
	global c1 kill_invalid_depend file_in_long file_in_short

	log_info "#############################################################"
	log_info "# Test local afterok"
	log_info "#############################################################"

	# Local dependency succeeds
	log_info "afterok: test that local dependency succeeds:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_short]
	set job_id2 [submit_job "--depend=afterok:$job_id1 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" "afterok:$job_id1\(unfulfilled\)"
	my_wait_for_fed_job $job_id1 "DONE" $c1
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id2 $c1

	# Local dependency fails
	log_info "afterok: test that local dependency fails:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "--depend=afterok:$job_id1 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" "afterok:$job_id1\(unfulfilled\)"
	cancel_job $job_id1 $c1
	wait_for_depend $job_id2 "DependencyNeverSatisfied" \
		"afterok:$job_id1\(failed\)"
	if { !$kill_invalid_depend } {
		cancel_job $job_id2 $c1
	}
}

proc test_remote_afterok { } {
	global c1 c2 kill_invalid_depend file_in_long file_in_short

	log_info "#############################################################"
	log_info "# Test remote afterok"
	log_info "#############################################################"

	# Remote dependency succeeds
	log_info "afterok: test that remote dependency succeeds:"
	set job_id1 [submit_job "-M$c2" $c2 $file_in_short]
	set job_id2 [submit_job "--depend=afterok:$job_id1 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c2
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" "afterok:$job_id1\(unfulfilled\)"
	my_wait_for_fed_job $job_id1 "DONE" $c2
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id2 $c1

	# Remote dependency fails
	log_info "afterok: test that remote dependency fails"
	set job_id1 [submit_job "-M$c2" $c2 $file_in_long]
	set job_id2 [submit_job "--depend=afterok:$job_id1 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c2
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" "afterok:$job_id1\(unfulfilled\)"
	cancel_job $job_id1 $c2
	wait_for_depend $job_id2 "DependencyNeverSatisfied" \
		"afterok:$job_id1\(failed\)"
	if { !$kill_invalid_depend } {
		cancel_job $job_id2 $c1
	}
}

proc test_local_afternotok { } {
	global c1 kill_invalid_depend file_in_long file_in_short

	log_info "#############################################################"
	log_info "# Test local afternotok"
	log_info "#############################################################"

	# Local dependency succeeds
	log_info "afternotok: test that local dependency succeeds:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "--depend=afternotok:$job_id1 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" \
		"afternotok:$job_id1\(unfulfilled\)"
	cancel_job $job_id1 $c1
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id2 $c1

	# Local dependency fails
	log_info "afternotok: test that local dependency fails:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_short]
	set job_id2 [submit_job "--depend=afternotok:$job_id1 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" \
		"afternotok:$job_id1\(unfulfilled\)"
	my_wait_for_fed_job $job_id1 "DONE" $c1
	wait_for_depend $job_id2 "DependencyNeverSatisfied" \
		"afternotok:$job_id1\(failed\)"
	if { !$kill_invalid_depend } {
		cancel_job $job_id2 $c1
	}
}

proc test_remote_afternotok { } {
	global c1 c2 kill_invalid_depend file_in_long file_in_short

	log_info "#############################################################"
	log_info "# Test remote afternotok"
	log_info "#############################################################"

	# Remote dependency succeeds
	log_info "afternotok: test that remote dependency succeeds:"
	set job_id1 [submit_job "-M$c2" $c2 $file_in_long]
	set job_id2 [submit_job "--depend=afternotok:$job_id1 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c2
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" \
		"afternotok:$job_id1\(unfulfilled\)"
	cancel_job $job_id1 $c2
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id2 $c1

	# Remote dependency fails
	log_info "afternotok: test that remote dependency fails"
	set job_id1 [submit_job "-M$c2" $c2 $file_in_short]
	set job_id2 [submit_job "--depend=afternotok:$job_id1 -M$c1" $c1 \
		$file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c2
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" \
		"afternotok:$job_id1\(unfulfilled\)"
	my_wait_for_fed_job $job_id1 "DONE" $c2
	wait_for_depend $job_id2 "DependencyNeverSatisfied" \
		"afternotok:$job_id1\(failed\)"
	if { !$kill_invalid_depend } {
		cancel_job $job_id2 $c1
	}
}

proc test_local_singleton { } {
	global c1 disable_remote_singleton file_in_long

	log_info "#############################################################"
	log_info "# Test local singleton"
	log_info "#############################################################"

	# Test one cluster
	log_info "singleton: test on one cluster"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "--depend=singleton -M$c1" $c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" "singleton(unfulfilled)"
	cancel_job $job_id1 $c1
	wait_for_depend $job_id2 "None" "(null)"
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	cancel_job $job_id2 $c1

}

proc test_remote_singleton { } {
	global c1 c2 c3 disable_remote_singleton file_in_long

	log_info "#############################################################"
	log_info "# Test remote singleton"
	log_info "#############################################################"

	# Test multiple clusters
	if { $disable_remote_singleton } {
		# Test that remote jobs don't affect the singleton dependency
		log_info "singleton: test that disable_remote_singleton works"
		set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
		set job_id2 [submit_job "-M$c2" $c2 $file_in_long]
		set job_id3 [submit_job "--depend=singleton -M$c1" $c1 \
			$file_in_long]

		my_wait_for_fed_job $job_id1 "RUNNING" $c1
		my_wait_for_fed_job $job_id2 "RUNNING" $c2
		my_wait_for_fed_job $job_id3 "PENDING" $c1
		wait_for_depend $job_id3 "Dependency" "singleton(unfulfilled)"
		# Cancel job 1 - job 3 should start running even though job 2 is
		# running on another cluster
		cancel_job $job_id1 $c1
		wait_for_depend $job_id3 "None" "(null)"
		my_wait_for_fed_job $job_id3 "RUNNING" $c1
		cancel_job $job_id2 $c2
		cancel_job $job_id3 $c1
	} else {
		# Test that singleton doesn't get cleared until jobs on all
		# clusters are done
		log_info "singleton: test with jobs on all clusters"
		set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
		set job_id2 [submit_job "-M$c2" $c2 $file_in_long]
		set job_id3 [submit_job "-M$c3" $c3 $file_in_long]
		set job_id4 [submit_job "--depend=singleton -M$c1" $c1 \
			$file_in_long]

		my_wait_for_fed_job $job_id1 "RUNNING" $c1
		my_wait_for_fed_job $job_id2 "RUNNING" $c2
		my_wait_for_fed_job $job_id3 "RUNNING" $c3
		my_wait_for_fed_job $job_id4 "PENDING" $c1
		wait_for_depend $job_id4 "Dependency" "singleton(unfulfilled)"

		# Job 4 shouldn't start until jobs 1, 2, and 3 are done.
		# Test that it starts when a remote job is finished last.
		cancel_job $job_id1 $c1
		# Should still have the same dependency
		wait_for_depend $job_id4 "Dependency" "singleton(unfulfilled)"
		cancel_job $job_id2 $c2
		cancel_job $job_id3 $c3
		# Now the dependency should be cleared
		wait_for_depend $job_id4 "None" "(null)"
		my_wait_for_fed_job $job_id4 "RUNNING" $c1
		cancel_job $job_id4 $c1
	}
}

proc test_add_remove_clusters { } {
	global c1 c2 c3 fed_name file_in_long \
		disable_remote_singleton kill_invalid_depend

	# Test adding/removing clusters from the federation
	# Removing a cluster from a federation should cause dependencies on
	# jobs on that cluster to fail.
	# Adding a cluster to a federation means that any singleton dependencies
	# have to be fulfilled on that cluster.
	log_info "#############################################################"
	log_info "# Test adding/removing a cluster from the federation"
	log_info "#############################################################"

	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "-M$c2" $c2 $file_in_long]
	set job_id3 [submit_job "-M$c3" $c3 $file_in_long]
	set job_id4 [submit_job "--depend=afterok:$job_id3 -M$c1" $c1 \
		$file_in_long]
	set job_id5 [submit_job "--depend=singleton -M$c1" $c1 $file_in_long]
	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "RUNNING" $c2
	my_wait_for_fed_job $job_id3 "RUNNING" $c3
	my_wait_for_fed_job $job_id4 "PENDING" $c1
	wait_for_depend $job_id4 "Dependency" "afterok:$job_id3\(unfulfilled\)"
	wait_for_depend $job_id5 "Dependency" "singleton(unfulfilled)"

	log_info "Test that removing cluster $c3 from fed $fed_name makes dependencies on jobs on $c3 fail"
	if { [remove_cluster_from_fed $c3 $fed_name] } {
		fail "Unable to remove cluster $c3 from federation $fed_name"
	}
	wait_for_depend $job_id4 "DependencyNeverSatisfied" \
		"afterok:$job_id3\(failed\)"
	if { !$kill_invalid_depend } {
		cancel_job $job_id4 $c1
	}

	if { $disable_remote_singleton } {
		cancel_job $job_id1 $c1
		cancel_job $job_id2 $c2
		cancel_job $job_id3 $c3
		cancel_job $job_id5 $c1
		return
	}

	log_info "Test that the singleton dependency was resent back to cluster $c3 when it was added back to the federation."
	if { [add_cluster_to_fed $c3 $fed_name] } {
		fail "Unable to add cluster $c3 to federation $fed_name"
	}
	cancel_job $job_id1 $c1
	cancel_job $job_id2 $c2
	cancel_job $job_id3 $c3
	wait_for_depend $job_id5 "None" "(null)"
	my_wait_for_fed_job $job_id5 "RUNNING" $c1
	cancel_job $job_id5 $c1
}

proc test_submit_to_all_clusters { } {
	global c1 c2 c3 file_in_long

	log_info "#############################################################"
	log_info "# Test submitting a dependent job to all clusters"
	log_info "#############################################################"

	log_info "Test that a dependent job is only on its origin cluster while dependent and that it is submitted to all clusters when its dependency is cleared."
	set job_id1 [submit_job "-M$c2" $c2 $file_in_long]
	set job_id2 [submit_job "--depend=afternotok:$job_id1 -M$c1,$c2,$c3 \
		--begin=now+60" $c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c2
	my_wait_for_fed_job $job_id2 "PENDING" $c1
	wait_for_depend $job_id2 "Dependency" \
		"afternotok:$job_id1\(unfulfilled\)"

	log_info "Test that job $job_id2 is not on clusters $c2 or $c3."
	if { [is_job_on_cluster $job_id2 $c2] || \
		[is_job_on_cluster $job_id2 $c3] } {
			fail "Job $job_id2 is in cluster $c2 and/or $c3 when it shouldn't be."
	}

	log_info "Test that job $job_id2 is submitted to all sibling clusters $c2 and $c3 when its dependency is fulfilled."
	cancel_job $job_id1 $c2
	wait_for_depend $job_id2 "BeginTime" "(null)"
	my_wait_for_fed_job $job_id2 "PENDING" "$c1"
	my_wait_for_fed_job $job_id2 "PENDING" "$c2"
	my_wait_for_fed_job $job_id2 "PENDING" "$c3"
	cancel_job $job_id2 "$c1,$c2,$c3"
}

proc test_local_or_dependencies { } {
	global c1 file_in_long kill_invalid_depend

	log_info "#############################################################"
	log_info "# Test local OR dependencies"
	log_info "#############################################################"

	log_info "OR dependencies: Test that one fulfilled dependency makes the whole dependency fulfilled:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id3 [submit_job \
		"--depend=afternotok:$job_id1?afternotok:$job_id2 -M$c1" \
		$c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	my_wait_for_fed_job $job_id3 "PENDING" $c1
	wait_for_depend $job_id3 "Dependency" \
		"afternotok:$job_id1\(unfulfilled\)?afternotok:$job_id2\(unfulfilled\)"

	cancel_job $job_id2 $c1
	wait_for_depend $job_id3 "None" "(null)"
	my_wait_for_fed_job $job_id3 "RUNNING" $c1
	cancel_job $job_id1 $c1
	cancel_job $job_id3 $c1

	log_info "OR dependencies: Test that the dependency doesn't fail until all dependencies have failed:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id3 [submit_job "--depend=afterok:$job_id1?afterok:$job_id2 \
		-M$c1" $c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	my_wait_for_fed_job $job_id3 "PENDING" $c1
	wait_for_depend $job_id3 "Dependency" \
		"afterok:$job_id1\(unfulfilled\)?afterok:$job_id2\(unfulfilled\)"

	cancel_job $job_id1 $c1
	wait_for_depend $job_id3 "Dependency" \
		"afterok:$job_id1\(failed\)?afterok:$job_id2\(unfulfilled\)"
	cancel_job $job_id2 $c1
	wait_for_depend $job_id3 "DependencyNeverSatisfied" \
		"afterok:$job_id1\(failed\)?afterok:$job_id2\(failed\)"
	if { !$kill_invalid_depend } {
		cancel_job $job_id3 $c1
	}
}

proc test_remote_or_dependencies { } {
	global c1 c2 file_in_long kill_invalid_depend

	log_info "#############################################################"
	log_info "# Test remote OR dependencies"
	log_info "#############################################################"

	log_info "OR dependencies: Test that one fulfilled dependency makes the whole dependency fulfilled:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "-M$c2" $c2 $file_in_long]
	set job_id3 [submit_job \
		"--depend=afternotok:$job_id1?afternotok:$job_id2 -M$c1" \
		$c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "RUNNING" $c2
	my_wait_for_fed_job $job_id3 "PENDING" $c1
	wait_for_depend $job_id3 "Dependency" \
		"afternotok:$job_id1\(unfulfilled\)?afternotok:$job_id2\(unfulfilled\)"

	cancel_job $job_id2 $c2
	wait_for_depend $job_id3 "None" "(null)"
	my_wait_for_fed_job $job_id3 "RUNNING" $c1
	cancel_job $job_id1 $c1
	cancel_job $job_id3 $c1

	log_info "OR dependencies: Test that the dependency doesn't fail until all dependencies have failed:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "-M$c2" $c2 $file_in_long]
	set job_id3 [submit_job "--depend=afterok:$job_id1?afterok:$job_id2 \
		-M$c1" $c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "RUNNING" $c2
	my_wait_for_fed_job $job_id3 "PENDING" $c1
	wait_for_depend $job_id3 "Dependency" \
		"afterok:$job_id1\(unfulfilled\)?afterok:$job_id2\(unfulfilled\)"

	cancel_job $job_id1 $c1
	wait_for_depend $job_id3 "Dependency" \
		"afterok:$job_id1\(failed\)?afterok:$job_id2\(unfulfilled\)"
	cancel_job $job_id2 $c2
	wait_for_depend $job_id3 "DependencyNeverSatisfied" \
		"afterok:$job_id1\(failed\)?afterok:$job_id2\(failed\)"
	if { !$kill_invalid_depend } {
		cancel_job $job_id3 $c1
	}
}

proc test_local_and_dependencies { } {
	global c1 file_in_long kill_invalid_depend

	log_info "#############################################################"
	log_info "# Test local AND dependencies"
	log_info "#############################################################"

	log_info "AND dependencies: Test that the dependency isn't fulfilled until all dependencies are fulfilled:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id3 [submit_job \
		"--depend=afternotok:$job_id1,afternotok:$job_id2 -M$c1" \
		$c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	my_wait_for_fed_job $job_id3 "PENDING" $c1
	wait_for_depend $job_id3 "Dependency" \
		"afternotok:$job_id1\(unfulfilled\),afternotok:$job_id2\(unfulfilled\)"

	cancel_job $job_id1 $c1
	wait_for_depend $job_id3 "Dependency" \
		"afternotok:$job_id2\(unfulfilled\)"
	cancel_job $job_id2 $c1
	wait_for_depend $job_id3 "None" "(null)"
	my_wait_for_fed_job $job_id3 "RUNNING" $c1
	cancel_job $job_id3 $c1

	log_info "AND dependencies: Test that the whole dependency fails when a single dependency fails:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id3 [submit_job "--depend=afterok:$job_id1,afterok:$job_id2 \
		-M$c1" $c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "RUNNING" $c1
	my_wait_for_fed_job $job_id3 "PENDING" $c1
	wait_for_depend $job_id3 "Dependency" \
		"afterok:$job_id1\(unfulfilled\),afterok:$job_id2\(unfulfilled\)"

	cancel_job $job_id2 $c1
	wait_for_depend $job_id3 "DependencyNeverSatisfied" \
		"afterok:$job_id1\(unfulfilled\),afterok:$job_id2\(failed\)"
	if { !$kill_invalid_depend } {
		cancel_job $job_id3 $c1
	}
	cancel_job $job_id1 $c1
}

proc test_remote_and_dependencies { } {
	global c1 c2 file_in_long kill_invalid_depend

	log_info "#############################################################"
	log_info "# Test remote AND dependencies"
	log_info "#############################################################"

	log_info "AND dependencies: Test that the dependency isn't fulfilled until all dependencies are fulfilled:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "-M$c2" $c2 $file_in_long]
	set job_id3 [submit_job \
		"--depend=afternotok:$job_id1,afternotok:$job_id2 -M$c1" \
		$c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "RUNNING" $c2
	my_wait_for_fed_job $job_id3 "PENDING" $c1
	wait_for_depend $job_id3 "Dependency" \
		"afternotok:$job_id1\(unfulfilled\),afternotok:$job_id2\(unfulfilled\)"

	cancel_job $job_id1 $c1
	wait_for_depend $job_id3 "Dependency" \
		"afternotok:$job_id2\(unfulfilled\)"
	cancel_job $job_id2 $c2
	wait_for_depend $job_id3 "None" "(null)"
	my_wait_for_fed_job $job_id3 "RUNNING" $c1
	cancel_job $job_id3 $c1

	log_info "AND dependencies: Test that the whole dependency fails when a single dependency fails:"
	set job_id1 [submit_job "-M$c1" $c1 $file_in_long]
	set job_id2 [submit_job "-M$c2" $c2 $file_in_long]
	set job_id3 [submit_job "--depend=afterok:$job_id1,afterok:$job_id2 \
		-M$c1" $c1 $file_in_long]

	my_wait_for_fed_job $job_id1 "RUNNING" $c1
	my_wait_for_fed_job $job_id2 "RUNNING" $c2
	my_wait_for_fed_job $job_id3 "PENDING" $c1
	wait_for_depend $job_id3 "Dependency" \
		"afterok:$job_id1\(unfulfilled\),afterok:$job_id2\(unfulfilled\)"

	cancel_job $job_id2 $c2
	wait_for_depend $job_id3 "DependencyNeverSatisfied" \
		"afterok:$job_id1\(unfulfilled\),afterok:$job_id2\(failed\)"
	if { !$kill_invalid_depend } {
		cancel_job $job_id3 $c1
	}
	cancel_job $job_id1 $c1
}

###############################################################################
# Begin test
###############################################################################

if { [test_account_storage] == 0 } {
	skip "This test can't be run without a usable AccountStorageType"
}

if {![test_federation_setup]} {

	if {[test_all_up]} {
		skip "All of the clusters in the federation must be up"
	}

	delete_federations $fed_name
	if { [setup_federation $fed_name] } {
		fail "Unable to set up federation $fed_name"
	}
} else {
	log_warn "Not running remote dependency tests"
	set c1 [get_cluster_name]
	set all_clusters "$c1"
	set my_scancel $scancel
	set my_scontrol $scontrol
}

# Use file_in_short when we have to wait for the job to end.
# Use file_in_long everywhere else.
make_bash_script $file_in_long "$bin_sleep 60"
make_bash_script $file_in_short "$bin_sleep 5"

set permit_job_expansion [test_scheduler_params "permit_job_expansion"]
log_info "permit_job_expansion: $permit_job_expansion"

set kill_invalid_depend [test_dependency_params "kill_invalid_depend"]
set disable_remote_singleton [test_dependency_params "disable_remote_singleton"]
log_info "kill_invalid_depend: $kill_invalid_depend; disable_remote_singleton: $disable_remote_singleton\n"

cancel_federation_jobs

test_local_after
test_local_afterany
# --depend=afterburstbuffer is tested in test35.6
test_local_aftercorr
test_local_afterok
test_local_afternotok
test_local_singleton
test_local_or_dependencies
test_local_and_dependencies
# test --depend=expand in another test.

if {![test_federation_setup]} {
	test_remote_after
	test_remote_afterany
	test_remote_aftercorr
	test_remote_afterok
	test_remote_afternotok
	test_remote_singleton
	test_remote_or_dependencies
	test_remote_and_dependencies
	test_add_remove_clusters
	test_submit_to_all_clusters
}