#!/bin/bash # ---------------------------------------------------------------------- # File: eos-http-upload--test # Author: Andreas-Joachim Peters - CERN # ---------------------------------------------------------------------- # ************************************************************************ # * EOS - the CERN Disk Storage System * # * Copyright (C) 2018 CERN/Switzerland * # * * # * This program is free software: you can redistribute it and/or modify * # * it under the terms of the GNU General Public License as published by * # * the Free Software Foundation, either version 3 of the License, or * # * (at your option) any later version. * # * * # * This program is distributed in the hope that it will be useful, * # * but WITHOUT ANY WARRANTY; without even the implied warranty of * # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * # * GNU General Public License for more details. * # * * # * You should have received a copy of the GNU General Public License * # * along with this program. If not, see .* # ************************************************************************ # usage: eos-http-upload-test rangeupload testfile /eos/dev/upload-directory/ port="${EOS_HTTPS_PORT:-443}" host=$(hostname -f) cacert="/etc/grid-security/certificates/rootCA.pem" capath="$(dirname "${cacert}")" cert="/root/.globus/usercert.pem" key="/root/.globus/userkey.pem" usage() { echo '''Usage: # File: eos-http-upload--test [-h|--host [:| (-p|--port) ]] : only rangeupload for the time being : name of the file : name of the destination directory [-c|--cert ] [-k|--key ] [-C|--cacert ] [-P|--capath ] [-h|--help] - usage & exit '''; } # Parser from: https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash usage_error () { echo >&2 "$(basename "$0"): $1"; exit 2; } assert_argument () { test "$1" != "$EOL" || usage_error "${POSITIONAL_ARGS[1]} requires an argument"; } # One loop, nothing more. POSITIONAL_ARGS=() if [ "$#" != 0 ]; then EOL=$(printf '\1\3\3\7') set -- "$@" "$EOL" while [ "$1" != "$EOL" ]; do opt="$1"; shift case "$opt" in -h|--host) assert_argument "$1" "$opt"; host="${1%:*}" && port="${1#*:}"; shift;; -p|--port) assert_argument "$1" "$opt"; port="$1"; shift;; -c|--cert) assert_argument "$1" "$opt"; cert="$1"; shift;; -k|--key) assert_argument "$1" "$opt"; key="$1"; shift;; -C|--cacert) assert_argument "$1" "$opt"; cacert="${1}" && capath="$(dirname "${cacert}")"; shift;; -P|--capath) assert_argument "$1" "$opt"; capath="${1}"; shift;; -h|--help) usage; exit 0;; # Arguments processing. You may remove any unneeded line after the 1st. -|''|[!-]*) set -- "$@" "$opt";; # positional argument, rotate to the end --*=*) set -- "${opt%%=*}" "${opt#*=}" "$@";; # convert '--name=arg' to '--name' 'arg' -[!-]?*) set -- "$(echo "${opt#-}" | sed 's /g')" "$@";; # convert '-abc' to '-a' '-b' '-c' --) while [ "$1" != "$EOL" ]; do set -- "$@" "$1"; shift; done;; # process remaining arguments as positional -*) usage_error "unknown option: '$opt'";; # catch misspelled options *) usage_error "this should NEVER happen ($opt)";; # sanity test for previous patterns esac done shift # $EOL fi rangeupload () { echo "# Testing HTTP range upload" NAME=$1 DIR=/tmp/X-UPLOAD/ mkdir -p $DIR rm -f $DIR/$NAME* CHUNK_NUMBER=4 dd if=/dev/zero of=$DIR/$NAME bs=1M count=32 split -b 10485760 -a 1 -d $DIR/$NAME $DIR/$NAME-chunking-`uuidgen | sed s/-//g`-$CHUNK_NUMBER- echo "# about to upload $DIR/$NAME" DEST_URL=https://${host}:${port}$2 echo "# to $DEST_URL" let LAST_CHUNK_NUMBER=$CHUNK_NUMBER-1 let i=0 UUID=`echo $RANDOM` ok=0 checksum=`eos-adler32 $DIR/$NAME | awk '{print $4}' | sed s/=/:/g`; for f in `ls $DIR/$NAME-chunking*`; do echo $f EOS_FN=$NAME let start=$i*10485760 let stop=$i+1; let stop=$stop*10485760; if [ $stop -ge 33554432 ]; then let stop=33554431; else let stop=$stop-1; fi echo $checksum start=$start stop=$stop curl --capath "${capath}" --cert "${cert}" --cacert "${cacert}" --key "${key}" --header "x-upload-checksum: $checksum" --header "x-upload-totalsize: 33554432" --header "x-upload-mtime: 1533100000" --header "x-upload-range: bytes=$start-$stop" -L -X PUT -T $f $DEST_URL$EOS_FN >> $DIR/$NAME.log 2>&1 let i=$i+1 done sleep 1 if [ $ok -eq 0 ]; then curl --capath "${capath}" --cert "${cert}" --cacert "${cacert}" --key "${key}" -i $DEST_URL$EOS_FN >> $DIR/$NAME.log 2>&1 ok=$? echo HEAD request on $DIR/$NAME gave error=$ok fi if [ $ok -eq 0 ]; then cks=`echo $checksum|sed s/adler32://g`; cat $DIR/$NAME.log | grep ETag | grep $cks >& /dev/null ok=$? echo CHECKSUM verification on $DIR/$NAME gave error=$ok fi return $ok; } echo "${POSITIONAL_ARGS[@]}" "$@" "$#" if [ "$#" -eq 3 ] && [ "$1" = "rangeupload" ]; then rangeupload $2 $3; exit $? else echo "usage: eos-http-upload-test rangeupload testfile /eos/dev/upload-directory/" fi exit -1