#!/bin/sh
#
# This file is released under the terms of the Artistic License.
# Please see the file LICENSE, included in this package, for details.
#
# Copyright (C) 2002-2006 Open Source Development Labs, Inc.
#               2002-2022 Mark Wong

trap 'echo "Test was interrupted by Control-C."; \
	killall dbt2-client dbt2-driver sar sadc pidstat; dbt2-${DBMS}-stop-db; killall dbt2-${DBMS}-db-stat sar sadc pidstat' INT
trap 'echo "Test was interrupted. Got TERM signal."; \
	killall dbt2-client dbt2-driver sar sadc pidstat; dbt2-${DBMS}-stop-db; killall dbt2-${DBMS}-db-stat sar sadc pidstat' TERM

do_sleep()
{
	echo "Sleeping $1 seconds"
	sleep $1
}

make_directories()
{
	COMMAND=""
	HOST=${1}
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	fi
	${COMMAND} mkdir -p ${OUTPUT_DIR}
}

oprofile_annotate()
{
	COMMAND=""
	DIR=${1}
	HOST=${2}
	OPROFILE_DIR=${DIR}/oprofile
	ANNOTATED_DIR=${OPROFILE_DIR}/annotated
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	else
		HOST=`hostname`
	fi
	echo "oprofile is annotating source: ${HOST}"
	${COMMAND} mkdir -p ${OPROFILE_DIR}
	${COMMAND} mkdir -p ${ANNOTATED_DIR}
	if [ -n "${COMMAND}" ]; then
		${COMMAND} "sudo opannotate --source \
				--assembly > ${OPROFILE_DIR}/assembly.txt 2>&1"
	else
		sudo opannotate --source --assembly > ${OPROFILE_DIR}/assembly.txt 2>&1
	fi
	${COMMAND} sudo opannotate --source --output-dir=${ANNOTATED_DIR}
	IAM=`whoami`
	${COMMAND} sudo chown -R ${IAM} ${OPROFILE_DIR}
}

oprofile_collect()
{
	COMMAND=""
	DIR=${1}
	HOST=${2}
	OPROFILE_DIR=${DIR}/oprofile
	ANNOTATED_DIR=${OPROFILE_DIR}/annotated
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	else
		HOST=`hostname`
	fi
	mkdir -p ${OPROFILE_DIR}
	mkdir -p ${ANNOTATED_DIR}
	#
	# I don't think we need any output dumped to the terminal.
	#
	echo "oprofile is dumping data: ${HOST}"
	${COMMAND} sudo opcontrol --dump
	if [ -n "${COMMAND}" ]; then
		${COMMAND} mkdir -p ${OPROFILE_DIR}
		${COMMAND} mkdir -p ${ANNOTATED_DIR}
		echo "Generating oprofile profile on '${HOST}'..."
		${COMMAND} "sudo opreport -l -p /lib/modules/`${COMMAND} uname -r` -o ${DIR}/oprofile.txt > ${OPROFILE_DIR}/oprofile.log 2>&1"
		echo "Generating oprofile callgraph on '${HOST}'..."
		${COMMAND} "sudo opreport -l -c -p /lib/modules/`${COMMAND} uname -r` -o ${OPROFILE_DIR}/callgraph.txt > ${OPROFILE_DIR}/oprofile.log 2>&1"
	else
		echo "Generating oprofile profile..."
		sudo opreport -l -p /lib/modules/`uname -r` \
				-o ${OPROFILE_DIR}/oprofile.txt > \
				${OPROFILE_DIR}/oprofile.log 2>&1
		echo "Generating oprofile callgraph..."
		sudo opreport -l -c -p /lib/modules/`uname -r` \
				-o ${OPROFILE_DIR}/callgraph.txt > \
				${OPROFILE_DIR}/oprofile.log 2>&1
	fi
	${COMMAND} sudo opcontrol --stop
	IAM=`whoami`
	${COMMAND} sudo chown -R ${IAM} ${OPROFILE_DIR}
}

oprofile_init()
{
	COMMAND=""
	HOST=${1}
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	else
		HOST=`hostname`
	fi
	echo "starting oprofile: ${HOST}"
	${COMMAND} sudo opcontrol \
			--vmlinux=/lib/modules/`${COMMAND} uname -r`/vmlinux \
			-c 100
	sleep 1
	START_ARGS=""
	MACHINE=`${COMMAND} uname -m`
	if [ "${MACHINE}" = "ppc64" ]; then
		#
		# Oprofile fails to work on ppc64 because the defaults settings
		# are invalid on this platform.  Why isn't it smart enough to
		# have valid default setting depending on arch?
		#
		START_ARGS="-e CYCLES:150000:0:1:1"
	fi
	${COMMAND} sudo opcontrol --start-daemon ${START_ARGS}
	sleep 1
	${COMMAND} sudo opcontrol --start
}

oprofile_reset()
{
	COMMAND=""
	HOST=${1}
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	else
		HOST=`hostname`
	fi
	echo "resetting oprofile counters: ${HOST}"
	${COMMAND} sudo opcontrol --reset
}

oprofile_stop()
{
	COMMAND=""
	HOST=${1}
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	else
		HOST=`hostname`
	fi
	echo "stopping oprofile daemon: ${HOST}"
	${COMMAND} sudo opcontrol --shutdown
}

perf_start()
{
	COMMAND=""
	DIR=${1}
	HOST=${2}

	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
		HOSTNAME=$($COMMAND hostname)
	else
		HOSTNAME=$(hostname)
	fi

	RC=`$COMMAND cat /proc/sys/kernel/perf_event_paranoid`
	if [ $RC -ne -1 ]; then
		echo "WARNING: Give the current user permission to use perf by setting"
		echo "         /proc/sys/kernel/perf_event_paranoid to -1"
		return 0
	fi
	PERFDATA="${DIR}/${HOSTNAME}/perf.data"
	$COMMAND perf record -o $PERFDATA -a -g -s sleep 10

	# Default permissions varies per distro, set it readable explicitly...
	$COMMAND chmod 644 $PERFDATA
}

perf_collect()
{
	COMMAND=""
	DIR=${1}
	HOST=${2}

	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
		HOSTNAME=$($COMMAND hostname)
	else
		COMMAND="eval"
		HOSTNAME=`hostname`
	fi

	THISDIR="${DIR}/${HOSTNAME}"
	PERFDATA="${THISDIR}/perf.data"
	echo "Generating perf report on '${HOSTNAME}'..."
	$COMMAND "perf report -i $PERFDATA -n > ${THISDIR}/perf-report.txt 2> /dev/null"
	echo "Generating perf annotated source on '${HOSTNAME}'..."
	$COMMAND "perf annotate -l -P -i $PERFDATA > ${THISDIR}/perf-annotate.txt 2> /dev/null"
	echo "Generating perf traces on '${HOSTNAME}'..."
	$COMMAND "perf script -L -i $PERFDATA > ${THISDIR}/perf-trace.txt 2> /dev/null"
	$COMMAND "perf script -i $PERFDATA > ${THISDIR}/perf.txt 2> /dev/null"
}

post_process_sar()
{
	# Sar data should be processed on the system they are collected on as sar
	# versions can vary between systems and thus not be able to read another
	# version's raw data file.
	COMMAND=""
	SARPATH=${1}
	HOST=${2}
	if [ ! "x${HOST}" = "x" ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		COMMAND="$SSH ${HOST}"
		HOSTNAME=$($COMMAND hostname)
		FILE="${SARPATH}/${HOSTNAME}/sar_raw.out"
		${COMMAND} "which sadf > /dev/null 2>&1"
		if [ $? -eq 0 ]; then
			SADF="sadf -d -U"
		    $COMMAND "$SADF 1 1 > /dev/null 2>&1"
			RC=$?
			if [ $RC -ne 0 ] && [ $RC -ne 2 ]; then
		        SADF="sadf -D"
		        $COMMAND "$SADF 1 1 > /dev/null 2>&1"
				RC=$?
				if [ $RC -ne 0 ] && [ $RC -ne 2 ]; then
		            echo "WARNING: Was not able to determine proper sadf flags"
		            SADF="true"
		        fi
		    fi
		else
			SADF=true
		fi
		${COMMAND} "${SADF} -P ALL ${FILE} > `dirname ${FILE}`/sar-cpu.csv"
		${COMMAND} "${SADF} ${FILE} -- -B > `dirname ${FILE}`/sar-paging.csv"
		$COMMAND "$SADF $FILE -- -d -p > `dirname $FILE`/sar-blockdev.csv"
		$COMMAND "$SADF 1 1 -- -R > /dev/null 2>&1"
		if [ $? -eq 0 ]; then
			$COMMAND "$SADF $FILE -- -R > `dirname $FILE`/sar-mem1.csv"
			$COMMAND "$SADF $FILE -- -r > `dirname $FILE`/sar-mem2.csv"
		else
			$COMMAND "$SADF $FILE -- -r > `dirname $FILE`/sar-mem.csv"
		fi
		${COMMAND} "${SADF} ${FILE} -- -W > `dirname ${FILE}`/sar-swap.csv"
		$COMMAND "$SADF $FILE -- -n DEV > `dirname $FILE`/sar-net.csv"
	else
		FILE="${SARPATH}/$(hostname)/sar_raw.out"
		which sadf > /dev/null 2>&1
		if [ $? -eq 0 ]; then
			SADF="sadf -d -U"
		    $SADF 1 1 > /dev/null 2>&1
			RC=$?
			if [ $RC -ne 0 ] && [ $RC -ne 2 ]; then
		        SADF="sadf -D"
				RC=$?
		        $SADF 1 1 > /dev/null 2>&1
				if [ $RC -ne 0 ] && [ $RC -ne 2 ]; then
		            echo "WARNING: Was not able to determine proper sadf flags"
		            SADF="true"
		        fi
		    fi
		else
			SADF=true
		fi
		${SADF} -P ALL ${FILE} > `dirname ${FILE}`/sar-cpu.csv
		${SADF} ${FILE} -- -B > `dirname ${FILE}`/sar-paging.csv
		${SADF} ${FILE} -- -d -p > `dirname ${FILE}`/sar-blockdev.csv
		$SADF 1 1 -- -R > /dev/null 2>&1
		if [ $? -eq 0 ]; then
			$SADF $FILE -- -R > `dirname $FILE`/sar-mem1.csv
			$SADF $FILE -- -r > `dirname $FILE`/sar-mem2.csv
		else
			$SADF $FILE -- -r > `dirname $FILE`/sar-mem.csv
		fi
		${SADF} ${FILE} -- -W > `dirname ${FILE}`/sar-swap.csv
		$SADF $FILE -- -n DEV > `dirname $FILE`/sar-net.csv
	fi
}

readprofile_collect()
{
	COMMAND=""
	DIR=${1}
	HOST=${2}
	RP_DIR=${DIR}/readprofile
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	else
		HOST=`hostname`
	fi
	echo "collecting readprofile data: ${HOST}"
	PROFILE=${RP_DIR}/readprofile.txt
	if [ -n "${COMMAND}" ]; then
		${COMMAND} mkdir -p ${RP_DIR}
		${COMMAND} "/usr/sbin/readprofile -n -m /boot/System.map-`${COMMAND} uname -r` > ${PROFILE}"
		$COMMAND "cat $PROFILE | sort -n -r -k1 > $RP_DIR/readprofile_ticks.txt"
		$COMMAND "cat $PROFILE | sort -n -r -k3 > $RP_DIR/readprofile_load.txt"
	else
		mkdir -p ${RP_DIR}
		/usr/sbin/readprofile -n -m /boot/System.map-`uname -r` > ${PROFILE}
		cat ${PROFILE} | sort -n -r -k1 > ${RP_DIR}/readprofile_ticks.txt
		cat ${PROFILE} | sort -n -r -k3 > ${RP_DIR}/readprofile_load.txt
	fi
}

readprofile_clear()
{
	COMMAND=""
	HOST=${1}
	if [ -n "${HOST}" ]; then
		COMMAND="$SSH ${HOST}"
	else
		HOST=`hostname`
	fi
	echo "clearing readprofile data: ${HOST}"
	${COMMAND} sudo /usr/sbin/readprofile -r
}

usage()
{
	if [ "$1" != "" ]; then
		echo
		echo "error: $1"
	fi
	echo
	echo "`basename $0` is the DBT-2 workload runner"
	echo ""
	echo "Usage:"
	echo "  `basename $0` [OPTIONS]"
	echo
	echo "Generate options:"
	echo "  -a DBMS       dbms under test (pgsql)"
	echo "  -A            enable altered execution mode"
	echo "  -b #          Number of warehouses per process, default all specified by -w"
	if [ ! "x$DRIVER3" = "x1" ]; then
		echo "  -c #          number of database connection to open"
		echo "  -C ADDRESS    address of client system, default localhost"
	fi
	echo "  -d #          seconds of the test duration, after ramp up"
	echo "  -D NAME       name of the database"
	if [ "x$DRIVER3" = "x1" ]; then
		echo "  -F #          forks per processor, default 1"
	fi
	echo "  -h            this usage message"
	echo "  -H ADDRESS    address of database system, default localhost"
	echo "  -l PORT       database listening port number"
	echo "  -L #          limit the number of terminals emulated"
	echo "  -n            disable thinking and keying time"
	echo "  -o PATH       path to store test results"
	echo "  -q            enable oprofile data collection"
	echo "  -r            enable perf data collection"
	echo "  -s #          millisecond delay between thread creation"
	echo "  -t #          number of districts per warehouse, default 10"
	echo "  -u            run test as an unprivileged user"
	echo "  -w #          number of warehouses to use in the database"
	echo "  -y            database server is a service"
	echo "  -z COMMENT    comments to save for the test"
	echo
	echo "MySQL specific options:"
	echo "  -U USER        database user"
	echo "  -x PASSWORD    database password"
	echo
	echo "PostgreSQL specific options:"
	echo "  -p OPTIONS    GUC command line options to pass to postgres"
	echo
	echo "DBMS options are:"
	echo "  cockroach  CockroachDB"
	echo "  mysql      MySQL"
	echo "  pgsql      PostgreSQL"
	echo "  sqlite     SQLite"
}

validate_parameter()
{
	if [ "$2" != "$3" ]; then
		usage "wrong argument '$2' for parameter '-$1'"
		exit 1
	fi
}

SSH="ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

DB_HOSTNAME="localhost"
DB_PASSWORD=""
CLIENT_HOSTNAME=""
WORKING_DIR=""

DB_USER=${DBUSER}
DBAAS=0
FPP=1
PRIVILEGED=0
THREAD_STARTUP_DELAY=1000 # milliseconds
USE_OPROFILE=0
USE_PERF=0
THREADS_PER_WAREHOUSE=10
USE_PGPOOL=0
DBNAME=${DBT2DBNAME}

while getopts "Aa:b:C:c:D:d:F:H:hi:L:l:no:p:qrS:s:t:U:uw:x:yz:" opt; do
	case $opt in
	A)
		ALTERED_ARG="-altered 1"
		;;
	a)
		DBMS=${OPTARG}
		export DATABASE=${DBMS}
		;;
	b)
		# Number of warehouses per driver.
		W_CHUNK=${OPTARG}
		;;
	C)
		CLIENT_HOSTNAME=$OPTARG
		;;
	c)
		# Check for numeric value
		DBCON=`echo $OPTARG | egrep "^[0-9]+$"`
		validate_parameter $opt $OPTARG $DBCON
		;;
	D)
		DBNAME=${OPTARG}
		;;
	d)
		DURATION=`echo $OPTARG | egrep "^[0-9]+$"`
		validate_parameter $opt $OPTARG $DURATION
		;;
	F)
		FPP="$OPTARG"
		FPP_FLAG="-fpp $FPP"
		;;
	H)
		DB_HOSTNAME=${OPTARG}
		;;
	h)
		usage
		exit 1
		;;
	L)
		TERMINALS_LIMIT="$OPTARG"
		TERMINALS_LIMIT_ARG="-L $TERMINALS_LIMIT"
		;;
	l)
		DB_PORT=`echo $OPTARG | egrep "^[0-9]+$"`
		validate_parameter $opt $OPTARG $DB_PORT
		;;
	n)
		NO_THINK="-ktd 0 -ktn 0 -kto 0 -ktp 0 -kts 0 -ttd 0 -ttn 0 -tto 0 -ttp 0 -tts 0"
		;;
	o)
		OUTPUT_DIR=${OPTARG}
		;;
	p)
		DB_PARAMS="${OPTARG}"
		DB_PARAMS_ARG="-p \"${OPTARG}\""
		;;
	q)
		USE_OPROFILE=1
		;;
	r)
		USE_PERF=1
		;;
	S)
		SOCK_FILE=${OPTARG}
		SOCK_FILE_ARG="-t ${OPTARG}"
		;;
	s)
		THREAD_STARTUP_DELAY=`echo $OPTARG | egrep "^[0-9]+$"`
		validate_parameter $opt $OPTARG $THREAD_STARTUP_DELAY
		;;
	t)
		THREADS_PER_WAREHOUSE=`echo ${OPTARG} | egrep "^[0-9]+$"`
		validate_parameter $opt $OPTARG $THREADS_PER_WAREHOUSE
		;;
	u)
		PRIVILEGED=1
		;;
	U)
		DB_USER=${OPTARG}
		;;
	w)
		WAREHOUSES=`echo $OPTARG | egrep "^[0-9]+$"`
		validate_parameter $opt $OPTARG $WAREHOUSES
		;;
	x)
		DB_PASSWORD=${OPTARG}
		;;
	y)
		DBAAS=1
		;;
	z)
		COMMENT=$OPTARG
		;;
	esac
done

# If no client hostnames have been specified, set it to localhost.
if [ "x${CLIENT_HOSTNAME}" = "x" ]; then
	CLIENT_HOSTNAME="localhost"
fi

# If no chunking of warehouses per process is defined, assumed we want to run
# everything from one process.
if [ "x${W_CHUNK}" = "x" ]; then
	W_CHUNK=${WAREHOUSES}
fi

# Check parameters.

if [ "x${DBMS}" = "x" ]; then
	echo -n "specify which dbms system to test using -a "
	echo "<cockroach|mysql|pgsql|sqlite>"
	exit 1
fi

if [ "x${DBNAME}" = "x" ] && [ ! "x${DBMS}" = "xsqlite" ]; then
	echo "specify the database name using -D "
	echo "or by setting the DBT2DBNAME environment variable"
	exit 1
fi

if [ "x${DBCON}" = "x" ] && [ ! "x$DRIVER3" = "x1" ]; then
	echo "specify the number of database connections using -c #"
	exit 1
fi

if [ "x${DURATION}" = "x" ]; then
	echo "specify the duration of the test in seconds using -d #"
	exit 1
fi

if [ "x${WAREHOUSES}" = "x" ]; then
	echo "specify the number of warehouses using -w #"
	exit 1
fi

if [ "x${OUTPUT_DIR}" = "x" ]; then
	echo "specify the location for results, directory must not exist, using -o <path>"
	exit 1
fi

if [ -d "${OUTPUT_DIR}" ]; then
	echo "directory '${OUTPUT_DIR}' exists, stopping to prevent possible"
	echo "clobbering of data, please specify another -o location"
	exit 1
fi

if [ ${THREADS_PER_WAREHOUSE} -lt 1 ] || [ ${THREADS_PER_WAREHOUSE} -gt 1000 ];
then
	usage "-t value should be in range [1..1000]. Please specify correct value"
	exit 1
fi

if [ ! "x${DB_PORT}" = "x" ]; then
	if [ "x$DRIVER3" = "x1" ]; then
		DB_PORT_ARG="-P ${DB_PORT}"
	else
		DB_PORT_ARG="-l ${DB_PORT}"
	fi
fi

CLIENT_OUTPUT_DIR=${OUTPUT_DIR}/client
DRIVER_OUTPUT_DIR=${OUTPUT_DIR}/driver
DB_OUTPUT_DIR=${OUTPUT_DIR}/db

SUMMARY="${OUTPUT_DIR}/summary.rst"

#
# Create the directories we will need.
#
make_directories
mkdir -p ${DRIVER_OUTPUT_DIR}/`hostname`
if [ ! "x${CLIENT_HOSTNAME}" = "xlocalhost" ]; then
	HOSTNAME=$($SSH $CLIENT_HOSTNAME hostname)
	make_directories $CLIENT_HOSTNAME
	${COMMAND} -- mkdir -p ${CLIENT_OUTPUT_DIR}/${HOSTNAME}
fi
if [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
	#
	# Create direcotires on the database server if remote.
	#
	if [ $DBAAS -eq 0 ]; then
		make_directories ${DB_HOSTNAME}
		$SSH "$DB_HOSTNAME" -- mkdir -p "$DB_OUTPUT_DIR"
		HOSTNAME=$($SSH "$DB_HOSTNAME" hostname)
		$SSH "$DB_HOSTNAME" dbt2-get-os-info -o "${DB_OUTPUT_DIR}/$HOSTNAME"
	fi
fi
mkdir -p ${DB_OUTPUT_DIR}

# Create a readme file in the output directory and date it.
date >> ${OUTPUT_DIR}/readme.txt
echo "$COMMENT" >> ${OUTPUT_DIR}/readme.txt
echo "Command line: $0 $@" >> ${OUTPUT_DIR}/readme.txt

# Output run information into the readme.txt.
echo "RDBMS: $DBMS" >> $OUTPUT_DIR/readme.txt
echo "Database Name: $DBNAME" >> $OUTPUT_DIR/readme.txt
echo "Database Scale Factor: $WAREHOUSES warehouses" >> $OUTPUT_DIR/readme.txt
echo "Test Duration: $DURATION seconds" >> ${OUTPUT_DIR}/readme.txt
echo "Database Connections: $DBCON" >> ${OUTPUT_DIR}/readme.txt

# Get any OS specific information, if this is a single tier configuration, then
# the OS parameters will be in the database output directory.
dbt2-get-os-info -o $DRIVER_OUTPUT_DIR/$(hostname)
if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
	DB_COMMAND="$SSH ${DB_HOSTNAME}"
else
	DB_COMMAND="eval"
fi
# Get the OS information from the database system.
if [ $DBAAS -eq 0 ]; then
	DB_COMMAND2="$DB_COMMAND"
else
	DB_COMMAND2="eval"
fi

if [ $PRIVILEGED -eq 1 ]; then
	${DB_COMMAND} "dbt2-${DBMS}-stop-db ${MISC_ARG}"
	if [ $? -ne 0 ]; then
		echo "cleaning up results..."
		rm -rf ${OUTPUT_DIR}
		exit 1
	fi
	${DB_COMMAND} "dbt2-${DBMS}-start-db ${DB_PARAMS_ARG} ${MISC_ARG} -o ${DB_OUTPUT_DIR}" || exit 1
fi

#
# Redisplay the test parameters.
#
echo "DBT-2 test for ${DBMS} started at $(date)..."
echo ""
echo "DATABASE SYSTEM: ${DB_HOSTNAME}"
if [ "${DBMS}" = "cockroach" ] || [ "${DBMS}" = "pgsql" ] || \
		[ "${DBMS}" = "mysql" ]; then
	echo "DATABASE NAME: ${DBNAME}"
fi

#
# Build up the client command line arguments.
#

CLIENT_COMMAND_ARGS="-a $DBMS"

if [ -n "${DB_USER}" ]; then
	echo "DATABASE USER: ${DB_USER}"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -u ${DB_USER}"
fi

if [ -n "${DB_PASSWORD}" ]; then
	echo "DATABASE PASSWORD: *******"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -a ${DB_PASSWORD}"
fi

if [ -n "${DB_SOCKET}" ]; then
	echo "DATABASE SOCKET: ${DB_SOCKET}"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -t ${DB_SOCKET}"
fi

if [ -n "${DB_PORT}" ]; then
	echo "DATABASE PORT: ${DB_PORT}"
	CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} ${DB_PORT_ARG}"
fi

OPTION1=`expr $WAREHOUSES \* $THREADS_PER_WAREHOUSE`
THREADS=$OPTION1
if [ ! "x$TERMINALS_LIMIT" = "x" ]; then
	OPTION2=$TERMINALS_LIMIT
	if [ $OPTION1 -lt $OPTION2 ]; then
		THREADS=$OPTION1
	else
		THREADS=$OPTION2
	fi
fi
echo "DATABASE CONNECTIONS: ${DBCON}"
echo "TERMINAL THREADS: ${THREADS}"
echo "TERMINALS PER WAREHOUSE: ${THREADS_PER_WAREHOUSE}"
echo "TERMINAL CAP: $TERMINALS_LIMIT"
echo "WAREHOUSES PER DRIVER/CLIENT PAIR: $W_CHUNK"
echo "SCALE FACTOR (WAREHOUSES): ${WAREHOUSES}"
echo "DURATION OF TEST (in sec): ${DURATION}"
echo ""

#
# Start the client.
#
echo "Stage 1. Starting up client..."
if [ ! "x$DRIVER3" = "x1" ]; then
	echo "database connection starting every $THREAD_STARTUP_DELAY milliseconds"
	if [ ${DBMS} = "cockroach" ]; then
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DB_HOSTNAME} -b ${DBNAME}"
	elif [ ${DBMS} = "mysql" ]; then
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DBNAME}"
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} ${SOCK_FILE_ARG} -u root"
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -h ${DB_HOSTNAME}"
	elif [ ${DBMS} = "pgsql" ]; then
		if [ ${USE_PGPOOL} -eq 1 ]; then
			echo "Starting pgpool..."
			pgpool -f ${DIR}/pgsql/pgpool.conf
			TMP_DB_HOSTNAME="localhost"
		else
			TMP_DB_HOSTNAME=${DB_HOSTNAME}
		fi
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${TMP_DB_HOSTNAME} -b ${DBNAME}"
	elif [ ${DBMS} = "sqlite" ]; then
		CLIENT_COMMAND_ARGS="${CLIENT_COMMAND_ARGS} -d ${DB_PARAMS}"
	fi
	CLIENT_COMMAND_ARGS="$CLIENT_COMMAND_ARGS -f -c $DBCON -s $THREAD_STARTUP_DELAY"
	END=0
	START=1

	if [ ! "x${CLIENT_HOSTNAME}" = "xlocalhost" ]; then
		CLIENT_COMMAND="$SSH $CLIENT_HOSTNAME --"
	else
		CLIENT_COMMAND="eval"
	fi

	#
	# Count of the number of clients to run.
	#
	SEG=0
	PORT=30000
	while [ ${END} -lt ${WAREHOUSES} ]; do
		START=`expr ${END} + 1`
		END=`expr ${START} + ${W_CHUNK}`
		END=`expr ${END} - 1`
		SEG=`expr ${SEG} + 1`

		if [ ${END} -gt ${WAREHOUSES} ]; then
			END=${WAREHOUSES}
		fi

		HOSTNAME=`$CLIENT_COMMAND hostname`
		CPDIR="${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
		mkdir -p ${CPDIR}

		CDIR="${CLIENT_OUTPUT_DIR}/${HOSTNAME}.${SEG}"
		$CLIENT_COMMAND mkdir -p ${CDIR}

		CARGS="${CLIENT_COMMAND_ARGS} -p ${PORT} -o ${CDIR}"
		if [ ! "x$DRIVER3" = "x1" ]; then
			$CLIENT_COMMAND "dbt2-client ${CARGS}" > \
					${CPDIR}/client-${SEG}.txt 2>&1 &
		fi

		PORT=`expr ${PORT} + 1`
	done

	# Sleep long enough for all the client database connections to be established.
	CLIENT_RAMPUP_TIME=$DBCON
	CLIENT_RAMPUP_TIME=`expr $CLIENT_RAMPUP_TIME \* $THREAD_STARTUP_DELAY`
	CLIENT_RAMPUP_TIME=`expr $CLIENT_RAMPUP_TIME / 1000`
	CLIENT_RAMPUP_TIME=`expr $CLIENT_RAMPUP_TIME + 1`
	do_sleep $CLIENT_RAMPUP_TIME
else
	echo "***skipping***"
fi

# Start collecting data before we start the test.

# Calculate how many terminals are being emulated.
OPTION1=`expr $W_CHUNK + 1`
OPTION1=`expr $OPTION1 \* $THREADS_PER_WAREHOUSE`
DRIVER_RAMPUP_TIME=$OPTION1
if [ ! "x$TERMINALS_LIMIT" = "x" ]; then
	OPTION2=$TERMINALS_LIMIT
	if [ $OPTION1 -lt $OPTION2 ]; then
		DRIVER_RAMPUP_TIME=$OPTION1
	else
		DRIVER_RAMPUP_TIME=$OPTION2
	fi
fi
if [ "x$DRIVER3" = "x1" ]; then
	DRIVER_RAMPUP_TIME=$(( ($(nproc) * $FPP) - 1 ))
fi
DRIVER_RAMPUP_TIME=`expr $DRIVER_RAMPUP_TIME \* $THREAD_STARTUP_DELAY`
DRIVER_RAMPUP_TIME=`expr $DRIVER_RAMPUP_TIME / 1000`
TOTAL_RUN_TIME=`expr $DRIVER_RAMPUP_TIME + $DURATION`
SAMPLE_LENGTH=60
ITERATIONS=`expr $TOTAL_RUN_TIME / $SAMPLE_LENGTH`
ITERATIONS=`expr ${ITERATIONS} + 1`

# Collect system statistics on the Client node, if remote.
if [ ! "x${CLIENT_HOSTNAME}" = "xlocalhost" ]; then
	HOSTNAME=`$CLIENT_COMMAND hostname`
	CARGS="--iter $ITERATIONS --sample $SAMPLE_LENGTH"
	CARGS="$CARGS --outdir ${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
	$CLIENT_COMMAND "dbt2-get-os-info -o ${CLIENT_OUTPUT_DIR}/${HOSTNAME}"
	$CLIENT_COMMAND "dbt2-sysstats $CARGS" > \
			${CLIENT_OUTPUT_DIR}/${HOSTNAME}/stats.txt 2>&1 &
fi

# Unless the reporting can be made more intelligent, if this is a 1-tier
# configuration, save the Driver node's stats in the database directory.
if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
	# Since the database system is a remote system, collect the Drive node's
	# system statistics in the driver directory.
	dbt2-sysstats \
			--iter ${ITERATIONS} \
			--sample ${SAMPLE_LENGTH} \
			--outdir ${DRIVER_OUTPUT_DIR}/`hostname` > \
			${DRIVER_OUTPUT_DIR}/`hostname`/stats.txt 2>&1 &

	# Don't collect system stats from the database system if the database is a
	# service.
	if [ $DBAAS -eq 0 ]; then
		HOSTNAME=$($DB_COMMAND hostname)
		CARGS="--iter ${ITERATIONS} --sample ${SAMPLE_LENGTH}"
		CARGS="$CARGS --outdir ${DB_OUTPUT_DIR}/${HOSTNAME}"
		${DB_COMMAND} "dbt2-sysstats $CARGS" \
				> ${DB_OUTPUT_DIR}/stats.txt 2>&1 &
	fi
else
	# This is supposed to be a 1-tier configuration so save the Driver node's
	# stats into the database directory.
	dbt2-sysstats \
			--iter ${ITERATIONS} \
			--sample ${SAMPLE_LENGTH} \
			--outdir ${DB_OUTPUT_DIR} > \
			${DRIVER_OUTPUT_DIR}/`hostname`/stats.txt 2>&1 &
fi

echo "collecting database statistics..."
HOSTNAME="$($DB_COMMAND2 hostname)"
$DB_COMMAND2 mkdir -p ${DB_OUTPUT_DIR}/${HOSTNAME}
CARGS="-D $DBNAME -o ${DB_OUTPUT_DIR}/${HOSTNAME} -i $ITERATIONS"
CARGS="$CARGS -s $SAMPLE_LENGTH $DB_PORT_ARG"
${DB_COMMAND2} "dbt2-${DBMS}-db-stat $CARGS > ${DB_OUTPUT_DIR}/${HOSTNAME}/dbstats.txt 2>&1" &

# Initialize oprofile before we start the driver.
if [ ${USE_OPROFILE} -eq 1 ]; then
	oprofile_init
	if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		oprofile_init ${DB_HOSTNAME}
	fi
fi

# Start the driver.
echo ''
echo "Stage 2. Starting up driver..."
echo "thread starting every $THREAD_STARTUP_DELAY milliseconds"

END=0
START=1
#
# Count of the number of drivers run.
#
SEG=0
PORT=30000
while [ ${END} -lt ${WAREHOUSES} ]; do
	START=`expr ${END} + 1`
	END=`expr ${START} + ${W_CHUNK}`
	END=`expr ${END} - 1`
	SEG=`expr ${SEG} + 1`

	if [ ${END} -gt ${WAREHOUSES} ]; then
		END=${WAREHOUSES}
	fi

	DDIR="${DRIVER_OUTPUT_DIR}/`hostname`.${SEG}"
	mkdir -p ${DDIR}
	DRIVER_COMMAND_ARGS="-l $DURATION -wmin $START"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -wmax $END -w $WAREHOUSES"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -sleep $THREAD_STARTUP_DELAY"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -outdir $DDIR"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS -tpw $THREADS_PER_WAREHOUSE"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS $NO_THINK $TERMINALS_LIMIT_ARG"
	DRIVER_COMMAND_ARGS="$DRIVER_COMMAND_ARGS $ALTERED_ARG $DB_PORT_ARG"
	if [ "x$DRIVER3" = "x1" ]; then
		echo "***experimental driver 3 active***"
		if [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
			DHNARG="-b $DBNAME -d $DB_HOSTNAME"
		fi
		dbt2-driver3 ${DRIVER_COMMAND_ARGS} -a $DBMS $DHNARG $FPP_FLAG > \
				${DRIVER_OUTPUT_DIR}/`hostname`/driver-${SEG}.txt 2>&1 \
				|| exit 1 &
	else
		dbt2-driver -d $CLIENT_HOSTNAME -p $PORT ${DRIVER_COMMAND_ARGS} > \
				${DRIVER_OUTPUT_DIR}/`hostname`/driver-${SEG}.txt 2>&1 \
				|| exit 1 &
	fi
	DRIVER_PIDLIST="$DRIVER_PIDLIST $!"
	PORT=`expr ${PORT} + 1`
done

echo -n "estimated rampup time: "
do_sleep $DRIVER_RAMPUP_TIME
echo "estimated rampup time has elapsed"

# Clear the readprofile data after the driver ramps up.
if [ -f "/proc/profile" ]; then
	readprofile_clear
	if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		readprofile_clear ${DB_HOSTNAME}
	fi
fi

# Reset the oprofile counters after the driver ramps up.
if [ ${USE_OPROFILE} -eq 1 ]; then
	oprofile_reset
	if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		oprofile_reset ${DB_HOSTNAME}
	fi
fi

# Start perf data collection
if [ ${USE_PERF} -eq 1 ]; then
	perf_start ${OUTPUT_DIR}/driver &
	if [ -n ${CLIENT_HOSTNAME} ] && [ ! "x${CLIENT_HOSTNAME}" = "xlocalhost" ]; then
		perf_start ${OUTPUT_DIR}/client ${CLIENT_HOSTNAME} &
	fi
	if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		perf_start ${DB_OUTPUT_DIR} ${DB_HOSTNAME} &
	fi
fi

# Sleep for the duration of the run.
echo -n "estimated steady state time: "
do_sleep $DURATION

echo ''
echo "Stage 3. Processing of results..."

if [ ! "x$DRIVER3" = "x1" ]; then
	# Client doesn't go away by itself like the driver does, so kill it.
	echo "Killing client..."
	if [ ! "x${CLIENT_HOSTNAME}" = "xlocalhost" ]; then
		CLIENT_COMMAND="$SSH $CLIENT_HOSTNAME --"
	else
		CLIENT_COMMAND="eval"
	fi
	$CLIENT_COMMAND "killall -q dbt2-client" > /dev/null 2>&1
fi

if [ $PRIVILEGED -eq 1 ]; then
	${DB_COMMAND} "dbt2-${DBMS}-stop-db ${MISC_ARG}"
	if [ ${DBMS} = "pgsql" ]; then
		if [ ${USE_PGPOOL} -eq 1 ]; then
			pgpool -f ${DIR}/pgsql/pgpool.conf stop
		fi
	fi
fi

# Collect readprofile data.
if [ -f "/proc/profile" ]; then
	readprofile_collect ${DRIVER_OUTPUT_DIR}/`hostname`
	if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		readprofile_collect ${DB_OUTPUT_DIR} ${DB_HOSTNAME}
	fi
fi

# Collect oprofile data.
if [ ${USE_OPROFILE} -eq 1 ]; then
	oprofile_collect ${OUTPUT_DIR}
	if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		oprofile_collect ${DB_OUTPUT_DIR} ${DB_HOSTNAME}
	fi
fi

# Collect perf data.
if [ $USE_PERF -eq 1 ]; then
	perf_collect ${OUTPUT_DIR}/driver &
	PERF_PIDLIST="$!"
	if [ -n ${CLIENT_HOSTNAME} ] && [ ! "x${CLIENT_HOSTNAME}" = "xlocalhost" ]; then
		perf_collect ${OUTPUT_DIR}/client ${CLIENT_HOSTNAME} &
		PERF_PIDLIST="$PERF_PIDLIST $!"
	fi
	if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		perf_collect ${DB_OUTPUT_DIR} ${DB_HOSTNAME} &
		PERF_PIDLIST="$PERF_PIDLIST $!"
	fi
	wait $PERF_PIDLIST
fi

if [ ${USE_OPROFILE} -eq 1 ]; then
	oprofile_annotate ${OUTPUT_DIR} &
	if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		oprofile_annotate ${DB_OUTPUT_DIR} ${DB_HOSTNAME} &
	fi
fi

if [ ${USE_OPROFILE} -eq 1 ]; then
	oprofile_stop
	if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
		oprofile_stop ${DB_HOSTNAME}
	fi
fi

# Post processing of Database Statistics
if [ -n ${DB_HOSTNAME} ] && [ ! "x${DB_HOSTNAME}" = "xlocalhost" ]; then
	# If a database system is specified, rsync all the logs back to the driver
	# system after creating the sar csv data.
	post_process_sar $DRIVER_OUTPUT_DIR &
	SAR_PIDLIST="$!"
	if [ ! "x${CLIENT_HOSTNAME}" = "xlocalhost" ]; then
		HOSTNAME=$($SSH $CLIENT_HOSTNAME hostname)
		post_process_sar $CLIENT_OUTPUT_DIR $CLIENT_HOSTNAME
		rsync -a -e "$SSH" --delete \
				"${CLIENT_HOSTNAME}:${CLIENT_OUTPUT_DIR}/" \
				${CLIENT_OUTPUT_DIR}/ &
		SAR_PIDLIST="$SAR_PIDLIST $!"
	fi

	# There will only be a sar output file if the database is on a separate
	# system.
	if [ $DBAAS -eq 0 ]; then
		post_process_sar $DB_OUTPUT_DIR $DB_HOSTNAME
		rsync -a -e "$SSH" --delete "${DB_HOSTNAME}:${DB_OUTPUT_DIR}/" \
				${DB_OUTPUT_DIR}/ &
		SAR_PIDLIST="$SAR_PIDLIST $!"
	fi
	wait $SAR_PIDLIST
else
	post_process_sar $DB_OUTPUT_DIR
fi

# Make sure all the drivers have exited, so all the mix files have flushed to
# disk before continuing processing.  Can't count on all of the sleeps to be
# precise...
wait $DRIVER_PIDLIST

cat > $SUMMARY << __EOF__ &
$(find $DRIVER_OUTPUT_DIR -name 'mix*.log' | xargs dbt2-post-process)
__EOF__
cat >${OUTPUT_DIR}/detailed-summary.rst << __EOF__ &
$(find $DRIVER_OUTPUT_DIR -name 'mix*.log' | VERBOSE=1 xargs dbt2-post-process)
__EOF__
wait

echo "Test completed."
echo "Results are in: ${OUTPUT_DIR}"
echo

cat $SUMMARY
