scripts: Add test Satellite working w/o Redis

Create a storj-sim test that checks that uplinks operations works when
satellite runs and can connect to Redis and when it cannot connect to
simulate a Redis downtime. Also verifies that the satellite can start
despite of Redis being downtime.

This test currently doesn't pass and it will be the one used to verify
the work that has to be done to make sure that the satellite allow the
clients to perform their operations despite of Redis being unavailable.
We require these changes before we deploy any customer face satellite on
a multi-region architecture.

NOTE that this test will be added later on to Jenkins to run this test
every time that we apply changes and at that time we'll see if it has to
be adjusted for being able to run on Jenkins because as it's now it may
not work because the scripts start and stop a Redis docker container.

Change-Id: I22acb22f0ca594583e36b45c88f8c03bac73b329

scripts/test-sim-redis-up-and-down-dev.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+# shellcheck source=/postgres-dev.sh
+source "${SCRIPTDIR}/postgres-dev.sh"
+
+"${SCRIPTDIR}/test-sim-redis-up-and-down.sh"

scripts/test-sim-redis-up-and-down.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+set +x
+
+# Required environment variables
+if [ -z "${STORJ_SIM_POSTGRES}" ]; then
+	echo "STORJ_SIM_POSTGRES environment variable must be set to a non-empty string"
+	exit 1
+fi
+
+# constants
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+readonly SCRIPT_DIR
+REDIS_CONTAINER_NAME=storj_sim_redis
+readonly REDIS_CONTAINER_NAME
+TMP_DIR=$(mktemp -d -t tmp.XXXXXXXXXX)
+readonly TMP_DIR
+
+# setup tmpdir for testfiles and cleanup
+cleanup() {
+	trap - EXIT
+
+	rm -rf "${TMP_DIR}"
+	docker container rm -f "${REDIS_CONTAINER_NAME}" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "install sim"
+make -C "$SCRIPT_DIR"/.. install-sim
+
+echo "overriding default max segment size to 6MiB"
+GOBIN="${TMP_DIR}" go install -v -ldflags "-X 'storj.io/uplink.maxSegmentSize=6MiB'" storj.io/storj/cmd/uplink
+
+# use modified version of uplink
+export PATH="${TMP_DIR}:${PATH}"
+export STORJ_NETWORK_DIR="${TMP_DIR}"
+
+STORJ_NETWORK_HOST4=${STORJ_NETWORK_HOST4:-127.0.0.1}
+
+redis_run() {
+	local retries=10
+
+	docker container run -d -p 6379:6379 --name "${REDIS_CONTAINER_NAME}" redis:5.0-alpine
+	until docker container exec "${REDIS_CONTAINER_NAME}" redis-cli ping >/dev/null 2>&1 ||
+		[ ${retries} -eq 0 ]; do
+		echo "waiting for Redis server to be ready, $((retries--)) remaining attemps..."
+		sleep 1
+	done
+
+	if [ ${retries} -eq 0 ]; then
+		echo "aborting, Redis server is not ready after several retrials"
+		exit 1
+	fi
+}
+
+redis_stop() {
+	docker container stop "${REDIS_CONTAINER_NAME}"
+}
+
+# setup the network
+storj-sim --failfast -x --satellites 1 --host "${STORJ_NETWORK_HOST4}" network \
+	--postgres="${STORJ_SIM_POSTGRES}" --redis="127.0.0.1:6379" setup
+
+# run test that checks that the satellite runs when Redis is up and down
+redis_run
+storj-sim --failfast -x --satellites 1 --host "${STORJ_NETWORK_HOST4}" network \
+	--redis="127.0.0.1:6379" test bash "${SCRIPT_DIR}/test-uplink-redis-up-and-down.sh" "${REDIS_CONTAINER_NAME}"
+
+# run test that checks that the satellite runs despite of not being able to connect to Redis
+redis_stop
+storj-sim --failfast -x --satellites 1 --host "${STORJ_NETWORK_HOST4}" network \
+	--redis="127.0.0.1:6379" test bash "${SCRIPT_DIR}/test-uplink.sh"

scripts/test-uplink-redis-up-and-down.sh

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+set -ueo pipefail
+
+redis_container_name="${1-}"
+
+# Required positional arguments
+if [ -z "${redis_container_name}" ]; then
+	echo "redis container name is required as a first positional script argument"
+	exit 1
+fi
+
+# constants
+BUCKET="bucket-123"
+readonly BUCKET
+UPLINK_DEBUG_ADDR=""
+readonly UPLINK_DEBUG_ADDR
+
+export STORJ_ACCESS="${GATEWAY_0_ACCESS}"
+export STORJ_DEBUG_ADDR="${UPLINK_DEBUG_ADDR}"
+
+# Vars
+temp_dirs=() # used to track all the created temporary directories
+
+cleanup() {
+	trap - EXIT
+
+	rm -rf "${temp_dirs[@]}"
+	echo "cleaned up test successfully"
+}
+trap cleanup EXIT
+
+random_bytes_file() {
+	size="${1}"
+	output="${2}"
+	head -c "${size}" </dev/urandom >"${output}"
+}
+
+compare_files() {
+	name=$(basename "${2}")
+	if cmp "${1}" "${2}"; then
+		echo "${name} matches uploaded file"
+	else
+		echo "${name} does not match uploaded file"
+		exit 1
+	fi
+}
+
+redis_start() {
+	docker container start "${redis_container_name}"
+}
+
+redis_stop() {
+	docker container stop "${redis_container_name}"
+}
+
+uplink_test() {
+	local temp_dir
+	temp_dir=$(mktemp -d -t tmp.XXXXXXXXXX)
+	temp_dirs+=("${temp_dir}")
+
+	local src_dir="${temp_dir}/source"
+	local dst_dir="${temp_dir}/dst"
+	mkdir -p "${src_dir}" "${dst_dir}"
+
+	local uplink_dir="${temp_dir}/uplink"
+
+	random_bytes_file "2KiB" "${src_dir}/small-upload-testfile" # create 2KiB file of random bytes (inline)
+	random_bytes_file "5MiB" "${src_dir}/big-upload-testfile"   # create 5MiB file of random bytes (remote)
+	# this is special case where we need to test at least one remote segment and inline segment of exact size 0
+	random_bytes_file "12MiB" "${src_dir}/multisegment-upload-testfile" # create 12MiB file of random bytes (1 remote segments + inline)
+	random_bytes_file "13MiB" "${src_dir}/diff-size-segments"           # create 13MiB file of random bytes (2 remote segments)
+
+	random_bytes_file "100KiB" "${src_dir}/put-file" # create 100KiB file of random bytes (remote)
+
+	uplink mb "sj://$BUCKET/"
+	uplink cp "${src_dir}/small-upload-testfile" "sj://$BUCKET/" --progress=false
+	uplink cp "${src_dir}/big-upload-testfile" "sj://$BUCKET/" --progress=false
+	uplink cp "${src_dir}/multisegment-upload-testfile" "sj://$BUCKET/" --progress=false
+	uplink cp "${src_dir}/diff-size-segments" "sj://$BUCKET/" --progress=false
+
+	uplink <"${src_dir}/put-file" put "sj://$BUCKET/put-file"
+
+	uplink --config-dir "${uplink_dir}" import named-access "${STORJ_ACCESS}"
+
+	local files
+	files=$(STORJ_ACCESS='' uplink --config-dir "${uplink_dir}" --access named-access \
+		ls "sj://${BUCKET}" | tee "${temp_dir}/list" | wc -l)
+	local expected_files="5"
+	if [ "${files}" == "${expected_files}" ]; then
+		echo "listing returns ${files} files"
+	else
+		echo "listing returns ${files} files but want ${expected_files}"
+		exit 1
+	fi
+
+	local size_check
+	size_check=$(awk <"${temp_dir}/list" '{if($4 == "0") print "invalid size";}')
+	if [ "${size_check}" != "" ]; then
+		echo "listing returns invalid size for one of the objects:"
+		cat "${temp_dir}/list"
+		exit 1
+	fi
+
+	uplink ls "sj://$BUCKET/non-existing-prefix"
+
+	uplink cp "sj://$BUCKET/small-upload-testfile" "${dst_dir}" --progress=false
+	uplink cp "sj://$BUCKET/big-upload-testfile" "${dst_dir}" --progress=false
+	uplink cp "sj://$BUCKET/multisegment-upload-testfile" "${dst_dir}" --progress=false
+	uplink cp "sj://$BUCKET/diff-size-segments" "${dst_dir}" --progress=false
+	uplink cp "sj://$BUCKET/put-file" "${dst_dir}" --progress=false
+	uplink cat "sj://$BUCKET/put-file" >>"${dst_dir}/put-file-from-cat"
+
+	uplink rm "sj://$BUCKET/small-upload-testfile"
+	uplink rm "sj://$BUCKET/big-upload-testfile"
+	uplink rm "sj://$BUCKET/multisegment-upload-testfile"
+	uplink rm "sj://$BUCKET/diff-size-segments"
+	uplink rm "sj://$BUCKET/put-file"
+
+	uplink ls "sj://$BUCKET"
+
+	uplink rb "sj://$BUCKET"
+
+	compare_files "${src_dir}/small-upload-testfile" "${dst_dir}/small-upload-testfile"
+	compare_files "${src_dir}/big-upload-testfile" "${dst_dir}/big-upload-testfile"
+	compare_files "${src_dir}/multisegment-upload-testfile" "${dst_dir}/multisegment-upload-testfile"
+	compare_files "${src_dir}/diff-size-segments" "${dst_dir}/diff-size-segments"
+	compare_files "${src_dir}/put-file" "${dst_dir}/put-file"
+	compare_files "${src_dir}/put-file" "${dst_dir}/put-file-from-cat"
+
+	# test deleting non empty bucket with --force flag
+	uplink mb "sj://$BUCKET/"
+
+	for i in $(seq -w 1 16); do
+		uplink cp "${src_dir}/small-upload-testfile" "sj://$BUCKET/small-file-$i" --progress=false
+	done
+
+	uplink rb "sj://$BUCKET" --force
+
+	if [ "$(uplink ls | grep -c "No buckets")" = "0" ]; then
+		echo "uplink didn't remove the entire bucket with the 'force' flag"
+		exit 1
+	fi
+}
+
+# Run the test with Redis container running
+uplink_test
+
+# Run the test with Redis container not running
+redis_stop
+uplink_test