scripts/tests/{backwardcompatibility,integrations}: add test scripts
Change-Id: Ib83cd0f083bab7f560a200fd95e62e5b21e60c27
This commit is contained in:
dlamarmorgan
Storj Robot
parent
0a8115b149
commit
0f4371e84c
@ -30,12 +30,14 @@ BUCKET="${BUCKET:-bucket-123}"
|
||||
PRISTINE_FILES_DIR="$STORJ_NETWORK_DIR/pristine/$BUCKET"
|
||||
DOWNLOAD_FILES_DIR="$STORJ_NETWORK_DIR/download/$BUCKET"
|
||||
|
||||
# override configured access with access where address is node ID + satellite addess
|
||||
STORJ_ACCESS=$(go run "$SCRIPTDIR"/../../update-access.go "$SATELLITE_0_DIR" "$GATEWAY_0_ACCESS")
|
||||
UPLINK_ACCESS="$STORJ_ACCESS"
|
||||
if [[ ! -v UPLINK_ACCESS ]]; then
|
||||
# override configured access with access where address is node ID + satellite addess
|
||||
STORJ_ACCESS=$(go run "$SCRIPTDIR"/../../update-access.go "$SATELLITE_0_DIR" "$GATEWAY_0_ACCESS")
|
||||
UPLINK_ACCESS="$STORJ_ACCESS"
|
||||
|
||||
export STORJ_ACCESS
|
||||
export UPLINK_ACCESS
|
||||
export STORJ_ACCESS
|
||||
export UPLINK_ACCESS
|
||||
fi
|
||||
|
||||
# workaround for issues with automatic accepting monitoring question
|
||||
# with first run we need to accept question y/n about monitoring
|
||||
@ -48,9 +50,9 @@ set -x
|
||||
if [[ "$1" == "upload" ]]; then
|
||||
mkdir -p "$PRISTINE_FILES_DIR" "$DOWNLOAD_FILES_DIR"
|
||||
|
||||
random_bytes_file "2KiB" "$PRISTINE_FILES_DIR/small-upload-testfile" # create 2kb file of random bytes (inline)
|
||||
random_bytes_file "5MiB" "$PRISTINE_FILES_DIR/big-upload-testfile" # create 5mb file of random bytes (remote)
|
||||
random_bytes_file "12MiB" "$PRISTINE_FILES_DIR/multisegment-upload-testfile" # create 12mb file of random bytes (remote)
|
||||
random_bytes_file "2048" "$PRISTINE_FILES_DIR/small-upload-testfile" # create 2kb file of random bytes (inline)
|
||||
random_bytes_file "5242880" "$PRISTINE_FILES_DIR/big-upload-testfile" # create 5mb file of random bytes (remote)
|
||||
random_bytes_file "12582912" "$PRISTINE_FILES_DIR/multisegment-upload-testfile" # create 12mb file of random bytes (remote)
|
||||
|
||||
# sometimes we overwrite files in the same bucket. allow the mb to fail because of an existing
|
||||
# bucket. if it fails for any other reason, the following cp will get it anyway.
|
||||
|
||||
138
scripts/tests/backwardcompatibility/test-storj-up-backwards.sh
Executable file
138
scripts/tests/backwardcompatibility/test-storj-up-backwards.sh
Executable file
@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env bash
|
||||
set -xueo pipefail
|
||||
|
||||
GOOS=$(go env GOOS)
|
||||
if { [ "$GOOS" != "linux" ]; } && { [ -z "${STORJ_CC_TARGET:-}" ] || [ -z "${STORJ_CXX_TARGET:-}" ]; }
|
||||
then
|
||||
echo "Both STORJ_CC_TARGET and STORJ_CXX_TARGET must be set when building for non-linux platforms. Exiting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DB=${1:-}
|
||||
|
||||
case "$DB" in
|
||||
'postgres') echo "running test with postgres DB"
|
||||
;;
|
||||
'cockroach') echo "running test with cockroach DB"
|
||||
;;
|
||||
*) echo "invalid DB specified, defaulting to cockroach"
|
||||
DB="cockroach"
|
||||
;;
|
||||
esac
|
||||
|
||||
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
export STORJ_NETWORK_DIR=$(mktemp -d -t tmp.XXXXXXXXXX)
|
||||
|
||||
cleanup(){
|
||||
if [ -f "$STORJ_NETWORK_DIR"/docker-compose.yaml ]
|
||||
then
|
||||
docker compose -f "$STORJ_NETWORK_DIR"/docker-compose.yaml down
|
||||
fi
|
||||
rm -rf "$STORJ_NETWORK_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
#### build release binaries ####
|
||||
RELEASE_BIN="$STORJ_NETWORK_DIR/bin/release"
|
||||
# replace this with a standard go install once go allows install cross-compiled binaries when GOBIN is set
|
||||
# https://github.com/golang/go/issues/57485
|
||||
git worktree add -f "$STORJ_NETWORK_DIR"/branch HEAD
|
||||
latestReleaseCommit="$(git rev-list --exclude='*rc*' --tags --max-count=1)"
|
||||
latestReleaseTag=$(git describe --tags "$latestReleaseCommit")
|
||||
echo "Checking out latest release tag: $latestReleaseTag"
|
||||
git worktree add -f "$STORJ_NETWORK_DIR"/release "$latestReleaseCommit"
|
||||
pushd "$STORJ_NETWORK_DIR"/release
|
||||
case $GOOS in
|
||||
linux) GOOS=linux GOARCH=$(go env GOARCH) go build -tags noquic -o "$RELEASE_BIN"/storagenode -v storj.io/storj/cmd/storagenode ;;
|
||||
*) CGO_ENABLED=1 GOOS=linux GOARCH=$(go env GOARCH) CC=$STORJ_CC_TARGET CXX=$STORJ_CXX_TARGET go build -tags noquic -o "$RELEASE_BIN"/storagenode -v storj.io/storj/cmd/storagenode ;;
|
||||
esac
|
||||
GOOS=linux GOARCH=$(go env GOARCH) go build -tags noquic -o "$RELEASE_BIN"/satellite -v storj.io/storj/cmd/satellite
|
||||
GOOS=$GOOS GOARCH=$(go env GOARCH) go build -tags noquic -o "$RELEASE_BIN"/uplink -v -ldflags "-X 'storj.io/uplink.maxSegmentSize=6MiB'" storj.io/storj/cmd/uplink
|
||||
popd
|
||||
|
||||
go install storj.io/storj-up@latest
|
||||
|
||||
#### setup the release network ####
|
||||
cd "$STORJ_NETWORK_DIR"
|
||||
if [ "$DB" == "cockroach" ]
|
||||
then
|
||||
storj-up init minimal,db
|
||||
else
|
||||
storj-up init minimal,redis
|
||||
storj-up add postgres
|
||||
storj-up port remove postgres 5432
|
||||
storj-up port add postgres 6543
|
||||
storj-up env set postgres PGPORT=6543
|
||||
storj-up env set satellite-api STORJ_DATABASE=postgres://postgres@postgres:6543/master?sslmode=disable
|
||||
storj-up env set satellite-api STORJ_METAINFO_DATABASE_URL=postgres://postgres@postgres:6543/master?sslmode=disable
|
||||
fi
|
||||
storj-up env set satellite-api STORJ_DATABASE_OPTIONS_MIGRATION_UNSAFE="full"
|
||||
storj-up local satellite-api,storagenode -d "$RELEASE_BIN"
|
||||
# persist the 5 nodes that will be restarted with branch binaries
|
||||
mkdir -p {storagenode1/storj,storagenode2/storj,storagenode3/storj,storagenode4/storj,storagenode5/storj}
|
||||
storj-up persist storagenode1,storagenode2,storagenode3,storagenode4,storagenode5
|
||||
|
||||
# start the services
|
||||
docker compose up -d
|
||||
if [ "$DB" == "cockroach" ]
|
||||
then
|
||||
storj-up health -d 90
|
||||
else
|
||||
storj-up health -d 90 -u postgres -p 6543
|
||||
fi
|
||||
eval $(storj-up credentials -e)
|
||||
|
||||
#### release tests ####
|
||||
# upload using everything release
|
||||
PATH="$RELEASE_BIN":"$PATH" "$SCRIPTDIR""/test-backwards.sh" -b release-network-release-uplink upload
|
||||
|
||||
# check that it worked with everything release
|
||||
PATH="$RELEASE_BIN":"$PATH" "$SCRIPTDIR""/test-backwards.sh" -b release-network-release-uplink download
|
||||
|
||||
#### build branch binaries ####
|
||||
BRANCH_BIN="$STORJ_NETWORK_DIR/bin/branch"
|
||||
cd "$SCRIPTDIR"
|
||||
case $GOOS in
|
||||
linux) GOBIN=$BRANCH_BIN GOOS=linux GOARCH=$(go env GOARCH) go install -race -v storj.io/storj/cmd/storagenode >/dev/null 2>&1 ;;
|
||||
*) CGO_ENABLED=1 GOBIN=$BRANCH_BIN GOOS=linux GOARCH=$(go env GOARCH) CC=$STORJ_CC_TARGET CXX=$STORJ_CXX_TARGET go install -race -v storj.io/storj/cmd/storagenode >/dev/null 2>&1 ;;
|
||||
esac
|
||||
GOBIN=$BRANCH_BIN GOOS=linux GOARCH=$(go env GOARCH) go install -race -v storj.io/storj/cmd/satellite >/dev/null 2>&1
|
||||
GOBIN=$BRANCH_BIN GOOS=$GOOS GOARCH=$(go env GOARCH) go install -race -v -ldflags "-X 'storj.io/uplink.maxSegmentSize=6MiB'" storj.io/storj/cmd/uplink >/dev/null 2>&1
|
||||
|
||||
#### setup the branch network ####
|
||||
cd "$STORJ_NETWORK_DIR"
|
||||
# Kill 1 node to run with 9 nodes and exercise more code paths with one node being offline.
|
||||
docker compose rm -sv storagenode10
|
||||
# update satellite and 5 storage nodes to use branch binaries.
|
||||
storj-up local satellite-api,storagenode1,storagenode2,storagenode3,storagenode4,storagenode5 -d $BRANCH_BIN
|
||||
# start the branch services
|
||||
docker compose up -d satellite-api storagenode1 storagenode2 storagenode3 storagenode4 storagenode5
|
||||
# wait for branch network to be ready
|
||||
docker compose exec -T storagenode1 storj-up util wait-for-satellite satellite-api:7777
|
||||
docker compose exec -T satellite-api storj-up util wait-for-port storagenode1:30001
|
||||
docker compose exec -T satellite-api storj-up util wait-for-port storagenode2:30011
|
||||
docker compose exec -T satellite-api storj-up util wait-for-port storagenode3:30021
|
||||
docker compose exec -T satellite-api storj-up util wait-for-port storagenode4:30031
|
||||
docker compose exec -T satellite-api storj-up util wait-for-port storagenode5:30041
|
||||
|
||||
# todo: replace with a proper health check
|
||||
sleep 60
|
||||
|
||||
#### Branch tests ####
|
||||
# check that branch uplink + branch network can read fully release data
|
||||
PATH="$BRANCH_BIN":"$PATH" "$SCRIPTDIR""/test-backwards.sh" -b release-network-release-uplink download
|
||||
|
||||
# check that branch uplink + branch network can upload
|
||||
PATH="$BRANCH_BIN":"$PATH" "$SCRIPTDIR""/test-backwards.sh" -b branch-network-branch-uplink upload
|
||||
|
||||
# check that release uplink + branch network can read fully release data
|
||||
PATH="$RELEASE_BIN":"$PATH" "$SCRIPTDIR""/test-backwards.sh" -b release-network-release-uplink download
|
||||
|
||||
# check that release uplink + branch network can read fully branch data
|
||||
PATH="$RELEASE_BIN":"$PATH" "$SCRIPTDIR""/test-backwards.sh" -b branch-network-branch-uplink download
|
||||
|
||||
# check that release uplink + branch network can upload
|
||||
PATH="$RELEASE_BIN":"$PATH" "$SCRIPTDIR""/test-backwards.sh" -b branch-network-release-uplink upload
|
||||
|
||||
# check that release uplink + branch network can read mixed data
|
||||
PATH="$RELEASE_BIN":"$PATH" "$SCRIPTDIR""/test-backwards.sh" -b branch-network-release-uplink download
|
||||
91
scripts/tests/integration/test-storj-up.sh
Executable file
91
scripts/tests/integration/test-storj-up.sh
Executable file
@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
set -xueo pipefail
|
||||
|
||||
DB=${1:-}
|
||||
|
||||
case "$DB" in
|
||||
'postgres') echo "running test with postgres DB"
|
||||
;;
|
||||
'cockroach') echo "running test with cockroach DB"
|
||||
;;
|
||||
*) echo "invalid DB specified, defaulting to cockroach"
|
||||
DB="cockroach"
|
||||
;;
|
||||
esac
|
||||
|
||||
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$SCRIPTDIR"
|
||||
|
||||
# setup tmpdir for test files and cleanup
|
||||
TMP=$(mktemp -d -t tmp.XXXXXXXXXX)
|
||||
TMP_BIN=$TMP/bin
|
||||
cleanup(){
|
||||
if [ -f "$TMP"/docker-compose.yaml ]
|
||||
then
|
||||
docker compose -f "$TMP"/docker-compose.yaml down
|
||||
fi
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# replace this with a standard go install once go allows install cross-compiled binaries when GOBIN is set
|
||||
# https://github.com/golang/go/issues/57485
|
||||
git worktree add -f "$TMP"/branch HEAD
|
||||
pushd "$TMP"/branch
|
||||
GOOS=linux GOARCH=$(go env GOARCH) go build -tags noquic -o "$TMP_BIN"/storagenode -v storj.io/storj/cmd/storagenode
|
||||
GOOS=linux GOARCH=$(go env GOARCH) go build -tags noquic -o "$TMP_BIN"/satellite -v storj.io/storj/cmd/satellite
|
||||
GOOS=linux GOARCH=$(go env GOARCH) go build -tags noquic -o "$TMP_BIN"/uplink -v -ldflags "-X 'storj.io/uplink.maxSegmentSize=6MiB'" storj.io/storj/cmd/uplink
|
||||
popd
|
||||
|
||||
go install storj.io/storj-up@latest
|
||||
|
||||
cd "$TMP"
|
||||
if [ "$DB" == "cockroach" ]
|
||||
then
|
||||
storj-up init minimal,db
|
||||
else
|
||||
storj-up init minimal,redis
|
||||
storj-up add postgres
|
||||
storj-up port remove postgres 5432
|
||||
storj-up port add postgres 6543
|
||||
storj-up env set postgres PGPORT=6543
|
||||
storj-up env set satellite-api STORJ_DATABASE=postgres://postgres@postgres:6543/master?sslmode=disable
|
||||
storj-up env set satellite-api STORJ_METAINFO_DATABASE_URL=postgres://postgres@postgres:6543/master?sslmode=disable
|
||||
fi
|
||||
storj-up env set satellite-api STORJ_DATABASE_OPTIONS_MIGRATION_UNSAFE="full"
|
||||
storj-up local satellite-api,storagenode -d "$TMP_BIN"
|
||||
|
||||
# start the services
|
||||
docker compose up -d
|
||||
if [ "$DB" == "cockroach" ]
|
||||
then
|
||||
storj-up health -d 90
|
||||
else
|
||||
storj-up health -d 90 -u postgres -p 6543
|
||||
fi
|
||||
eval $(storj-up credentials -e)
|
||||
#todo: remove these two lines when storj-sim is gone from all integration tests
|
||||
export GATEWAY_0_ACCESS=$UPLINK_ACCESS
|
||||
export SATELLITE_0_DIR=$TMP
|
||||
|
||||
# run tests
|
||||
PATH="$TMP_BIN":"$PATH" "$SCRIPTDIR"/test-uplink.sh
|
||||
PATH="$TMP_BIN":"$PATH" "$SCRIPTDIR"/test-uplink-share.sh
|
||||
# todo: this doesn't really test anything. we should probably make a separate test for it
|
||||
if [ "$DB" == "cockroach" ]
|
||||
then
|
||||
PATH="$TMP_BIN":"$PATH" STORJ_DATABASE=cockroach://root@localhost:26257/master?sslmode=disable "$SCRIPTDIR"/test-billing.sh
|
||||
else
|
||||
PATH="$TMP_BIN":"$PATH" STORJ_DATABASE=postgres://postgres@localhost:6543/master?sslmode=disable "$SCRIPTDIR"/test-billing.sh
|
||||
fi
|
||||
PATH="$TMP_BIN":"$PATH" "$SCRIPTDIR"/test-uplink-rs-upload.sh
|
||||
|
||||
# change RS values and try download
|
||||
storj-up env set satellite-api STORJ_METAINFO_RS_ERASURE_SHARE_SIZE=256
|
||||
storj-up env set satellite-api STORJ_METAINFO_RS_MIN=2
|
||||
storj-up env set satellite-api STORJ_METAINFO_RS_REPAIR=3
|
||||
storj-up env set satellite-api STORJ_METAINFO_RS_SUCCESS=6
|
||||
storj-up env set satellite-api STORJ_METAINFO_RS_TOTAL=8
|
||||
docker compose up -d
|
||||
docker compose exec -T storagenode1 storj-up util wait-for-satellite satellite-api:7777
|
||||
PATH="$TMP_BIN":"$PATH" "$SCRIPTDIR"/test-uplink-rs-download.sh
|
||||
0
scripts/tests/integration/test-uplink-share.sh
Normal file → Executable file
0
scripts/tests/integration/test-uplink-share.sh
Normal file → Executable file
Loading…
Reference in New Issue
Block a user