scripts/tests: uplink no longer respects --client.segment-size
this is going to make all the tests slower but it is what it is test-sim-aws.sh is removed because it was moved to storj/gateway repo. Change-Id: I10727e747a4c3740b1c9054ce7d17313b4fa310b
This commit is contained in:
JT Olio
Michal Niewrzal
parent
aeab599d21
commit
3b66ba6f02
@ -17,7 +17,7 @@ setup(){
|
||||
}
|
||||
random_bytes_file "2KiB" "$TEST_FILES_DIR/small-upload-testfile" # create 2kb file of random bytes (inline)
|
||||
random_bytes_file "5MiB" "$TEST_FILES_DIR/big-upload-testfile" # create 5mb file of random bytes (remote)
|
||||
random_bytes_file "12MiB" "$TEST_FILES_DIR/multisegment-upload-testfile" # create 12mb file of random bytes (remote)
|
||||
random_bytes_file "64MiB" "$TEST_FILES_DIR/multisegment-upload-testfile" # create 65mb file of random bytes (remote)
|
||||
|
||||
echo "setup test successfully"
|
||||
}
|
||||
|
||||
@ -1,74 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -ueo pipefail
|
||||
|
||||
#setup tmpdir for testfiles and cleanup
|
||||
TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
|
||||
cleanup(){
|
||||
rm -rf "$TMPDIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
SRC_DIR=$TMPDIR/source
|
||||
DST_DIR=$TMPDIR/dst
|
||||
mkdir -p "$SRC_DIR" "$DST_DIR"
|
||||
|
||||
|
||||
export AWS_CONFIG_FILE=$TMPDIR/.aws/config
|
||||
export AWS_SHARED_CREDENTIALS_FILE=$TMPDIR/.aws/credentials
|
||||
|
||||
aws configure set aws_access_key_id "$GATEWAY_0_ACCESS_KEY"
|
||||
aws configure set aws_secret_access_key "$GATEWAY_0_SECRET_KEY"
|
||||
aws configure set default.region us-east-1
|
||||
|
||||
random_bytes_file () {
|
||||
size=$1
|
||||
output=$2
|
||||
dd if=/dev/urandom of="$output" count=1 bs="$size" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
random_bytes_file 1x1024x1024 "$SRC_DIR/small-upload-testfile" # create 1mb file of random bytes (inline)
|
||||
random_bytes_file 5x1024x1024 "$SRC_DIR/big-upload-testfile" # create 5mb file of random bytes (remote)
|
||||
random_bytes_file 5x1024 "$SRC_DIR/multipart-upload-testfile" # create 5kb file of random bytes (remote)
|
||||
|
||||
echo "Creating Bucket"
|
||||
aws s3 --endpoint="http://$GATEWAY_0_ADDR" mb s3://bucket
|
||||
|
||||
echo "Uploading Files"
|
||||
aws configure set default.s3.multipart_threshold 1TB
|
||||
aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp "$SRC_DIR/small-upload-testfile" s3://bucket/small-testfile
|
||||
aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp "$SRC_DIR/big-upload-testfile" s3://bucket/big-testfile
|
||||
|
||||
# Wait 5 seconds to trigger any error related to one of the different intervals
|
||||
sleep 5
|
||||
|
||||
echo "Uploading Multipart File"
|
||||
aws configure set default.s3.multipart_threshold 4KB
|
||||
aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp "$SRC_DIR/multipart-upload-testfile" s3://bucket/multipart-testfile
|
||||
|
||||
echo "Downloading Files"
|
||||
aws s3 --endpoint="http://$GATEWAY_0_ADDR" ls s3://bucket
|
||||
aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp s3://bucket/small-testfile "$DST_DIR/small-download-testfile"
|
||||
aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp s3://bucket/big-testfile "$DST_DIR/big-download-testfile"
|
||||
aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp s3://bucket/multipart-testfile "$DST_DIR/multipart-download-testfile"
|
||||
aws s3 --endpoint="http://$GATEWAY_0_ADDR" rb s3://bucket --force
|
||||
|
||||
if cmp "$SRC_DIR/small-upload-testfile" "$DST_DIR/small-download-testfile"
|
||||
then
|
||||
echo "small-upload-testfile file matches uploaded file";
|
||||
else
|
||||
echo "small-upload-testfile file does not match uploaded file";
|
||||
fi
|
||||
|
||||
if cmp "$SRC_DIR/big-upload-testfile" "$DST_DIR/big-download-testfile"
|
||||
then
|
||||
echo "big-upload-testfile file matches uploaded file";
|
||||
else
|
||||
echo "big-upload-testfile file does not match uploaded file";
|
||||
fi
|
||||
|
||||
if cmp "$SRC_DIR/multipart-upload-testfile" "$DST_DIR/multipart-download-testfile"
|
||||
then
|
||||
echo "multipart-upload-testfile file matches uploaded file";
|
||||
else
|
||||
echo "multipart-upload-testfile file does not match uploaded file";
|
||||
fi
|
||||
@ -50,9 +50,6 @@ PATH=$RELEASE_DIR/bin:$PATH storj-sim -x --host $STORJ_NETWORK_HOST4 network --p
|
||||
# run upload part of backward compatibility tests from the lastest release branch
|
||||
PATH=$RELEASE_DIR/bin:$PATH storj-sim -x --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-backwards.sh upload
|
||||
|
||||
# set the segment size lower to make test run faster
|
||||
echo client.segment-size: 6 MiB >> `storj-sim network env GATEWAY_0_DIR`/config.yaml
|
||||
|
||||
SATELLITE_CONFIG=$(storj-sim network env SATELLITE_0_DIR)/config.yaml
|
||||
|
||||
# this replaces anywhere that has "/release/" in the config file, which currently just renames the static dir paths
|
||||
|
||||
@ -27,11 +27,7 @@ else
|
||||
storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network --postgres=$STORJ_SIM_POSTGRES setup
|
||||
fi
|
||||
|
||||
# set the segment size lower to make test run faster
|
||||
echo client.segment-size: "6 MiB" >> `storj-sim network env GATEWAY_0_DIR`/config.yaml
|
||||
|
||||
# run aws-cli tests
|
||||
storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-sim-aws.sh
|
||||
storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-uplink.sh
|
||||
storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-uplink-share.sh
|
||||
storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network destroy
|
||||
|
||||
@ -38,8 +38,8 @@ DST_DIR=$TMPDIR/dst
|
||||
|
||||
mkdir -p "$SRC_DIR" "$DST_DIR"
|
||||
|
||||
random_bytes_file "2048" "$SRC_DIR/another-testfile" # create 2kb file of random bytes (inline)
|
||||
random_bytes_file "5120" "$SRC_DIR/testfile" # create 5kb file of random bytes (inline)
|
||||
random_bytes_file "2KiB" "$SRC_DIR/another-testfile" # create 2kb file of random bytes (inline)
|
||||
random_bytes_file "5KiB" "$SRC_DIR/testfile" # create 5kb file of random bytes (remote)
|
||||
|
||||
uplink --access "$GATEWAY_0_ACCESS" mb "sj://$BUCKET_WITHOUT_ACCESS/"
|
||||
uplink --access "$GATEWAY_0_ACCESS" mb "sj://$BUCKET_WITH_ACCESS/"
|
||||
|
||||
@ -34,11 +34,13 @@ compare_files () {
|
||||
fi
|
||||
}
|
||||
|
||||
random_bytes_file "2048" "$SRC_DIR/small-upload-testfile" # create 2kb file of random bytes (inline)
|
||||
random_bytes_file "5242880" "$SRC_DIR/big-upload-testfile" # create 5mb file of random bytes (remote)
|
||||
random_bytes_file "12582912" "$SRC_DIR/multisegment-upload-testfile" # create 2 x 6mb file of random bytes (remote)
|
||||
random_bytes_file "9437184" "$SRC_DIR/diff-size-segments" # create 9mb file of random bytes (remote)
|
||||
random_bytes_file "100KiB" "$SRC_DIR/put-file" # create 100KiB file of random bytes (remote)
|
||||
random_bytes_file "2KiB" "$SRC_DIR/small-upload-testfile" # create 2KiB file of random bytes (inline)
|
||||
random_bytes_file "5MiB" "$SRC_DIR/big-upload-testfile" # create 5MiB file of random bytes (remote)
|
||||
# this is special case where we need to test at least one remote segment and inline segment of exact size 0
|
||||
random_bytes_file "64MiB" "$SRC_DIR/multisegment-upload-testfile" # create 64MiB file of random bytes (1 remote segments + inline)
|
||||
random_bytes_file "68MiB" "$SRC_DIR/diff-size-segments" # create 68MiB file of random bytes (2 remote segments)
|
||||
|
||||
random_bytes_file "100KiB" "$SRC_DIR/put-file" # create 100KiB file of random bytes (remote)
|
||||
|
||||
UPLINK_DEBUG_ADDR=""
|
||||
|
||||
@ -47,10 +49,10 @@ export STORJ_DEBUG_ADDR=$UPLINK_DEBUG_ADDR
|
||||
|
||||
uplink mb "sj://$BUCKET/"
|
||||
|
||||
uplink cp "$SRC_DIR/small-upload-testfile" "sj://$BUCKET/" --progress=false
|
||||
uplink cp "$SRC_DIR/small-upload-testfile" "sj://$BUCKET/" --progress=false
|
||||
uplink cp "$SRC_DIR/big-upload-testfile" "sj://$BUCKET/" --progress=false
|
||||
uplink cp "$SRC_DIR/multisegment-upload-testfile" "sj://$BUCKET/" --progress=false
|
||||
uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/" --progress=false
|
||||
uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/" --progress=false
|
||||
|
||||
cat "$SRC_DIR/put-file" | uplink put "sj://$BUCKET/put-file"
|
||||
|
||||
@ -73,13 +75,13 @@ then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
uplink ls "sj://$BUCKET/non-existing-prefix"
|
||||
uplink ls "sj://$BUCKET/non-existing-prefix"
|
||||
|
||||
uplink cp "sj://$BUCKET/small-upload-testfile" "$DST_DIR" --progress=false
|
||||
uplink cp "sj://$BUCKET/big-upload-testfile" "$DST_DIR" --progress=false
|
||||
uplink cp "sj://$BUCKET/multisegment-upload-testfile" "$DST_DIR" --progress=false
|
||||
uplink cp "sj://$BUCKET/diff-size-segments" "$DST_DIR" --progress=false
|
||||
uplink cp "sj://$BUCKET/put-file" "$DST_DIR" --progress=false
|
||||
uplink cp "sj://$BUCKET/big-upload-testfile" "$DST_DIR" --progress=false
|
||||
uplink cp "sj://$BUCKET/multisegment-upload-testfile" "$DST_DIR" --progress=false
|
||||
uplink cp "sj://$BUCKET/diff-size-segments" "$DST_DIR" --progress=false
|
||||
uplink cp "sj://$BUCKET/put-file" "$DST_DIR" --progress=false
|
||||
uplink cat "sj://$BUCKET/put-file" >> "$DST_DIR/put-file-from-cat"
|
||||
|
||||
uplink rm "sj://$BUCKET/small-upload-testfile"
|
||||
@ -97,4 +99,4 @@ compare_files "$SRC_DIR/big-upload-testfile" "$DST_DIR/big-upload-testf
|
||||
compare_files "$SRC_DIR/multisegment-upload-testfile" "$DST_DIR/multisegment-upload-testfile"
|
||||
compare_files "$SRC_DIR/diff-size-segments" "$DST_DIR/diff-size-segments"
|
||||
compare_files "$SRC_DIR/put-file" "$DST_DIR/put-file"
|
||||
compare_files "$SRC_DIR/put-file" "$DST_DIR/put-file-from-cat"
|
||||
compare_files "$SRC_DIR/put-file" "$DST_DIR/put-file-from-cat"
|
||||
|
||||
@ -29,7 +29,7 @@ if [ ! -d ${main_cfg_dir}/uplink-old-api ]; then
|
||||
access=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_ACCESS)
|
||||
new_access=$(go run $update_access_script_path $(storj-sim --config-dir=$main_cfg_dir network env SATELLITE_0_DIR) $access)
|
||||
old_sat_api_addr="127.0.0.1:30000"
|
||||
uplink import --satellite-addr="$old_sat_api_addr" --config-dir="${main_cfg_dir}/uplink-old-api" "$new_access" --client.segment-size="64.0 KiB"
|
||||
uplink import --satellite-addr="$old_sat_api_addr" --config-dir="${main_cfg_dir}/uplink-old-api" "$new_access"
|
||||
fi
|
||||
|
||||
echo -e "\nConfig directory for uplink:"
|
||||
|
||||
@ -19,9 +19,9 @@ setup(){
|
||||
output=$2
|
||||
head -c $size </dev/urandom > $output
|
||||
}
|
||||
random_bytes_file "2048" "$test_files_dir/small-upload-testfile" # create 2kb file of random bytes (inline)
|
||||
random_bytes_file "5120" "$test_files_dir/big-upload-testfile" # create 5kb file of random bytes (remote)
|
||||
random_bytes_file "131072" "$test_files_dir/multisegment-upload-testfile" # create 128kb file of random bytes (remote)
|
||||
random_bytes_file "2KiB" "$test_files_dir/small-upload-testfile" # create 2kb file of random bytes (inline)
|
||||
random_bytes_file "5KiB" "$test_files_dir/big-upload-testfile" # create 5kb file of random bytes (remote)
|
||||
random_bytes_file "64MiB" "$test_files_dir/multisegment-upload-testfile" # create 64mb file of random bytes (remote + inline)
|
||||
|
||||
echo "setup test successfully"
|
||||
}
|
||||
@ -59,9 +59,9 @@ if [ ! -d ${main_cfg_dir}/uplink ]; then
|
||||
if [[ ${#should_use_access} -gt 0 ]]; then
|
||||
access=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_ACCESS)
|
||||
new_access=$(go run $update_access_script_path $(storj-sim --config-dir=$main_cfg_dir network env SATELLITE_0_DIR) $access)
|
||||
uplink import --config-dir="${main_cfg_dir}/uplink" "${new_access}" --client.segment-size="64.0 KiB"
|
||||
uplink import --config-dir="${main_cfg_dir}/uplink" "${new_access}"
|
||||
else
|
||||
uplink setup --config-dir="${main_cfg_dir}/uplink" --non-interactive --api-key="$api_key" --satellite-addr="$sat_addr" --enc.encryption-key="test" --client.segment-size="64.0 KiB"
|
||||
uplink setup --config-dir="${main_cfg_dir}/uplink" --non-interactive --api-key="$api_key" --satellite-addr="$sat_addr" --enc.encryption-key="test"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user