From 3b66ba6f02008a0b177b4a56dd041b6e87affcbc Mon Sep 17 00:00:00 2001 From: JT Olio Date: Thu, 19 Mar 2020 15:37:00 -0600 Subject: [PATCH] scripts/tests: uplink no longer respects --client.segment-size this is going to make all the tests slower but it is what it is test-sim-aws.sh is removed because it was moved to storj/gateway repo. Change-Id: I10727e747a4c3740b1c9054ce7d17313b4fa310b --- scripts/test-backwards.sh | 2 +- scripts/test-sim-aws.sh | 74 ------------------- scripts/test-sim-backwards.sh | 3 - scripts/test-sim.sh | 4 - scripts/test-uplink-share.sh | 4 +- scripts/test-uplink.sh | 28 +++---- .../rollingupgrade/test-rolling-upgrade.sh | 2 +- scripts/tests/testversions/test-versions.sh | 10 +-- 8 files changed, 24 insertions(+), 103 deletions(-) delete mode 100755 scripts/test-sim-aws.sh diff --git a/scripts/test-backwards.sh b/scripts/test-backwards.sh index 4be1886c6..fe396ec73 100755 --- a/scripts/test-backwards.sh +++ b/scripts/test-backwards.sh @@ -17,7 +17,7 @@ setup(){ } random_bytes_file "2KiB" "$TEST_FILES_DIR/small-upload-testfile" # create 2kb file of random bytes (inline) random_bytes_file "5MiB" "$TEST_FILES_DIR/big-upload-testfile" # create 5mb file of random bytes (remote) - random_bytes_file "12MiB" "$TEST_FILES_DIR/multisegment-upload-testfile" # create 12mb file of random bytes (remote) + random_bytes_file "64MiB" "$TEST_FILES_DIR/multisegment-upload-testfile" # create 65mb file of random bytes (remote) echo "setup test successfully" } diff --git a/scripts/test-sim-aws.sh b/scripts/test-sim-aws.sh deleted file mode 100755 index 97c02a017..000000000 --- a/scripts/test-sim-aws.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -set -ueo pipefail - -#setup tmpdir for testfiles and cleanup -TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX) -cleanup(){ - rm -rf "$TMPDIR" -} -trap cleanup EXIT - -SRC_DIR=$TMPDIR/source -DST_DIR=$TMPDIR/dst -mkdir -p "$SRC_DIR" "$DST_DIR" - - -export AWS_CONFIG_FILE=$TMPDIR/.aws/config -export AWS_SHARED_CREDENTIALS_FILE=$TMPDIR/.aws/credentials - -aws configure set aws_access_key_id "$GATEWAY_0_ACCESS_KEY" -aws configure set aws_secret_access_key "$GATEWAY_0_SECRET_KEY" -aws configure set default.region us-east-1 - -random_bytes_file () { - size=$1 - output=$2 - dd if=/dev/urandom of="$output" count=1 bs="$size" >/dev/null 2>&1 -} - -random_bytes_file 1x1024x1024 "$SRC_DIR/small-upload-testfile" # create 1mb file of random bytes (inline) -random_bytes_file 5x1024x1024 "$SRC_DIR/big-upload-testfile" # create 5mb file of random bytes (remote) -random_bytes_file 5x1024 "$SRC_DIR/multipart-upload-testfile" # create 5kb file of random bytes (remote) - -echo "Creating Bucket" -aws s3 --endpoint="http://$GATEWAY_0_ADDR" mb s3://bucket - -echo "Uploading Files" -aws configure set default.s3.multipart_threshold 1TB -aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp "$SRC_DIR/small-upload-testfile" s3://bucket/small-testfile -aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp "$SRC_DIR/big-upload-testfile" s3://bucket/big-testfile - -# Wait 5 seconds to trigger any error related to one of the different intervals -sleep 5 - -echo "Uploading Multipart File" -aws configure set default.s3.multipart_threshold 4KB -aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp "$SRC_DIR/multipart-upload-testfile" s3://bucket/multipart-testfile - -echo "Downloading Files" -aws s3 --endpoint="http://$GATEWAY_0_ADDR" ls s3://bucket -aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp s3://bucket/small-testfile "$DST_DIR/small-download-testfile" -aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp s3://bucket/big-testfile "$DST_DIR/big-download-testfile" -aws s3 --endpoint="http://$GATEWAY_0_ADDR" cp s3://bucket/multipart-testfile "$DST_DIR/multipart-download-testfile" -aws s3 --endpoint="http://$GATEWAY_0_ADDR" rb s3://bucket --force - -if cmp "$SRC_DIR/small-upload-testfile" "$DST_DIR/small-download-testfile" -then - echo "small-upload-testfile file matches uploaded file"; -else - echo "small-upload-testfile file does not match uploaded file"; -fi - -if cmp "$SRC_DIR/big-upload-testfile" "$DST_DIR/big-download-testfile" -then - echo "big-upload-testfile file matches uploaded file"; -else - echo "big-upload-testfile file does not match uploaded file"; -fi - -if cmp "$SRC_DIR/multipart-upload-testfile" "$DST_DIR/multipart-download-testfile" -then - echo "multipart-upload-testfile file matches uploaded file"; -else - echo "multipart-upload-testfile file does not match uploaded file"; -fi diff --git a/scripts/test-sim-backwards.sh b/scripts/test-sim-backwards.sh index b68cbbf66..290403276 100755 --- a/scripts/test-sim-backwards.sh +++ b/scripts/test-sim-backwards.sh @@ -50,9 +50,6 @@ PATH=$RELEASE_DIR/bin:$PATH storj-sim -x --host $STORJ_NETWORK_HOST4 network --p # run upload part of backward compatibility tests from the lastest release branch PATH=$RELEASE_DIR/bin:$PATH storj-sim -x --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-backwards.sh upload -# set the segment size lower to make test run faster -echo client.segment-size: 6 MiB >> `storj-sim network env GATEWAY_0_DIR`/config.yaml - SATELLITE_CONFIG=$(storj-sim network env SATELLITE_0_DIR)/config.yaml # this replaces anywhere that has "/release/" in the config file, which currently just renames the static dir paths diff --git a/scripts/test-sim.sh b/scripts/test-sim.sh index 0a1a8d125..e50cd04ca 100755 --- a/scripts/test-sim.sh +++ b/scripts/test-sim.sh @@ -27,11 +27,7 @@ else storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network --postgres=$STORJ_SIM_POSTGRES setup fi -# set the segment size lower to make test run faster -echo client.segment-size: "6 MiB" >> `storj-sim network env GATEWAY_0_DIR`/config.yaml - # run aws-cli tests -storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-sim-aws.sh storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-uplink.sh storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-uplink-share.sh storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network destroy diff --git a/scripts/test-uplink-share.sh b/scripts/test-uplink-share.sh index e6f3fe833..81434122d 100644 --- a/scripts/test-uplink-share.sh +++ b/scripts/test-uplink-share.sh @@ -38,8 +38,8 @@ DST_DIR=$TMPDIR/dst mkdir -p "$SRC_DIR" "$DST_DIR" -random_bytes_file "2048" "$SRC_DIR/another-testfile" # create 2kb file of random bytes (inline) -random_bytes_file "5120" "$SRC_DIR/testfile" # create 5kb file of random bytes (inline) +random_bytes_file "2KiB" "$SRC_DIR/another-testfile" # create 2kb file of random bytes (inline) +random_bytes_file "5KiB" "$SRC_DIR/testfile" # create 5kb file of random bytes (remote) uplink --access "$GATEWAY_0_ACCESS" mb "sj://$BUCKET_WITHOUT_ACCESS/" uplink --access "$GATEWAY_0_ACCESS" mb "sj://$BUCKET_WITH_ACCESS/" diff --git a/scripts/test-uplink.sh b/scripts/test-uplink.sh index 1996d3cca..2fb42ccf5 100755 --- a/scripts/test-uplink.sh +++ b/scripts/test-uplink.sh @@ -34,11 +34,13 @@ compare_files () { fi } -random_bytes_file "2048" "$SRC_DIR/small-upload-testfile" # create 2kb file of random bytes (inline) -random_bytes_file "5242880" "$SRC_DIR/big-upload-testfile" # create 5mb file of random bytes (remote) -random_bytes_file "12582912" "$SRC_DIR/multisegment-upload-testfile" # create 2 x 6mb file of random bytes (remote) -random_bytes_file "9437184" "$SRC_DIR/diff-size-segments" # create 9mb file of random bytes (remote) -random_bytes_file "100KiB" "$SRC_DIR/put-file" # create 100KiB file of random bytes (remote) +random_bytes_file "2KiB" "$SRC_DIR/small-upload-testfile" # create 2KiB file of random bytes (inline) +random_bytes_file "5MiB" "$SRC_DIR/big-upload-testfile" # create 5MiB file of random bytes (remote) +# this is special case where we need to test at least one remote segment and inline segment of exact size 0 +random_bytes_file "64MiB" "$SRC_DIR/multisegment-upload-testfile" # create 64MiB file of random bytes (1 remote segments + inline) +random_bytes_file "68MiB" "$SRC_DIR/diff-size-segments" # create 68MiB file of random bytes (2 remote segments) + +random_bytes_file "100KiB" "$SRC_DIR/put-file" # create 100KiB file of random bytes (remote) UPLINK_DEBUG_ADDR="" @@ -47,10 +49,10 @@ export STORJ_DEBUG_ADDR=$UPLINK_DEBUG_ADDR uplink mb "sj://$BUCKET/" -uplink cp "$SRC_DIR/small-upload-testfile" "sj://$BUCKET/" --progress=false +uplink cp "$SRC_DIR/small-upload-testfile" "sj://$BUCKET/" --progress=false uplink cp "$SRC_DIR/big-upload-testfile" "sj://$BUCKET/" --progress=false uplink cp "$SRC_DIR/multisegment-upload-testfile" "sj://$BUCKET/" --progress=false -uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/" --progress=false +uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/" --progress=false cat "$SRC_DIR/put-file" | uplink put "sj://$BUCKET/put-file" @@ -73,13 +75,13 @@ then exit 1 fi -uplink ls "sj://$BUCKET/non-existing-prefix" +uplink ls "sj://$BUCKET/non-existing-prefix" uplink cp "sj://$BUCKET/small-upload-testfile" "$DST_DIR" --progress=false -uplink cp "sj://$BUCKET/big-upload-testfile" "$DST_DIR" --progress=false -uplink cp "sj://$BUCKET/multisegment-upload-testfile" "$DST_DIR" --progress=false -uplink cp "sj://$BUCKET/diff-size-segments" "$DST_DIR" --progress=false -uplink cp "sj://$BUCKET/put-file" "$DST_DIR" --progress=false +uplink cp "sj://$BUCKET/big-upload-testfile" "$DST_DIR" --progress=false +uplink cp "sj://$BUCKET/multisegment-upload-testfile" "$DST_DIR" --progress=false +uplink cp "sj://$BUCKET/diff-size-segments" "$DST_DIR" --progress=false +uplink cp "sj://$BUCKET/put-file" "$DST_DIR" --progress=false uplink cat "sj://$BUCKET/put-file" >> "$DST_DIR/put-file-from-cat" uplink rm "sj://$BUCKET/small-upload-testfile" @@ -97,4 +99,4 @@ compare_files "$SRC_DIR/big-upload-testfile" "$DST_DIR/big-upload-testf compare_files "$SRC_DIR/multisegment-upload-testfile" "$DST_DIR/multisegment-upload-testfile" compare_files "$SRC_DIR/diff-size-segments" "$DST_DIR/diff-size-segments" compare_files "$SRC_DIR/put-file" "$DST_DIR/put-file" -compare_files "$SRC_DIR/put-file" "$DST_DIR/put-file-from-cat" \ No newline at end of file +compare_files "$SRC_DIR/put-file" "$DST_DIR/put-file-from-cat" diff --git a/scripts/tests/rollingupgrade/test-rolling-upgrade.sh b/scripts/tests/rollingupgrade/test-rolling-upgrade.sh index 0a6190f20..b5f0117fc 100755 --- a/scripts/tests/rollingupgrade/test-rolling-upgrade.sh +++ b/scripts/tests/rollingupgrade/test-rolling-upgrade.sh @@ -29,7 +29,7 @@ if [ ! -d ${main_cfg_dir}/uplink-old-api ]; then access=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_ACCESS) new_access=$(go run $update_access_script_path $(storj-sim --config-dir=$main_cfg_dir network env SATELLITE_0_DIR) $access) old_sat_api_addr="127.0.0.1:30000" - uplink import --satellite-addr="$old_sat_api_addr" --config-dir="${main_cfg_dir}/uplink-old-api" "$new_access" --client.segment-size="64.0 KiB" + uplink import --satellite-addr="$old_sat_api_addr" --config-dir="${main_cfg_dir}/uplink-old-api" "$new_access" fi echo -e "\nConfig directory for uplink:" diff --git a/scripts/tests/testversions/test-versions.sh b/scripts/tests/testversions/test-versions.sh index 47696352e..8e9dcf556 100755 --- a/scripts/tests/testversions/test-versions.sh +++ b/scripts/tests/testversions/test-versions.sh @@ -19,9 +19,9 @@ setup(){ output=$2 head -c $size $output } - random_bytes_file "2048" "$test_files_dir/small-upload-testfile" # create 2kb file of random bytes (inline) - random_bytes_file "5120" "$test_files_dir/big-upload-testfile" # create 5kb file of random bytes (remote) - random_bytes_file "131072" "$test_files_dir/multisegment-upload-testfile" # create 128kb file of random bytes (remote) + random_bytes_file "2KiB" "$test_files_dir/small-upload-testfile" # create 2kb file of random bytes (inline) + random_bytes_file "5KiB" "$test_files_dir/big-upload-testfile" # create 5kb file of random bytes (remote) + random_bytes_file "64MiB" "$test_files_dir/multisegment-upload-testfile" # create 64mb file of random bytes (remote + inline) echo "setup test successfully" } @@ -59,9 +59,9 @@ if [ ! -d ${main_cfg_dir}/uplink ]; then if [[ ${#should_use_access} -gt 0 ]]; then access=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_ACCESS) new_access=$(go run $update_access_script_path $(storj-sim --config-dir=$main_cfg_dir network env SATELLITE_0_DIR) $access) - uplink import --config-dir="${main_cfg_dir}/uplink" "${new_access}" --client.segment-size="64.0 KiB" + uplink import --config-dir="${main_cfg_dir}/uplink" "${new_access}" else - uplink setup --config-dir="${main_cfg_dir}/uplink" --non-interactive --api-key="$api_key" --satellite-addr="$sat_addr" --enc.encryption-key="test" --client.segment-size="64.0 KiB" + uplink setup --config-dir="${main_cfg_dir}/uplink" --non-interactive --api-key="$api_key" --satellite-addr="$sat_addr" --enc.encryption-key="test" fi fi