d32626fe8c
Updates config migration to occur for any v0.30.x release rather than specifically 30.4 Also updates the config for the rolling upgrade test to use 64 kib segments, and use smaller files for the final upload of rolling upgrade. Change-Id: I941f77fe2b9011b45f28a5f3a2430e882d2ae6b3
203 lines
8.5 KiB
Bash
Executable File
203 lines
8.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -ueo pipefail
|
|
|
|
main_cfg_dir=$1
|
|
command=$2
|
|
uplink_version=$3
|
|
|
|
bucket="bucket-123"
|
|
test_files_dir="${main_cfg_dir}/testfiles"
|
|
stage1_dst_dir="${main_cfg_dir}/stage1"
|
|
stage2_dst_dir="${main_cfg_dir}/stage2"
|
|
|
|
setup(){
|
|
mkdir -p "$test_files_dir" "$stage1_dst_dir" "$stage2_dst_dir"
|
|
random_bytes_file () {
|
|
size=$1
|
|
output=$2
|
|
head -c $size </dev/urandom > $output
|
|
}
|
|
random_bytes_file "2048" "$test_files_dir/small-upload-testfile" # create 2kb file of random bytes (inline)
|
|
random_bytes_file "5120" "$test_files_dir/big-upload-testfile" # create 5kb file of random bytes (remote)
|
|
random_bytes_file "143360" "$test_files_dir/multisegment-upload-testfile" # create 140kb file of random bytes (remote)
|
|
|
|
echo "setup test successfully"
|
|
}
|
|
|
|
wait_for_all_background_jobs_to_finish(){
|
|
for job in `jobs -p`
|
|
do
|
|
echo "wait for $job"
|
|
RESULT=0
|
|
wait $job || RESULT=1
|
|
if [ "$RESULT" == "1" ]; then
|
|
echo "job $job failed"
|
|
fi
|
|
done
|
|
}
|
|
|
|
echo "Begin test-versions.sh, storj-sim config directory:" ${main_cfg_dir}
|
|
|
|
echo "which storj-sim: $(which storj-sim)"
|
|
echo "Shasum for storj-sim:"
|
|
shasum $(which storj-sim)
|
|
|
|
echo -e "\nConfig directory for uplink:"
|
|
echo "${main_cfg_dir}/uplink"
|
|
echo "which uplink: $(which uplink)"
|
|
echo "Shasum for uplink:"
|
|
shasum $(which uplink)
|
|
|
|
# for oldest uplink versions, access is not supported, and we need to configure separate values for api key, sat addr, and encryption key
|
|
if [ ! -d ${main_cfg_dir}/uplink ]; then
|
|
mkdir -p ${main_cfg_dir}/uplink
|
|
api_key=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_API_KEY)
|
|
sat_addr=$(storj-sim --config-dir=$main_cfg_dir network env SATELLITE_0_ADDR)
|
|
should_use_access=$(echo $uplink_version | awk 'BEGIN{FS="[v.]"} $3 >= 30 || $2 >= 1 {print $0}')
|
|
if [[ ${#should_use_access} -gt 0 ]]; then
|
|
access=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_ACCESS)
|
|
uplink import --config-dir="${main_cfg_dir}/uplink" "${access}" --client.segment-size="64.0 KiB"
|
|
else
|
|
uplink setup --config-dir="${main_cfg_dir}/uplink" --non-interactive --api-key="$api_key" --satellite-addr="$sat_addr" --enc.encryption-key="test" --client.segment-size="64.0 KiB"
|
|
fi
|
|
fi
|
|
|
|
if [[ $uplink_version = "v0.29.10" ]]; then
|
|
uplink share --config-dir="${main_cfg_dir}/uplink" | grep "Scope" | awk -F ": " '{print $2}' | tee ${main_cfg_dir}/uplink/access.txt
|
|
fi
|
|
|
|
# after version v0.30.x we need to use access instead of separate values for api key, sat addr, and encryption key
|
|
should_use_access=$(echo $uplink_version | awk 'BEGIN{FS="[v.]"} $3 == 30 {print $0}')
|
|
if [[ ${#should_use_access} -gt 0 ]] && [ -e ${main_cfg_dir}/uplink/access.txt ]
|
|
then
|
|
# the access provided by storj-sim uses an empty encryption key; we cannot do uplink setup above with an empty encryption key
|
|
# therefore, we use a hack -> get an access key from the existing uplink config, then import that same access key
|
|
|
|
# super hack:
|
|
access=$(head -n 1 ${main_cfg_dir}/uplink/access.txt)
|
|
echo "import for uplink $access"
|
|
uplink import --config-dir="${main_cfg_dir}/uplink" "${access}"
|
|
rm -rf ${main_cfg_dir}/uplink/access.txt
|
|
fi
|
|
|
|
echo -e "\nConfig directory for satellite:"
|
|
echo "${main_cfg_dir}/satellite/0"
|
|
echo "Shasum for satellite:"
|
|
shasum ${main_cfg_dir}/satellite/0/satellite
|
|
|
|
echo -e "\nStoragenode config directories:"
|
|
for i in {0..9}
|
|
do
|
|
echo -e "\nConfig directory for sn ${i}:"
|
|
echo "${main_cfg_dir}/storagenode/${i}"
|
|
echo "Shasum for sn ${i} binary:"
|
|
shasum ${main_cfg_dir}/storagenode/${i}/storagenode
|
|
done
|
|
|
|
if [[ "$command" == "upload" ]]; then
|
|
setup
|
|
bucket_name=${bucket}-${uplink_version}
|
|
download_dst_dir=${stage1_dst_dir}/${uplink_version}
|
|
mkdir -p "$download_dst_dir"
|
|
|
|
uplink mb "sj://$bucket_name/" --config-dir="${main_cfg_dir}/uplink"
|
|
|
|
# run each upload in parallel
|
|
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/small-upload-testfile" "sj://$bucket_name/" &
|
|
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/big-upload-testfile" "sj://$bucket_name/" &
|
|
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/multisegment-upload-testfile" "sj://$bucket_name/" &
|
|
wait_for_all_background_jobs_to_finish
|
|
|
|
# run each download in parallel
|
|
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" &
|
|
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" &
|
|
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" &
|
|
wait_for_all_background_jobs_to_finish
|
|
|
|
if cmp "${test_files_dir}/small-upload-testfile" "${download_dst_dir}/small-upload-testfile"
|
|
then
|
|
echo "upload test on release tag: small upload testfile matches uploaded file"
|
|
else
|
|
echo "upload test on release tag: small upload testfile does not match uploaded file"
|
|
exit 1
|
|
fi
|
|
|
|
if cmp "${test_files_dir}/big-upload-testfile" "${download_dst_dir}/big-upload-testfile"
|
|
then
|
|
echo "upload test on release tag: big upload testfile matches uploaded file"
|
|
else
|
|
echo "upload test on release tag: big upload testfile does not match uploaded file"
|
|
exit 1
|
|
fi
|
|
|
|
if cmp "${test_files_dir}/multisegment-upload-testfile" "${download_dst_dir}/multisegment-upload-testfile"
|
|
then
|
|
echo "upload test on release tag: multisegment upload testfile matches uploaded file"
|
|
else
|
|
echo "upload test on release tag: multisegment upload testfile does not match uploaded file"
|
|
exit 1
|
|
fi
|
|
|
|
rm -rf ${test_files_dir}
|
|
fi
|
|
|
|
if [[ "$command" == "download" ]]; then
|
|
existing_bucket_name_suffixes=$4
|
|
|
|
# download all uploaded files from stage 1 with currently selected uplink
|
|
for suffix in ${existing_bucket_name_suffixes}; do
|
|
bucket_name=${bucket}-${suffix}
|
|
original_dst_dir=${stage1_dst_dir}/${suffix}
|
|
download_dst_dir=${stage2_dst_dir}/${suffix}
|
|
mkdir -p "$download_dst_dir"
|
|
|
|
echo "bucket name: ${bucket_name}"
|
|
echo "download folder name: ${download_dst_dir}"
|
|
# run each download in parallel
|
|
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" &
|
|
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" &
|
|
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" &
|
|
wait_for_all_background_jobs_to_finish
|
|
|
|
if cmp "${original_dst_dir}/small-upload-testfile" "${download_dst_dir}/small-upload-testfile"
|
|
then
|
|
echo "download test on current branch: small upload testfile matches uploaded file"
|
|
else
|
|
echo "download test on current branch: small upload testfile does not match uploaded file"
|
|
exit 1
|
|
fi
|
|
|
|
if cmp "${original_dst_dir}/big-upload-testfile" "${download_dst_dir}/big-upload-testfile"
|
|
then
|
|
echo "download test on current branch: big upload testfile matches uploaded file"
|
|
else
|
|
echo "download test on current branch: big upload testfile does not match uploaded file"
|
|
exit 1
|
|
fi
|
|
|
|
if cmp "${original_dst_dir}/multisegment-upload-testfile" "${download_dst_dir}/multisegment-upload-testfile"
|
|
then
|
|
echo "download test on current branch: multisegment upload testfile matches uploaded file"
|
|
else
|
|
echo "download test on current branch: multisegment upload testfile does not match uploaded file"
|
|
exit 1
|
|
fi
|
|
|
|
rm -rf ${download_dst_dir}
|
|
done
|
|
fi
|
|
|
|
if [[ "$command" == "cleanup" ]]; then
|
|
uplink_versions=$3
|
|
for ul_version in ${uplink_versions}; do
|
|
bucket_name=${bucket}-${ul_version}
|
|
uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/small-upload-testfile"
|
|
uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/big-upload-testfile"
|
|
uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/multisegment-upload-testfile"
|
|
uplink rb --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name"
|
|
done
|
|
fi
|
|
|
|
echo "Done with test-versions.sh"
|