scripts/tests/testversions;scripts/tests/rollingupgrade: update test versions script
Fix uplink setup step for uplink versions that requires an access field. Update how script selects uplink versions to test. Use significantly smaller remote files for test (performance). Change-Id: If590b8798767e2a0621fb84cd3b8852d02f6d1da
This commit is contained in:
Moby von Briesen
parent
33790e0f7e
commit
0def7a9d2a
@ -25,7 +25,7 @@ populate_sno_versions(){
|
||||
# in stage 2: satellite core uses latest release version and satellite api uses master. Storage nodes are split into half on latest release version and half on master. Uplink uses the all versions from stage 1 plus master
|
||||
git fetch --tags
|
||||
current_release_version=$(git describe --tags `git rev-list --tags --max-count=1`)
|
||||
major_release_tags=$(git tag -l --sort -version:refname | sort -k2,2 -t'.' --unique | grep -e "^v0\.\(1[5-9]\)\|2[2-9]")
|
||||
major_release_tags=$(git tag -l --sort -version:refname | sort -n -k2,2 -t'.' --unique | awk 'BEGIN{FS="[v.]"} $2 >= 0 && $3 >= 15 {print $0}')
|
||||
stage1_sat_version=$current_release_version
|
||||
stage1_uplink_versions=$major_release_tags
|
||||
stage1_storagenode_versions=$(populate_sno_versions $current_release_version 10)
|
||||
@ -134,7 +134,7 @@ fi
|
||||
|
||||
if [ -z ${STORJ_SIM_REDIS} ]; then
|
||||
echo "STORJ_SIM_REDIS is required for the satellite DB. Example: STORJ_SIM_REDIS=127.0.0.1:[port]"
|
||||
exit 1
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setting up environments for versions" ${unique_versions}
|
||||
@ -217,6 +217,8 @@ for ul_version in ${stage1_uplink_versions}; do
|
||||
ln -f ${src_ul_version_dir}/bin/uplink $test_dir/bin/uplink
|
||||
PATH=$test_dir/bin:$PATH storj-sim -x --host "${STORJ_NETWORK_HOST4}" --config-dir "${test_dir}/local-network" network test bash "${scriptdir}/test-versions.sh" "${test_dir}/local-network" "upload" "${ul_version}"
|
||||
done
|
||||
# Remove current uplink config to regenerate uplink config for older uplink version
|
||||
rm -rf "${test_dir}/local-network/uplink"
|
||||
|
||||
echo -e "\nSetting up stage 2 in ${test_dir}"
|
||||
setup_stage "${test_dir}" "${stage2_sat_version}" "${stage2_storagenode_versions}"
|
||||
@ -227,7 +229,7 @@ for ul_version in ${stage2_uplink_versions}; do
|
||||
echo "Stage 2 Uplink version: ${ul_version}"
|
||||
src_ul_version_dir=$(version_dir ${ul_version})
|
||||
ln -f ${src_ul_version_dir}/bin/uplink $test_dir/bin/uplink
|
||||
PATH=$test_dir/bin:$PATH storj-sim -x --host "${STORJ_NETWORK_HOST4}" --config-dir "${test_dir}/local-network" network test bash "${scriptdir}/test-versions.sh" "${test_dir}/local-network" "download" "${stage1_uplink_versions}"
|
||||
PATH=$test_dir/bin:$PATH storj-sim -x --host "${STORJ_NETWORK_HOST4}" --config-dir "${test_dir}/local-network" network test bash "${scriptdir}/test-versions.sh" "${test_dir}/local-network" "download" "${ul_version}" "${stage1_uplink_versions}"
|
||||
done
|
||||
|
||||
|
||||
|
||||
@ -4,6 +4,7 @@ set -ueo pipefail
|
||||
|
||||
main_cfg_dir=$1
|
||||
command=$2
|
||||
uplink_version=$3
|
||||
|
||||
bucket="bucket-123"
|
||||
test_files_dir="${main_cfg_dir}/testfiles"
|
||||
@ -18,8 +19,8 @@ setup(){
|
||||
head -c $size </dev/urandom > $output
|
||||
}
|
||||
random_bytes_file "2048" "$test_files_dir/small-upload-testfile" # create 2kb file of random bytes (inline)
|
||||
random_bytes_file "5242880" "$test_files_dir/big-upload-testfile" # create 5mb file of random bytes (remote)
|
||||
random_bytes_file "134217728" "$test_files_dir/multisegment-upload-testfile" # create 128mb file of random bytes (remote)
|
||||
random_bytes_file "5120" "$test_files_dir/big-upload-testfile" # create 5kb file of random bytes (remote)
|
||||
random_bytes_file "143360" "$test_files_dir/multisegment-upload-testfile" # create 140kb file of random bytes (remote)
|
||||
|
||||
echo "setup test successfully"
|
||||
}
|
||||
@ -31,7 +32,7 @@ wait_for_all_background_jobs_to_finish(){
|
||||
RESULT=0
|
||||
wait $job || RESULT=1
|
||||
if [ "$RESULT" == "1" ]; then
|
||||
exit $?
|
||||
echo "job $job failed"
|
||||
fi
|
||||
done
|
||||
}
|
||||
@ -48,10 +49,35 @@ echo "which uplink: $(which uplink)"
|
||||
echo "Shasum for uplink:"
|
||||
shasum $(which uplink)
|
||||
|
||||
# for oldest uplink versions, access is not supported, and we need to configure separate values for api key, sat addr, and encryption key
|
||||
if [ ! -d ${main_cfg_dir}/uplink ]; then
|
||||
mkdir -p ${main_cfg_dir}/uplink
|
||||
access=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_ACCESS)
|
||||
uplink import --config-dir="${main_cfg_dir}/uplink" "$access"
|
||||
api_key=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_API_KEY)
|
||||
sat_addr=$(storj-sim --config-dir=$main_cfg_dir network env SATELLITE_0_ADDR)
|
||||
should_use_access=$(echo $uplink_version | awk 'BEGIN{FS="[v.]"} $3 >= 30 {print $0}')
|
||||
if [[ ${#should_use_access} -gt 0 ]]; then
|
||||
access=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_ACCESS)
|
||||
uplink import --config-dir="${main_cfg_dir}/uplink" "${access}"
|
||||
else
|
||||
uplink setup --config-dir="${main_cfg_dir}/uplink" --non-interactive --api-key="$api_key" --satellite-addr="$sat_addr" --enc.encryption-key="test" --client.segment-size="64.0 KiB"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $uplink_version = "v0.29.10" ]]; then
|
||||
uplink share --config-dir="${main_cfg_dir}/uplink" | grep "Scope" | awk -F ": " '{print $2}' | tee ${main_cfg_dir}/uplink/access.txt
|
||||
fi
|
||||
|
||||
# after this version we need to use access instead of separate values for api key, sat addr, and encryption key
|
||||
if [[ $uplink_version = "v0.30.4" ]] && [ -e ${main_cfg_dir}/uplink/access.txt ]
|
||||
then
|
||||
# the access provided by storj-sim uses an empty encryption key; we cannot do uplink setup above with an empty encryption key
|
||||
# therefore, we use a hack -> get an access key from the existing uplink config, then import that same access key
|
||||
|
||||
# super hack:
|
||||
access=$(head -n 1 ${main_cfg_dir}/uplink/access.txt)
|
||||
echo "import for uplink $access"
|
||||
uplink import --config-dir="${main_cfg_dir}/uplink" "${access}"
|
||||
rm -rf ${main_cfg_dir}/uplink/access.txt
|
||||
fi
|
||||
|
||||
echo -e "\nConfig directory for satellite:"
|
||||
@ -69,7 +95,6 @@ do
|
||||
done
|
||||
|
||||
if [[ "$command" == "upload" ]]; then
|
||||
uplink_version=$3
|
||||
setup
|
||||
bucket_name=${bucket}-${uplink_version}
|
||||
download_dst_dir=${stage1_dst_dir}/${uplink_version}
|
||||
@ -117,7 +142,7 @@ if [[ "$command" == "upload" ]]; then
|
||||
fi
|
||||
|
||||
if [[ "$command" == "download" ]]; then
|
||||
existing_bucket_name_suffixes=$3
|
||||
existing_bucket_name_suffixes=$4
|
||||
|
||||
# download all uploaded files from stage 1 with currently selected uplink
|
||||
for suffix in ${existing_bucket_name_suffixes}; do
|
||||
|
||||
Loading…
Reference in New Issue
Block a user