scripts/tests/testversions: make binary installation and upload/download running in parallel
Change-Id: I16d87f7e16e2daf30e4d7ee5490b76c175b06930
This commit is contained in:
Yingrong Zhao
Yingrong Zhao
parent
80b41af8f1
commit
71c5c2213f
2
Jenkinsfile
vendored
2
Jenkinsfile
vendored
@ -29,7 +29,7 @@ node('node') {
|
||||
sh 'docker exec postgres createdb -U postgres teststorj'
|
||||
// fetch the remote master branch
|
||||
sh 'git fetch --no-tags --progress -- https://github.com/storj/storj.git +refs/heads/master:refs/remotes/origin/master'
|
||||
sh 'docker run -u $(id -u):$(id -g) --rm -i -v $PWD:$PWD -w $PWD --entrypoint $PWD/scripts/test-sim-versions.sh -e STORJ_SIM_POSTGRES -e STORJ_SIM_REDIS --link redis:redis --link postgres:postgres -e CC=gcc storjlabs/golang:1.13.5'
|
||||
sh 'docker run -u $(id -u):$(id -g) --rm -i -v $PWD:$PWD -w $PWD --entrypoint $PWD/scripts/tests/testversions/test-sim-versions.sh -e STORJ_SIM_POSTGRES -e STORJ_SIM_REDIS --link redis:redis --link postgres:postgres -e CC=gcc storjlabs/golang:1.13.5'
|
||||
}
|
||||
catch(err){
|
||||
throw err
|
||||
|
||||
@ -134,52 +134,70 @@ fi
|
||||
|
||||
if [ -z ${STORJ_SIM_REDIS} ]; then
|
||||
echo "STORJ_SIM_REDIS is required for the satellite DB. Example: STORJ_SIM_REDIS=127.0.0.1:[port]"
|
||||
exit 1
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setting up environments for versions" ${unique_versions}
|
||||
|
||||
# Get latest release tags and clean up git worktree
|
||||
git fetch
|
||||
# clean up git worktree
|
||||
git worktree prune
|
||||
for version in ${unique_versions}; do
|
||||
dir=$(version_dir ${version})
|
||||
bin_dir=${dir}/bin
|
||||
# run in parallel
|
||||
(
|
||||
dir=$(version_dir ${version})
|
||||
bin_dir=${dir}/bin
|
||||
|
||||
echo -e "\nAdding worktree for ${version} in ${dir}."
|
||||
if [[ $version = "master" ]]
|
||||
then
|
||||
git worktree add -f "$dir" "origin/master"
|
||||
else
|
||||
git worktree add -f "$dir" "${version}"
|
||||
fi
|
||||
rm -f ${dir}/private/version/release.go
|
||||
rm -f ${dir}/internal/version/release.go
|
||||
if [[ $version = $current_release_version || $version = "master" ]]
|
||||
then
|
||||
echo "Installing storj-sim for ${version} in ${dir}."
|
||||
pushd ${dir}
|
||||
install_sim ${bin_dir}
|
||||
echo "finished installing"
|
||||
popd
|
||||
echo "Setting up storj-sim for ${version}. Bin: ${bin_dir}, Config: ${dir}/local-network"
|
||||
PATH=${bin_dir}:$PATH storj-sim -x --host="${STORJ_NETWORK_HOST4}" --postgres="${STORJ_SIM_POSTGRES}" --config-dir "${dir}/local-network" network setup > /dev/null 2>&1
|
||||
echo "Finished setting up. ${dir}/local-network:" $(ls ${dir}/local-network)
|
||||
echo "Binary shasums:"
|
||||
shasum ${bin_dir}/satellite
|
||||
shasum ${bin_dir}/storagenode
|
||||
shasum ${bin_dir}/uplink
|
||||
shasum ${bin_dir}/gateway
|
||||
else
|
||||
echo "Installing uplink and gateway for ${version} in ${dir}."
|
||||
pushd ${dir}
|
||||
mkdir -p ${bin_dir}
|
||||
go install -race -v -o ${bin_dir}/uplink storj.io/storj/cmd/uplink >/dev/null 2>&1
|
||||
go install -race -v -o ${bin_dir}/gateway storj.io/storj/cmd/gateway >/dev/null 2>&1
|
||||
popd
|
||||
echo "Finished installing. ${bin_dir}:" $(ls ${bin_dir})
|
||||
echo "Binary shasums:"
|
||||
shasum ${bin_dir}/uplink
|
||||
shasum ${bin_dir}/gateway
|
||||
echo -e "\nAdding worktree for ${version} in ${dir}."
|
||||
if [[ $version = "master" ]]
|
||||
then
|
||||
git worktree add -f "$dir" "origin/master"
|
||||
else
|
||||
git worktree add -f "$dir" "${version}"
|
||||
fi
|
||||
rm -f ${dir}/private/version/release.go
|
||||
rm -f ${dir}/internal/version/release.go
|
||||
if [[ $version = $current_release_version || $version = "master" ]]
|
||||
then
|
||||
echo "Installing storj-sim for ${version} in ${dir}."
|
||||
pushd ${dir}
|
||||
install_sim ${bin_dir}
|
||||
echo "finished installing"
|
||||
popd
|
||||
# uncomment for local testing
|
||||
# GOBIN=${bin_dir} make -C ${dir} install-sim >/dev/null 2>&1
|
||||
echo "Setting up storj-sim for ${version}. Bin: ${bin_dir}, Config: ${dir}/local-network"
|
||||
PATH=${bin_dir}:$PATH storj-sim -x --host="${STORJ_NETWORK_HOST4}" --postgres="${STORJ_SIM_POSTGRES}" --config-dir "${dir}/local-network" network setup > /dev/null 2>&1
|
||||
echo "Finished setting up. ${dir}/local-network:" $(ls ${dir}/local-network)
|
||||
echo "Binary shasums:"
|
||||
shasum ${bin_dir}/satellite
|
||||
shasum ${bin_dir}/storagenode
|
||||
shasum ${bin_dir}/uplink
|
||||
shasum ${bin_dir}/gateway
|
||||
else
|
||||
echo "Installing uplink and gateway for ${version} in ${dir}."
|
||||
pushd ${dir}
|
||||
mkdir -p ${bin_dir}
|
||||
go install -race -v -o ${bin_dir}/uplink storj.io/storj/cmd/uplink >/dev/null 2>&1
|
||||
go install -race -v -o ${bin_dir}/gateway storj.io/storj/cmd/gateway >/dev/null 2>&1
|
||||
# uncomment for local testing
|
||||
# GOBIN=${bin_dir} go install -race -v storj.io/storj/cmd/uplink >/dev/null 2>&1
|
||||
# GOBIN=${bin_dir} go install -race -v storj.io/storj/cmd/gateway >/dev/null 2>&1
|
||||
popd
|
||||
echo "Finished installing. ${bin_dir}:" $(ls ${bin_dir})
|
||||
echo "Binary shasums:"
|
||||
shasum ${bin_dir}/uplink
|
||||
shasum ${bin_dir}/gateway
|
||||
fi
|
||||
) &
|
||||
done
|
||||
|
||||
for job in `jobs -p`
|
||||
do
|
||||
echo "wait for $job"
|
||||
RESULT=0
|
||||
wait $job || RESULT=1
|
||||
if [ "$RESULT" == "1" ]; then
|
||||
exit $?
|
||||
fi
|
||||
done
|
||||
|
||||
@ -24,6 +24,18 @@ setup(){
|
||||
echo "setup test successfully"
|
||||
}
|
||||
|
||||
wait_for_all_background_jobs_to_finish(){
|
||||
for job in `jobs -p`
|
||||
do
|
||||
echo "wait for $job"
|
||||
RESULT=0
|
||||
wait $job || RESULT=1
|
||||
if [ "$RESULT" == "1" ]; then
|
||||
exit $?
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
echo "Begin test-versions.sh, storj-sim config directory:" ${main_cfg_dir}
|
||||
|
||||
echo "which storj-sim: $(which storj-sim)"
|
||||
@ -66,13 +78,17 @@ if [[ "$command" == "upload" ]]; then
|
||||
|
||||
uplink mb "sj://$bucket_name/" --config-dir="${main_cfg_dir}/uplink"
|
||||
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/small-upload-testfile" "sj://$bucket_name/"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/big-upload-testfile" "sj://$bucket_name/"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/multisegment-upload-testfile" "sj://$bucket_name/"
|
||||
# run each upload in parallel
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/small-upload-testfile" "sj://$bucket_name/" &
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/big-upload-testfile" "sj://$bucket_name/" &
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/multisegment-upload-testfile" "sj://$bucket_name/" &
|
||||
wait_for_all_background_jobs_to_finish
|
||||
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}"
|
||||
# run each download in parallel
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" &
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" &
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" &
|
||||
wait_for_all_background_jobs_to_finish
|
||||
|
||||
if cmp "${test_files_dir}/small-upload-testfile" "${download_dst_dir}/small-upload-testfile"
|
||||
then
|
||||
@ -98,6 +114,7 @@ if [[ "$command" == "upload" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -rf ${test_files_dir}
|
||||
fi
|
||||
|
||||
if [[ "$command" == "download" ]]; then
|
||||
@ -112,9 +129,11 @@ if [[ "$command" == "download" ]]; then
|
||||
|
||||
echo "bucket name: ${bucket_name}"
|
||||
echo "download folder name: ${download_dst_dir}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}"
|
||||
# run each download in parallel
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" &
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" &
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" &
|
||||
wait_for_all_background_jobs_to_finish
|
||||
|
||||
if cmp "${original_dst_dir}/small-upload-testfile" "${download_dst_dir}/small-upload-testfile"
|
||||
then
|
||||
Loading…
Reference in New Issue
Block a user