rollingupgrade: add migration to metabase
For time of transition from pointerdb to metabase we need add migration step to rollingupgrade tests and comment few cases. Change-Id: Ib12ae6aa14be35f9bf4ff3efb55cfc6957d4ceba
This commit is contained in:
Michał Niewrzał
Michal Niewrzal
parent
6ddcacbe78
commit
0ef0a3f967
4
Jenkinsfile
vendored
4
Jenkinsfile
vendored
@ -53,9 +53,11 @@ node('node') {
|
||||
|
||||
env.STORJ_SIM_POSTGRES = 'postgres://postgres@postgres:5432/teststorj?sslmode=disable'
|
||||
env.STORJ_SIM_REDIS = 'redis:6379'
|
||||
env.STORJ_MIGRATION_DB = 'postgres://postgres@postgres:5432/teststorj?sslmode=disable&options=--search_path=satellite/0/meta'
|
||||
|
||||
echo "STORJ_SIM_POSTGRES: $STORJ_SIM_POSTGRES"
|
||||
echo "STORJ_SIM_REDIS: $STORJ_SIM_REDIS"
|
||||
echo "STORJ_MIGRATION_DB: $STORJ_MIGRATION_DB"
|
||||
sh 'docker run --rm -d -e POSTGRES_HOST_AUTH_METHOD=trust --name postgres-$BUILD_NUMBER postgres:12.3'
|
||||
sh 'docker run --rm -d --name redis-$BUILD_NUMBER redis:latest'
|
||||
|
||||
@ -67,7 +69,7 @@ node('node') {
|
||||
sh 'docker exec postgres-$BUILD_NUMBER createdb -U postgres teststorj'
|
||||
// fetch the remote main branch
|
||||
sh 'git fetch --no-tags --progress -- https://github.com/storj/storj.git +refs/heads/main:refs/remotes/origin/main'
|
||||
sh 'docker run -u $(id -u):$(id -g) --rm -i -v $PWD:$PWD -w $PWD --entrypoint $PWD/scripts/tests/rollingupgrade/test-sim-rolling-upgrade.sh -e BRANCH_NAME -e STORJ_SIM_POSTGRES -e STORJ_SIM_REDIS --link redis-$BUILD_NUMBER:redis --link postgres-$BUILD_NUMBER:postgres -e CC=gcc storjlabs/golang:1.16'
|
||||
sh 'docker run -u $(id -u):$(id -g) --rm -i -v $PWD:$PWD -w $PWD --entrypoint $PWD/scripts/tests/rollingupgrade/test-sim-rolling-upgrade.sh -e BRANCH_NAME -e STORJ_SIM_POSTGRES -e STORJ_SIM_REDIS -e STORJ_MIGRATION_DB --link redis-$BUILD_NUMBER:redis --link postgres-$BUILD_NUMBER:postgres -e CC=gcc storjlabs/golang:1.16'
|
||||
}
|
||||
catch(err){
|
||||
throw err
|
||||
|
||||
@ -56,19 +56,22 @@ uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_d
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/big-upload-testfile" "sj://$bucket_name/"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/multisegment-upload-testfile" "sj://$bucket_name/"
|
||||
|
||||
# TODO we should be able to uncomment those cases when we will have at least one point release of multipart satellite after merging to main
|
||||
# old api uploads
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "${test_files_dir}/small-upload-testfile" "sj://$old_api_bucket_name/"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "${test_files_dir}/big-upload-testfile" "sj://$old_api_bucket_name/"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "${test_files_dir}/multisegment-upload-testfile" "sj://$old_api_bucket_name/"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "${test_files_dir}/small-upload-testfile" "sj://$old_api_bucket_name/"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "${test_files_dir}/big-upload-testfile" "sj://$old_api_bucket_name/"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "${test_files_dir}/multisegment-upload-testfile" "sj://$old_api_bucket_name/"
|
||||
|
||||
# new api downloads of new api uploaded files
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}"
|
||||
|
||||
# TODO we should be able to uncomment those cases when we will have at least one point release of multipart satellite after merging to main
|
||||
# new api downloads of old api uploaded files
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$old_api_bucket_name/small-upload-testfile" "${download_dst_dir2}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$old_api_bucket_name/big-upload-testfile" "${download_dst_dir2}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$old_api_bucket_name/multisegment-upload-testfile" "${download_dst_dir2}"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$old_api_bucket_name/small-upload-testfile" "${download_dst_dir2}"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$old_api_bucket_name/big-upload-testfile" "${download_dst_dir2}"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$old_api_bucket_name/multisegment-upload-testfile" "${download_dst_dir2}"
|
||||
|
||||
echo "checking files uploaded with new api and downloaded with new api"
|
||||
if cmp "${test_files_dir}/small-upload-testfile" "${download_dst_dir}/small-upload-testfile"
|
||||
@ -93,83 +96,85 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "checking files uploaded with old api and downloaded with new api"
|
||||
if cmp "${test_files_dir}/small-upload-testfile" "${download_dst_dir2}/small-upload-testfile"
|
||||
then
|
||||
echo "download test on current branch: small upload testfile matches uploaded file"
|
||||
else
|
||||
echo "download test on current branch: small upload testfile does not match uploaded file"
|
||||
exit 1
|
||||
fi
|
||||
if cmp "${test_files_dir}/big-upload-testfile" "${download_dst_dir2}/big-upload-testfile"
|
||||
then
|
||||
echo "download test on current branch: big upload testfile matches uploaded file"
|
||||
else
|
||||
echo "download test on current branch: big upload testfile does not match uploaded file"
|
||||
exit 1
|
||||
fi
|
||||
if cmp "${test_files_dir}/multisegment-upload-testfile" "${download_dst_dir2}/multisegment-upload-testfile"
|
||||
then
|
||||
echo "download test on current branch: multisegment upload testfile matches uploaded file"
|
||||
else
|
||||
echo "download test on current branch: multisegment upload testfile does not match uploaded file"
|
||||
exit 1
|
||||
fi
|
||||
# TODO we should be able to uncomment those cases when we will have at least one point release of multipart satellite after merging to main
|
||||
# echo "checking files uploaded with old api and downloaded with new api"
|
||||
# if cmp "${test_files_dir}/small-upload-testfile" "${download_dst_dir2}/small-upload-testfile"
|
||||
# then
|
||||
# echo "download test on current branch: small upload testfile matches uploaded file"
|
||||
# else
|
||||
# echo "download test on current branch: small upload testfile does not match uploaded file"
|
||||
# exit 1
|
||||
# fi
|
||||
# if cmp "${test_files_dir}/big-upload-testfile" "${download_dst_dir2}/big-upload-testfile"
|
||||
# then
|
||||
# echo "download test on current branch: big upload testfile matches uploaded file"
|
||||
# else
|
||||
# echo "download test on current branch: big upload testfile does not match uploaded file"
|
||||
# exit 1
|
||||
# fi
|
||||
# if cmp "${test_files_dir}/multisegment-upload-testfile" "${download_dst_dir2}/multisegment-upload-testfile"
|
||||
# then
|
||||
# echo "download test on current branch: multisegment upload testfile matches uploaded file"
|
||||
# else
|
||||
# echo "download test on current branch: multisegment upload testfile does not match uploaded file"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
# TODO we should be able to uncomment those cases when we will have at least one point release of multipart satellite after merging to main
|
||||
# old api downloads of new api uploaded files
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$bucket_name/small-upload-testfile" "${old_api_download_dst_dir}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$bucket_name/big-upload-testfile" "${old_api_download_dst_dir}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${old_api_download_dst_dir}"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$bucket_name/small-upload-testfile" "${old_api_download_dst_dir}"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$bucket_name/big-upload-testfile" "${old_api_download_dst_dir}"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${old_api_download_dst_dir}"
|
||||
# old api downloads of old api uploaded files
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$old_api_bucket_name/small-upload-testfile" "${old_api_download_dst_dir2}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$old_api_bucket_name/big-upload-testfile" "${old_api_download_dst_dir2}"
|
||||
uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$old_api_bucket_name/multisegment-upload-testfile" "${old_api_download_dst_dir2}"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$old_api_bucket_name/small-upload-testfile" "${old_api_download_dst_dir2}"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$old_api_bucket_name/big-upload-testfile" "${old_api_download_dst_dir2}"
|
||||
# uplink cp --config-dir="${main_cfg_dir}/uplink-old-api" --progress=false "sj://$old_api_bucket_name/multisegment-upload-testfile" "${old_api_download_dst_dir2}"
|
||||
|
||||
echo "checking files uploaded with new api and downloaded with old api"
|
||||
if cmp "${test_files_dir}/small-upload-testfile" "${old_api_download_dst_dir}/small-upload-testfile"
|
||||
then
|
||||
echo "download test on current branch: small upload testfile matches uploaded file"
|
||||
else
|
||||
echo "download test on current branch: small upload testfile does not match uploaded file"
|
||||
exit 1
|
||||
fi
|
||||
if cmp "${test_files_dir}/big-upload-testfile" "${old_api_download_dst_dir}/big-upload-testfile"
|
||||
then
|
||||
echo "download test on current branch: big upload testfile matches uploaded file"
|
||||
else
|
||||
echo "download test on current branch: big upload testfile does not match uploaded file"
|
||||
exit 1
|
||||
fi
|
||||
if cmp "${test_files_dir}/multisegment-upload-testfile" "${old_api_download_dst_dir}/multisegment-upload-testfile"
|
||||
then
|
||||
echo "download test on current branch: multisegment upload testfile matches uploaded file"
|
||||
else
|
||||
echo "download test on current branch: multisegment upload testfile does not match uploaded file"
|
||||
exit 1
|
||||
fi
|
||||
# echo "checking files uploaded with new api and downloaded with old api"
|
||||
# if cmp "${test_files_dir}/small-upload-testfile" "${old_api_download_dst_dir}/small-upload-testfile"
|
||||
# then
|
||||
# echo "download test on current branch: small upload testfile matches uploaded file"
|
||||
# else
|
||||
# echo "download test on current branch: small upload testfile does not match uploaded file"
|
||||
# exit 1
|
||||
# fi
|
||||
# if cmp "${test_files_dir}/big-upload-testfile" "${old_api_download_dst_dir}/big-upload-testfile"
|
||||
# then
|
||||
# echo "download test on current branch: big upload testfile matches uploaded file"
|
||||
# else
|
||||
# echo "download test on current branch: big upload testfile does not match uploaded file"
|
||||
# exit 1
|
||||
# fi
|
||||
# if cmp "${test_files_dir}/multisegment-upload-testfile" "${old_api_download_dst_dir}/multisegment-upload-testfile"
|
||||
# then
|
||||
# echo "download test on current branch: multisegment upload testfile matches uploaded file"
|
||||
# else
|
||||
# echo "download test on current branch: multisegment upload testfile does not match uploaded file"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
echo "checking files uploaded with old api and downloaded with old api"
|
||||
if cmp "${test_files_dir}/small-upload-testfile" "${old_api_download_dst_dir2}/small-upload-testfile"
|
||||
then
|
||||
echo "download test on current branch: small upload testfile matches uploaded file"
|
||||
else
|
||||
echo "download test on current branch: small upload testfile does not match uploaded file"
|
||||
exit 1
|
||||
fi
|
||||
if cmp "${test_files_dir}/big-upload-testfile" "${old_api_download_dst_dir2}/big-upload-testfile"
|
||||
then
|
||||
echo "download test on current branch: big upload testfile matches uploaded file"
|
||||
else
|
||||
echo "download test on current branch: big upload testfile does not match uploaded file"
|
||||
exit 1
|
||||
fi
|
||||
if cmp "${test_files_dir}/multisegment-upload-testfile" "${old_api_download_dst_dir2}/multisegment-upload-testfile"
|
||||
then
|
||||
echo "download test on current branch: multisegment upload testfile matches uploaded file"
|
||||
else
|
||||
echo "download test on current branch: multisegment upload testfile does not match uploaded file"
|
||||
exit 1
|
||||
fi
|
||||
# echo "checking files uploaded with old api and downloaded with old api"
|
||||
# if cmp "${test_files_dir}/small-upload-testfile" "${old_api_download_dst_dir2}/small-upload-testfile"
|
||||
# then
|
||||
# echo "download test on current branch: small upload testfile matches uploaded file"
|
||||
# else
|
||||
# echo "download test on current branch: small upload testfile does not match uploaded file"
|
||||
# exit 1
|
||||
# fi
|
||||
# if cmp "${test_files_dir}/big-upload-testfile" "${old_api_download_dst_dir2}/big-upload-testfile"
|
||||
# then
|
||||
# echo "download test on current branch: big upload testfile matches uploaded file"
|
||||
# else
|
||||
# echo "download test on current branch: big upload testfile does not match uploaded file"
|
||||
# exit 1
|
||||
# fi
|
||||
# if cmp "${test_files_dir}/multisegment-upload-testfile" "${old_api_download_dst_dir2}/multisegment-upload-testfile"
|
||||
# then
|
||||
# echo "download test on current branch: multisegment upload testfile matches uploaded file"
|
||||
# else
|
||||
# echo "download test on current branch: multisegment upload testfile does not match uploaded file"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
rm -rf ${download_dst_dir}
|
||||
rm -rf ${download_dst_dir2}
|
||||
@ -179,7 +184,8 @@ uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/big-upload-te
|
||||
uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/multisegment-upload-testfile"
|
||||
uplink rb --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name"
|
||||
|
||||
uplink rm --config-dir="${main_cfg_dir}/uplink-old-api" "sj://$old_api_bucket_name/small-upload-testfile"
|
||||
uplink rm --config-dir="${main_cfg_dir}/uplink-old-api" "sj://$old_api_bucket_name/big-upload-testfile"
|
||||
uplink rm --config-dir="${main_cfg_dir}/uplink-old-api" "sj://$old_api_bucket_name/multisegment-upload-testfile"
|
||||
uplink rb --config-dir="${main_cfg_dir}/uplink-old-api" "sj://$old_api_bucket_name"
|
||||
# TODO we should be able to uncomment those cases when we will have at least one point release of multipart satellite after merging to main
|
||||
# uplink rm --config-dir="${main_cfg_dir}/uplink-old-api" "sj://$old_api_bucket_name/small-upload-testfile"
|
||||
# uplink rm --config-dir="${main_cfg_dir}/uplink-old-api" "sj://$old_api_bucket_name/big-upload-testfile"
|
||||
# uplink rm --config-dir="${main_cfg_dir}/uplink-old-api" "sj://$old_api_bucket_name/multisegment-upload-testfile"
|
||||
# uplink rb --config-dir="${main_cfg_dir}/uplink-old-api" "sj://$old_api_bucket_name"
|
||||
|
||||
@ -24,6 +24,9 @@ until $(docker logs postgres-$BUILD_NUMBER | grep "database system is ready to a
|
||||
done
|
||||
|
||||
docker exec postgres-$BUILD_NUMBER createdb -U postgres teststorj
|
||||
|
||||
export STORJ_MIGRATION_DB="${STORJ_SIM_POSTGRES}&options=--search_path=satellite/0/meta"
|
||||
|
||||
# fetch the remote main branch
|
||||
git fetch --no-tags --progress -- https://github.com/storj/storj.git +refs/heads/main:refs/remotes/origin/main
|
||||
$SCRIPTDIR/test-sim-rolling-upgrade.sh
|
||||
|
||||
@ -141,6 +141,10 @@ setup_stage(){
|
||||
if [[ $stage == "2" ]]
|
||||
then
|
||||
mv $dest_sat_cfg_dir/satellite $dest_sat_cfg_dir/old_satellite
|
||||
|
||||
go install storj.io/storj/cmd/metainfo-migration
|
||||
STORJ_MIGRATION_DB=${STORJ_MIGRATION_DB:-$STORJ_SIM_POSTGRES}
|
||||
metainfo-migration --pointerdb "${STORJ_MIGRATION_DB}" --metabasedb "${STORJ_MIGRATION_DB}"
|
||||
fi
|
||||
|
||||
# ln binary and copy config.yaml for desired version
|
||||
|
||||
Loading…
Reference in New Issue
Block a user