scripts: add test-versions stage to private Jenkins

test-sim-versions.sh tests upgrading the satellite, storagenodes, and uplinks from the most recent release to master, and ensures that compatibility across all uplink versions since v0.15 is maintained.

Change-Id: I80a54236d0eb2d681716caf4b825a883bdc25ef1

Jenkinsfile

@@ -9,6 +9,37 @@ node('node') {
       echo "Current build result: ${currentBuild.result}"
     }
 
+    stage('Run Versions Test') {
+        try {
+          echo "Running Versions test"
+
+          env.STORJ_SIM_POSTGRES = 'postgres://postgres@postgres:5432/teststorj?sslmode=disable'
+          env.STORJ_SIM_REDIS = 'redis:6379'
+
+          echo "STORJ_SIM_POSTGRES: $STORJ_SIM_POSTGRES"
+          echo "STORJ_SIM_REDIS: $STORJ_SIM_REDIS"
+          sh 'docker run --rm -d --name postgres postgres:9.6'
+          sh 'docker run --rm -d --name redis redis:latest'
+
+          sh '''until $(docker logs postgres | grep "database system is ready to accept connections" > /dev/null)
+                do printf '.'
+                sleep 5
+                done
+            '''
+          sh 'docker exec postgres createdb -U postgres teststorj'
+          // fetch the remote master branch
+          sh 'git fetch --no-tags --progress -- https://github.com/storj/storj.git +refs/heads/master:refs/remotes/origin/master'
+          sh 'docker run -u $(id -u):$(id -g) --rm -i -v $PWD:$PWD -w $PWD --entrypoint $PWD/scripts/test-sim-versions.sh -e STORJ_SIM_POSTGRES -e STORJ_SIM_REDIS --link redis:redis --link postgres:postgres -e CC=gcc storjlabs/golang:1.13.5'
+        }
+        catch(err){
+            throw err
+        }
+        finally {
+          sh 'docker stop postgres || true'
+          sh 'docker stop redis || true'
+        }
+    }
+
     stage('Build Binaries') {
       sh 'make binaries'
 
@@ -74,7 +105,6 @@ node('node') {
 
   }
   finally {
-
     stage('Cleanup') {
       sh 'make clean-images'
       deleteDir()

scripts/test-sim-versions.sh

@@ -4,11 +4,12 @@ set -ueo pipefail
 set +x
 
 cleanup(){
-    for version in ${unique_versions}; do
+    ret=$?
-        git worktree remove --force $(version_dir $version)
+    echo "EXIT STATUS: $ret"
-    done
+    git worktree prune
     rm -rf "$TMP"
     echo "cleaned up test successfully"
+    exit "$ret"
 }
 trap cleanup EXIT
 
@@ -61,6 +62,26 @@ replace_in_file(){
     esac
 }
 
+# mirroring install-sim from the Makefile since it won't work on private Jenkins
+install_sim(){
+    local bin_dir="$1"
+    mkdir -p ${bin_dir}
+
+    go build -race -v -tags=grpc -o ${bin_dir}/storagenode-grpc storj.io/storj/cmd/storagenode >/dev/null 2>&1
+    go build -race -v -tags=drpc -o ${bin_dir}/storagenode-drpc storj.io/storj/cmd/storagenode >/dev/null 2>&1
+    go build -race -v -tags=grpc -o ${bin_dir}/satellite-grpc storj.io/storj/cmd/satellite >/dev/null 2>&1
+    go build -race -v -tags=drpc -o ${bin_dir}/satellite-drpc storj.io/storj/cmd/satellite >/dev/null 2>&1
+
+    go install -race -v -o ${bin_dir}/storagenode storj.io/storj/cmd/storagenode >/dev/null 2>&1
+    go install -race -v -o ${bin_dir}/satellite storj.io/storj/cmd/satellite >/dev/null 2>&1
+    go install -race -v -o ${bin_dir}/storj-sim storj.io/storj/cmd/storj-sim >/dev/null 2>&1
+    go install -race -v -o ${bin_dir}/versioncontrol storj.io/storj/cmd/versioncontrol >/dev/null 2>&1
+    go install -race -v -o ${bin_dir}/uplink storj.io/storj/cmd/uplink >/dev/null 2>&1
+    go install -race -v -o ${bin_dir}/gateway storj.io/storj/cmd/gateway >/dev/null 2>&1
+    go install -race -v -o ${bin_dir}/identity storj.io/storj/cmd/identity >/dev/null 2>&1
+    go install -race -v -o ${bin_dir}/certificates storj.io/storj/cmd/certificates >/dev/null 2>&1
+
+}
 
 setup_stage(){
     local test_dir=$1
@@ -109,29 +130,42 @@ unique_versions=$(find_unique_versions "$stage1_sat_version" "$stage1_uplink_ver
 
 STORJ_NETWORK_HOST4=${STORJ_NETWORK_HOST4:-127.0.0.1}
 STORJ_SIM_POSTGRES=${STORJ_SIM_POSTGRES:-""}
+STORJ_SIM_REDIS=${STORJ_SIM_REDIS:-""}
 
 if [ -z ${STORJ_SIM_POSTGRES} ]; then
     echo "STORJ_SIM_POSTGRES is required for the satellite DB. Example: STORJ_SIM_POSTGRES=postgres://[user]:[pass]@[host]/[db]?sslmode=disable"
     exit 1
 fi
 
+if [ -z ${STORJ_SIM_REDIS} ]; then
+    echo "STORJ_SIM_REDIS is required for the satellite DB. Example: STORJ_SIM_REDIS=127.0.0.1:[port]"
+    exit 1
+fi
+
 echo "Setting up environments for versions" ${unique_versions}
 
 # Get latest release tags and clean up git worktree
 git fetch
-git worktree prune
 for version in ${unique_versions}; do
     dir=$(version_dir ${version})
     bin_dir=${dir}/bin
 
     echo -e "\nAdding worktree for ${version} in ${dir}."
-    git worktree add -f ${dir} ${version}
+    if [[ $version = "master" ]]
+    then
+        git worktree add -f "$dir" "origin/master"
+    else
+        git worktree add -f "$dir" "${version}"
+    fi
     rm -f ${dir}/private/version/release.go
     rm -f ${dir}/internal/version/release.go
     if [[ $version = $current_release_version || $version = "master" ]]
     then
         echo "Installing storj-sim for ${version} in ${dir}."
-        GOBIN=${bin_dir} make -C "${dir}" install-sim > /dev/null 2>&1
+        pushd ${dir}
+        install_sim ${bin_dir}
+        echo "finished installing"
+        popd
         echo "Setting up storj-sim for ${version}. Bin: ${bin_dir}, Config: ${dir}/local-network"
         PATH=${bin_dir}:$PATH storj-sim -x --host="${STORJ_NETWORK_HOST4}" --postgres="${STORJ_SIM_POSTGRES}" --config-dir "${dir}/local-network" network setup > /dev/null 2>&1
         echo "Finished setting up. ${dir}/local-network:" $(ls ${dir}/local-network)
@@ -142,10 +176,11 @@ for version in ${unique_versions}; do
         shasum ${bin_dir}/gateway
     else
         echo "Installing uplink and gateway for ${version} in ${dir}."
-        cd ${dir}
+        pushd ${dir}
-        GOBIN=${bin_dir} go install -race -v storj.io/storj/cmd/uplink > /dev/null 2>&1
+        mkdir -p ${bin_dir}
-        GOBIN=${bin_dir} go install -race -v storj.io/storj/cmd/gateway > /dev/null 2>&1
+        go install -race -v -o ${bin_dir}/uplink storj.io/storj/cmd/uplink >/dev/null 2>&1
-        cd -
+        go install -race -v -o ${bin_dir}/gateway storj.io/storj/cmd/gateway >/dev/null 2>&1
+        popd
         echo "Finished installing. ${bin_dir}:" $(ls ${bin_dir})
         echo "Binary shasums:"
         shasum ${bin_dir}/uplink
@@ -164,7 +199,7 @@ setup_stage "${test_dir}" "${stage1_sat_version}" "${stage1_storagenode_versions
 
 # Uploading files to the network using the latest release version for each uplink version
 for ul_version in ${stage1_uplink_versions}; do
-    echo "Uplink version: ${ul_version}"
+    echo "Stage 1 Uplink version: ${ul_version}"
     src_ul_version_dir=$(version_dir ${ul_version})
     ln -f ${src_ul_version_dir}/bin/uplink $test_dir/bin/uplink
     PATH=$test_dir/bin:$PATH storj-sim -x --host "${STORJ_NETWORK_HOST4}" --config-dir "${test_dir}/local-network" network test bash "${scriptdir}/test-versions.sh" "${test_dir}/local-network" "upload" "${ul_version}"
@@ -176,7 +211,7 @@ echo -e "\nRunning stage 2."
 
 # Downloading every file uploaded in stage 1 from the network using the latest commit from master branch for each uplink version
 for ul_version in ${stage2_uplink_versions}; do
-    echo "Uplink version: ${ul_version}"
+    echo "Stage 2 Uplink version: ${ul_version}"
     src_ul_version_dir=$(version_dir ${ul_version})
     ln -f ${src_ul_version_dir}/bin/uplink $test_dir/bin/uplink
     PATH=$test_dir/bin:$PATH storj-sim -x --host "${STORJ_NETWORK_HOST4}" --config-dir "${test_dir}/local-network" network test bash "${scriptdir}/test-versions.sh" "${test_dir}/local-network" "download" "${stage1_uplink_versions}"