storagenode: docker image autoupdate binaries

Get storagenode and storagenode-updater binaries during
run of the container to not to release new docker image
on each new version of the storagenode binary.

Fixes https://github.com/storj/storj/issues/4176

Change-Id: I994c4942136a2cc7298eb0346238689eb406ae5b

Makefile

@@ -213,15 +213,16 @@ satellite-image: satellite_linux_arm satellite_linux_arm64 satellite_linux_amd64
 		-f cmd/satellite/Dockerfile .
 
 .PHONY: storagenode-image
-storagenode-image: storagenode_linux_arm storagenode_linux_arm64 storagenode_linux_amd64 ## Build storagenode Docker image
+storagenode-image: ## Build storagenode Docker image
 	${DOCKER_BUILD} --pull=true -t storjlabs/storagenode:${TAG}${CUSTOMTAG}-amd64 \
 		-f cmd/storagenode/Dockerfile .
 	${DOCKER_BUILD} --pull=true -t storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm32v6 \
-		--build-arg=GOARCH=arm --build-arg=DOCKER_ARCH=arm32v6 \
-		-f cmd/storagenode/Dockerfile .
+    	--build-arg=GOARCH=arm --build-arg=DOCKER_ARCH=arm32v6 --build-arg=APK_ARCH=armhf \
+        -f cmd/storagenode/Dockerfile .
 	${DOCKER_BUILD} --pull=true -t storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm64v8 \
-		--build-arg=GOARCH=arm64 --build-arg=DOCKER_ARCH=arm64v8 \
-		-f cmd/storagenode/Dockerfile .
+		--build-arg=GOARCH=arm --build-arg=DOCKER_ARCH=arm64v8 --build-arg=APK_ARCH=aarch64 \
+        -f cmd/storagenode/Dockerfile .
+
 .PHONY: uplink-image
 uplink-image: uplink_linux_arm uplink_linux_arm64 uplink_linux_amd64 ## Build uplink Docker image
 	${DOCKER_BUILD} --pull=true -t storjlabs/uplink:${TAG}${CUSTOMTAG}-amd64 \

cmd/storagenode/Dockerfile

@@ -1,24 +1,34 @@
 ARG DOCKER_ARCH
 
-# Fetch ca-certificates file for arch independent builds below
-FROM alpine as ca-cert
-RUN apk -U add ca-certificates
+FROM ${DOCKER_ARCH:-amd64}/alpine as alpine-arch
 
-FROM ${DOCKER_ARCH:-amd64}/alpine
+# Fetch ca-certificates file and supervisor apk pkg with dependecies for arch independent builds below
+FROM alpine as alpine-generic
+ARG APK_ARCH=x86_64
+RUN apk -U add ca-certificates && cat /etc/apk/arch
+RUN echo $APK_ARCH > /etc/apk/arch && cat /etc/apk/arch
+RUN mkdir /tmp/apk-supervisor
+RUN rm -rf /etc/apk/keys/*
+COPY --from=alpine-arch /etc/apk/keys /etc/apk/keys
+RUN apk fetch --no-cache -R -o /tmp/apk-supervisor supervisor
+
+FROM alpine-arch
 ARG TAG
 ARG GOARCH
-ENV GOARCH ${GOARCH}
+ARG VERSION_SERVER_URL
+ENV GOARCH ${GOARCH:-amd64}
+ENV VERSION_SERVER_URL ${VERSION_SERVER_URL:-https://version.storj.io}
 EXPOSE 28967
+EXPOSE 14002
+COPY --from=alpine-generic /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=alpine-generic /tmp/apk-supervisor/* /var/lib/apk-supervisor/
+COPY cmd/storagenode/docker/ /
 WORKDIR /app
-COPY --from=ca-cert /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY release/${TAG}/storagenode_linux_${GOARCH:-amd64} /app/storagenode
-COPY cmd/storagenode/entrypoint /entrypoint
-COPY cmd/storagenode/dashboard.sh /app/dashboard.sh
 ENTRYPOINT ["/entrypoint"]
 
-# Remove after the alpha
 ENV ADDRESS="" \
     EMAIL="" \
     WALLET="" \
     STORAGE="2.0TB" \
-    SETUP="false"
+    SETUP="false" \
+    AUTO_UPDATE="true"

cmd/storagenode/docker/bin/stop-supervisor

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+printf "READY\n";
+
+while read -r; do
+  kill -SIGQUIT $PPID
+done < /dev/stdin

cmd/storagenode/docker/bin/systemctl

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if [ "$*" != "show --property=MainPID storagenode" ]; then
+  echo "invalid command" >&2
+  exit 1
+fi
+
+printf 'MainPID=%s' "$(supervisorctl pid storagenode)"

cmd/storagenode/docker/entrypoint

@@ -0,0 +1,101 @@
+#!/bin/sh
+set -euo pipefail
+
+get_default_url() {
+  process=$1
+  version=$2
+  wget -O- "${VERSION_SERVER_URL}/processes/${process}/${version}/url?os=linux&arch=${GOARCH}"
+}
+
+get_binary() {
+  binary=$1
+  url=$2
+  wget -O "/tmp/${binary}.zip" "${url}"
+  unzip -p "/tmp/${binary}.zip" > "/app/${binary}"
+  rm "/tmp/${binary}.zip"
+  chmod u+x "/app/${binary}"
+}
+
+# install storagenode and storagenode-updater binaries
+# during run of the container to not to release new docker image
+# on each new version of the storagenode binary.
+if [ ! -f "storagenode-updater" ]; then
+  echo "downloading storagenode-updater"
+  get_binary storagenode-updater "$(get_default_url storagenode-updater minimum)"
+
+  if ./storagenode-updater should-update storagenode-updater \
+        --binary-location /app/storagenode-updater \
+        --identity-dir identity \
+        --version.server-address="${VERSION_SERVER_URL}" 2>/dev/null
+  then
+    echo "updating storagenode-updater"
+    get_binary storagenode-updater "$(get_default_url storagenode-updater suggested)"
+  fi
+fi
+
+if [ ! -f "storagenode" ]; then
+  echo "downloading storagenode"
+
+  if ./storagenode-updater should-update storagenode \
+      --identity-dir identity \
+      --version.server-address="${VERSION_SERVER_URL}" 2>/dev/null
+  then
+    get_binary storagenode "$(get_default_url storagenode suggested)"
+  else
+    get_binary storagenode "$(get_default_url storagenode minimum)"
+  fi
+fi
+
+RUN_PARAMS="${RUN_PARAMS:-} --config-dir config"
+RUN_PARAMS="${RUN_PARAMS} --identity-dir identity"
+RUN_PARAMS="${RUN_PARAMS} --metrics.app-suffix=-alpha"
+RUN_PARAMS="${RUN_PARAMS} --metrics.interval=30m"
+
+if [ -n "${VERSION_SERVER_URL:-}" ]; then
+  RUN_PARAMS="${RUN_PARAMS} --version.server-address=${VERSION_SERVER_URL}"
+fi
+
+if [ "${AUTO_UPDATE:-}" != "true" ]; then
+  AUTO_UPDATE="false"
+fi
+
+SNO_RUN_PARAMS="${RUN_PARAMS} --console.address=:14002"
+if [ -n "${STORAGE:-}" ]; then
+  SNO_RUN_PARAMS="${SNO_RUN_PARAMS} --storage.allocated-disk-space=${STORAGE}"
+fi
+
+if [ -n "${ADDRESS:-}" ]; then
+  SNO_RUN_PARAMS="${SNO_RUN_PARAMS} --contact.external-address=${ADDRESS}"
+fi
+
+if [ -n "${EMAIL:-}" ]; then
+  SNO_RUN_PARAMS="${SNO_RUN_PARAMS} --operator.email=${EMAIL}"
+fi
+
+if [ -n "${WALLET:-}" ]; then
+  SNO_RUN_PARAMS="${SNO_RUN_PARAMS} --operator.wallet=${WALLET}"
+fi
+
+if [ "${SETUP:-}" = "true" ]; then
+  echo "Running ./storagenode setup $SNO_RUN_PARAMS"
+  exec ./storagenode setup "$SNO_RUN_PARAMS" "${*}"
+else
+    # install supervisor
+    # TODO: use docker buildx instead of installing packages at runtime?
+    if ! apk list supervisor 2>/dev/null | grep "supervisor"; then
+      echo "installing supervisor"
+      mkdir -p /var/log/supervisor
+      # perform installation during runtime to not use dockerfile RUN directive to support arch-independent build process
+      apk add --no-cache --repositories-file=/dev/null /var/lib/apk-supervisor/*.apk
+
+      sed -i \
+      "s#^command=/app/storagenode-updater\$#command=/app/storagenode-updater run --binary-location /app/storagenode ${RUN_PARAMS} ${*}#" \
+      /etc/supervisord.conf
+
+      sed -i \
+      "s#^command=/app/storagenode\$#command=/app/storagenode run ${SNO_RUN_PARAMS} ${*}#" \
+      /etc/supervisord.conf
+    fi
+
+  exec /usr/bin/supervisord -c /etc/supervisord.conf
+fi

cmd/storagenode/docker/etc/supervisord.conf

@@ -0,0 +1,36 @@
+[supervisord]
+user=root
+nodaemon=true
+logfile=/var/log/supervisor/supervisord.log
+pidfile=/run/supervisord.pid
+childlogdir=/var/log/supervisor
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[supervisorctl]
+serverurl = http://127.0.0.1:9001
+
+[program:storagenode-updater]
+command=/app/storagenode-updater
+autostart=%(ENV_AUTO_UPDATE)s
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+
+[program:storagenode]
+command=/app/storagenode
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+
+[eventlistener:processes]
+command=/bin/stop-supervisor
+events=PROCESS_STATE_EXITED, PROCESS_STATE_FATAL

cmd/storagenode/entrypoint

@@ -1,20 +0,0 @@
-#!/bin/sh
-set -euo pipefail
-
-if [[ "${SETUP:-}" == "true" ]]
-then
-./storagenode setup --config-dir config --identity-dir identity
-else
-RUN_PARAMS="${RUN_PARAMS:-} --config-dir config"
-RUN_PARAMS="${RUN_PARAMS:-} --identity-dir identity"
-
-RUN_PARAMS="${RUN_PARAMS:-} --metrics.app-suffix=-alpha"
-RUN_PARAMS="${RUN_PARAMS:-} --metrics.interval=30m"
-RUN_PARAMS="${RUN_PARAMS:-} --contact.external-address=${ADDRESS}"
-RUN_PARAMS="${RUN_PARAMS:-} --operator.email=${EMAIL}"
-RUN_PARAMS="${RUN_PARAMS:-} --operator.wallet=${WALLET}"
-RUN_PARAMS="${RUN_PARAMS:-} --console.address=:14002"
-RUN_PARAMS="${RUN_PARAMS:-} --storage.allocated-disk-space=${STORAGE}"
-
-exec ./storagenode run $RUN_PARAMS "$@"
-fi