From 6277d93c82361cad07d9be6566e29b4c97bcba9a Mon Sep 17 00:00:00 2001 From: Clement Sam Date: Thu, 26 Jan 2023 13:49:12 +0000 Subject: [PATCH] cmd/storagenode: remove storagenode dockerfiles Updates https://github.com/storj/storagenode-docker/issues/1 Change-Id: I6b2f42194cb0e32f1b3978072fd6e1cdec753704 --- Makefile | 73 +--------- cmd/storagenode/Dockerfile | 27 ---- cmd/storagenode/Dockerfile.base | 14 -- .../docker-base/bin/stop-supervisor | 7 - .../etc/supervisor/supervisord.conf | 41 ------ cmd/storagenode/docker/app/dashboard.sh | 3 - cmd/storagenode/docker/bin/systemctl | 8 -- cmd/storagenode/docker/entrypoint | 129 ------------------ 8 files changed, 2 insertions(+), 300 deletions(-) delete mode 100644 cmd/storagenode/Dockerfile delete mode 100644 cmd/storagenode/Dockerfile.base delete mode 100755 cmd/storagenode/docker-base/bin/stop-supervisor delete mode 100644 cmd/storagenode/docker-base/etc/supervisor/supervisord.conf delete mode 100755 cmd/storagenode/docker/app/dashboard.sh delete mode 100755 cmd/storagenode/docker/bin/systemctl delete mode 100755 cmd/storagenode/docker/entrypoint diff --git a/Makefile b/Makefile index 166215006..3d647f9a6 100644 --- a/Makefile +++ b/Makefile @@ -296,7 +296,7 @@ satellite-wasm: scripts/build-wasm.sh ;\ .PHONY: images -images: multinode-image satellite-image uplink-image storagenode-image versioncontrol-image ## Build multinode, satellite, storagenode, and versioncontrol Docker images +images: multinode-image satellite-image uplink-image versioncontrol-image ## Build multinode, satellite and versioncontrol Docker images echo Built version: ${TAG} .PHONY: multinode-image @@ -332,58 +332,6 @@ satellite-image: satellite_linux_arm satellite_linux_arm64 satellite_linux_amd64 --build-arg=GOARCH=arm64 --build-arg=DOCKER_ARCH=arm64v8 \ -f cmd/satellite/Dockerfile . -.PHONY: storagenode-image -storagenode-image: ## Build storagenode Docker image - ${DOCKER_BUILD} --pull=true -t storjlabs/storagenode:${TAG}${CUSTOMTAG}-amd64 \ - --build-arg=GOARCH=amd64 \ - -f cmd/storagenode/Dockerfile . - ${DOCKER_BUILD} --pull=true -t storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm32v5 \ - --build-arg=GOARCH=arm --build-arg=DOCKER_ARCH=arm32v5 --build-arg=DOCKER_PLATFORM=linux/arm/v5 \ - -f cmd/storagenode/Dockerfile . - ${DOCKER_BUILD} --pull=true -t storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm64v8 \ - --build-arg=GOARCH=arm64 --build-arg=DOCKER_ARCH=arm64v8 --build-arg=DOCKER_PLATFORM=linux/arm64 \ - -f cmd/storagenode/Dockerfile . - -.PHONY: storagenode-base-image -storagenode-base-image: ## Build storagenode Docker base image. Requires buildx. This image is expected to be built manually using buildx and QEMU. - ${DOCKER_BUILDX} --pull=true -t storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-amd64 \ - -f cmd/storagenode/Dockerfile.base . - ${DOCKER_BUILDX} --pull=true -t storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm32v5 \ - --build-arg=GOARCH=arm --build-arg=DOCKER_ARCH=arm32v5 \ - -f cmd/storagenode/Dockerfile.base . - ${DOCKER_BUILDX} --pull=true -t storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm64v8 \ - --build-arg=GOARCH=arm64 --build-arg=DOCKER_ARCH=arm64v8 \ - -f cmd/storagenode/Dockerfile.base . - -.PHONY: push-storagenode-base-image -push-storagenode-base-image: ## Push the storagenode base image to dockerhub - docker push storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-amd64 - docker push storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm32v5 - docker push storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm64v8 - # create, annotate and push manifests for latest-amd64 - docker manifest create storjlabs/storagenode-base:latest-amd64 storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-amd64 - docker manifest annotate storjlabs/storagenode-base:latest-amd64 storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-amd64 --os linux --arch amd64 - docker manifest push --purge storjlabs/storagenode-base:latest-amd64 - # create, annotate and push manifests for latest-arm32v5 - docker manifest create storjlabs/storagenode-base:latest-arm32v5 storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm32v5 - docker manifest annotate storjlabs/storagenode-base:latest-arm32v5 storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm32v5 --os linux --arch arm --variant v5 - docker manifest push --purge storjlabs/storagenode-base:latest-arm32v5 - # create, annotate and push manifests for latest-arm64v8 - docker manifest create storjlabs/storagenode-base:latest-arm64v8 storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm64v8 - docker manifest annotate storjlabs/storagenode-base:latest-arm64v8 storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge storjlabs/storagenode-base:latest-arm64v8 - # create, annotate and push manifests for main ${GIT_TAG}${CUSTOMTAG} tag without arch extension and latest tag - for t in ${GIT_TAG}${CUSTOMTAG} latest; do \ - docker manifest create storjlabs/storagenode-base:$$t \ - storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-amd64 \ - storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm32v5 \ - storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm64v8 \ - && docker manifest annotate storjlabs/storagenode-base:$$t storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-amd64 --os linux --arch amd64 \ - && docker manifest annotate storjlabs/storagenode-base:$$t storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm32v5 --os linux --arch arm --variant v5 \ - && docker manifest annotate storjlabs/storagenode-base:$$t storjlabs/storagenode-base:${GIT_TAG}${CUSTOMTAG}-arm64v8 --os linux --arch arm64 --variant v8 \ - && docker manifest push --purge storjlabs/storagenode-base:$$t \ - ; done - .PHONY: versioncontrol-image versioncontrol-image: versioncontrol_linux_arm versioncontrol_linux_arm64 versioncontrol_linux_amd64 ## Build versioncontrol Docker image ${DOCKER_BUILD} --pull=true -t storjlabs/versioncontrol:${TAG}${CUSTOMTAG}-amd64 \ @@ -490,7 +438,7 @@ sign-windows-installer: .PHONY: push-images push-images: ## Push Docker images to Docker Hub (jenkins) # images have to be pushed before a manifest can be created - for c in multinode satellite uplink storagenode versioncontrol ; do \ + for c in multinode satellite uplink versioncontrol ; do \ docker push storjlabs/$$c:${TAG}${CUSTOMTAG}-amd64 \ && docker push storjlabs/$$c:${TAG}${CUSTOMTAG}-arm32v5 \ && docker push storjlabs/$$c:${TAG}${CUSTOMTAG}-arm64v8 \ @@ -506,22 +454,6 @@ push-images: ## Push Docker images to Docker Hub (jenkins) ; done \ ; done -.PHONY: push-storagenode-images -push-storagenode-images: - docker push storjlabs/storagenode:${TAG}${CUSTOMTAG}-amd64 \ - && docker push storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm32v5 \ - && docker push storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm64v8 \ - && for t in ${TAG}${CUSTOMTAG} ${LATEST_TAG}; do \ - docker manifest create storjlabs/storagenode:$$t \ - storjlabs/storagenode:${TAG}${CUSTOMTAG}-amd64 \ - storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm32v5 \ - storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm64v8 \ - && docker manifest annotate storjlabs/storagenode:$$t storjlabs/storagenode:${TAG}${CUSTOMTAG}-amd64 --os linux --arch amd64 \ - && docker manifest annotate storjlabs/storagenode:$$t storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm32v5 --os linux --arch arm --variant v5 \ - && docker manifest annotate storjlabs/storagenode:$$t storjlabs/storagenode:${TAG}${CUSTOMTAG}-arm64v8 --os linux --arch arm64 --variant v8 \ - && docker manifest push --purge storjlabs/storagenode:$$t \ - ; done - .PHONY: binaries-upload binaries-upload: ## Upload binaries to Google Storage (jenkins) cd "release/${TAG}"; for f in *; do \ @@ -554,7 +486,6 @@ binaries-clean: ## Remove all local release binaries (jenkins) clean-images: -docker rmi storjlabs/multinode:${TAG}${CUSTOMTAG} -docker rmi storjlabs/satellite:${TAG}${CUSTOMTAG} - -docker rmi storjlabs/storagenode:${TAG}${CUSTOMTAG} -docker rmi storjlabs/versioncontrol:${TAG}${CUSTOMTAG} ##@ Tooling diff --git a/cmd/storagenode/Dockerfile b/cmd/storagenode/Dockerfile deleted file mode 100644 index 663a4144e..000000000 --- a/cmd/storagenode/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -ARG DOCKER_ARCH -ARG DOCKER_PLATFORM - -FROM --platform=${DOCKER_PLATFORM:-linux/amd64} storjlabs/storagenode-base:70e276ecb-${DOCKER_ARCH:-amd64} -ARG TAG -ARG GOARCH -ARG VERSION_SERVER_URL -ARG SUPERVISOR_SERVER -ENV GOARCH ${GOARCH:-amd64} -ENV VERSION_SERVER_URL ${VERSION_SERVER_URL:-https://version.storj.io} -ENV SUPERVISOR_SERVER ${SUPERVISOR_SERVER:-unix} -EXPOSE 28967 -EXPOSE 14002 -# copy the files individually to avoid overriding the permissions on the folders -COPY cmd/storagenode/docker/entrypoint /entrypoint -COPY cmd/storagenode/docker/app/dashboard.sh /app/dashboard.sh -COPY cmd/storagenode/docker/bin/systemctl /bin/systemctl -WORKDIR /app -ENTRYPOINT ["/entrypoint"] - -ENV ADDRESS="" \ - EMAIL="" \ - WALLET="" \ - STORAGE="2.0TB" \ - SETUP="false" \ - AUTO_UPDATE="true" \ - LOG_LEVEL="" diff --git a/cmd/storagenode/Dockerfile.base b/cmd/storagenode/Dockerfile.base deleted file mode 100644 index bd16848c8..000000000 --- a/cmd/storagenode/Dockerfile.base +++ /dev/null @@ -1,14 +0,0 @@ -ARG DOCKER_ARCH - -FROM ${DOCKER_ARCH:-amd64}/debian:buster-slim -RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates supervisor unzip wget -RUN update-ca-certificates -RUN mkdir -p /var/log/supervisor /app -COPY cmd/storagenode/docker-base/ / -# set permissions to allow non-root access -RUN chmod -R a+rw /etc/supervisor /var/log/supervisor /app -# remove the default supervisord.conf -RUN rm -rf /etc/supervisord.conf -# create a symlink to custom supervisord config file at the default location -RUN ln -s /etc/supervisor/supervisord.conf /etc/supervisord.conf diff --git a/cmd/storagenode/docker-base/bin/stop-supervisor b/cmd/storagenode/docker-base/bin/stop-supervisor deleted file mode 100755 index 3caa55167..000000000 --- a/cmd/storagenode/docker-base/bin/stop-supervisor +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -printf "READY\n"; - -while read -r; do - kill -SIGQUIT $PPID -done < /dev/stdin \ No newline at end of file diff --git a/cmd/storagenode/docker-base/etc/supervisor/supervisord.conf b/cmd/storagenode/docker-base/etc/supervisor/supervisord.conf deleted file mode 100644 index 76b3c83e6..000000000 --- a/cmd/storagenode/docker-base/etc/supervisor/supervisord.conf +++ /dev/null @@ -1,41 +0,0 @@ -[supervisord] -user=root -nodaemon=true -logfile=/var/log/supervisor/supervisord.log -pidfile=/etc/supervisor/supervisord.pid -childlogdir=/var/log/supervisor - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[unix_http_server] -file=/etc/supervisor/supervisor.sock -username=dummy -password=dummy - -[supervisorctl] -serverurl=unix:///etc/supervisor/supervisor.sock -username=dummy -password=dummy - -[program:storagenode-updater] -command=/app/storagenode-updater -autostart=%(ENV_AUTO_UPDATE)s -autorestart=true -environment=STORJ_ENV_PREFIX=STORJUPDATER -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 - -[program:storagenode] -command=/app/storagenode -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stdout -stderr_logfile_maxbytes=0 - -[eventlistener:processes-exit-eventlistener] -command=/bin/stop-supervisor -events=PROCESS_STATE_EXITED, PROCESS_STATE_FATAL \ No newline at end of file diff --git a/cmd/storagenode/docker/app/dashboard.sh b/cmd/storagenode/docker/app/dashboard.sh deleted file mode 100755 index 683eef12b..000000000 --- a/cmd/storagenode/docker/app/dashboard.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -./storagenode dashboard --config-dir /app/config --identity-dir /app/identity $@ \ No newline at end of file diff --git a/cmd/storagenode/docker/bin/systemctl b/cmd/storagenode/docker/bin/systemctl deleted file mode 100755 index aee4da1e7..000000000 --- a/cmd/storagenode/docker/bin/systemctl +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -if [ "$*" != "show --property=MainPID storagenode" ]; then - echo "invalid command" >&2 - exit 1 -fi - -printf 'MainPID=%s' "$(supervisorctl -c /etc/supervisor/supervisord.conf pid storagenode)" diff --git a/cmd/storagenode/docker/entrypoint b/cmd/storagenode/docker/entrypoint deleted file mode 100755 index 5cd460e3a..000000000 --- a/cmd/storagenode/docker/entrypoint +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash -set -euo pipefail - -get_default_url() { - process=$1 - version=$2 - wget -O- "${VERSION_SERVER_URL}/processes/${process}/${version}/url?os=linux&arch=${GOARCH}" -} - -get_binary() { - binary=$1 - url=$2 - wget -O "/tmp/${binary}.zip" "${url}" - unzip -p "/tmp/${binary}.zip" > "/app/${binary}" - rm "/tmp/${binary}.zip" - chmod u+x "/app/${binary}" -} - -# install storagenode and storagenode-updater binaries -# during run of the container to not to release new docker image -# on each new version of the storagenode binary. -if [ ! -f "storagenode-updater" ]; then - echo "downloading storagenode-updater" - get_binary storagenode-updater "$(get_default_url storagenode-updater minimum)" - - if ./storagenode-updater should-update storagenode-updater \ - --binary-location /app/storagenode-updater \ - --identity-dir identity \ - --version.server-address="${VERSION_SERVER_URL}" 2>/dev/null - then - echo "updating storagenode-updater" - get_binary storagenode-updater "$(get_default_url storagenode-updater suggested)" - fi -fi - -if [ ! -f "storagenode" ]; then - echo "downloading storagenode" - - if ./storagenode-updater should-update storagenode \ - --identity-dir identity \ - --version.server-address="${VERSION_SERVER_URL}" 2>/dev/null - then - get_binary storagenode "$(get_default_url storagenode suggested)" - else - get_binary storagenode "$(get_default_url storagenode minimum)" - fi -fi - -SUPERVISOR_SERVER="${SUPERVISOR_SERVER:-unix}" - -RUN_PARAMS="${RUN_PARAMS:-} --config-dir config" -RUN_PARAMS="${RUN_PARAMS} --identity-dir identity" -RUN_PARAMS="${RUN_PARAMS} --metrics.app-suffix=-alpha" -RUN_PARAMS="${RUN_PARAMS} --metrics.interval=30m" - -if [ -n "${VERSION_SERVER_URL:-}" ]; then - RUN_PARAMS="${RUN_PARAMS} --version.server-address=${VERSION_SERVER_URL}" -fi - -if [ "${AUTO_UPDATE:-}" != "true" ]; then - AUTO_UPDATE="false" -fi - -SNO_RUN_PARAMS="${RUN_PARAMS} --console.address=:14002" -if [ -n "${STORAGE:-}" ]; then - SNO_RUN_PARAMS="${SNO_RUN_PARAMS} --storage.allocated-disk-space=${STORAGE}" -fi - -if [ -n "${ADDRESS:-}" ]; then - SNO_RUN_PARAMS="${SNO_RUN_PARAMS} --contact.external-address=${ADDRESS}" -fi - -if [ -n "${EMAIL:-}" ]; then - SNO_RUN_PARAMS="${SNO_RUN_PARAMS} --operator.email=${EMAIL}" -fi - -if [ -n "${WALLET:-}" ]; then - SNO_RUN_PARAMS="${SNO_RUN_PARAMS} --operator.wallet=${WALLET}" -fi - -if [ -n "${LOG_LEVEL:-}" ]; then - SNO_RUN_PARAMS="${SNO_RUN_PARAMS} --log.level=${LOG_LEVEL}" -fi - -if [ "${SETUP:-}" = "true" ]; then - echo "Running ./storagenode setup $SNO_RUN_PARAMS ${*}" - exec ./storagenode setup ${SNO_RUN_PARAMS} ${*} -else - sed -i \ - "s#^command=/app/storagenode-updater\$#command=/app/storagenode-updater run --binary-location /app/storagenode ${RUN_PARAMS} #" \ - /etc/supervisor/supervisord.conf - - sed -i \ - "s#^command=/app/storagenode\$#command=/app/storagenode run ${SNO_RUN_PARAMS} ${*}#" \ - /etc/supervisor/supervisord.conf - - # remove explicit user flag when container is run as non-root - if [ $EUID != "0" ]; then - sed -i "s#^user=root##" /etc/supervisor/supervisord.conf - fi - - # - case ${SUPERVISOR_SERVER} in - unix) # default - ;; - public_port) - # replace unix_http_server section to inet_http_server - sed -i "s#^\[unix_http_server\]\$#\[inet_http_server\]#" /etc/supervisor/supervisord.conf - # replace unix socket file with tcp public port - sed -i "s#^file=/etc/supervisor/supervisor.sock\$#port=*:9001#" /etc/supervisor/supervisord.conf - # set server url to http server address - sed -i "s#^serverurl=unix:///etc/supervisor/supervisor.sock\$#serverurl=http://127.0.0.1:9001#" /etc/supervisor/supervisord.conf - ;; - private_port) - # replace unix_http_server section to inet_http_server - sed -i "s#^\[unix_http_server\]\$#\[inet_http_server\]#" /etc/supervisor/supervisord.conf - # replace unix socket file with tcp private port .i.e. listens on only localhost - sed -i "s#^file=/etc/supervisor/supervisor.sock\$#port=127.0.0.1:9001#" /etc/supervisor/supervisord.conf - # set server url to http server address - sed -i "s#^serverurl=unix:///etc/supervisor/supervisor.sock\$#serverurl=http://127.0.0.1:9001#" /etc/supervisor/supervisord.conf - ;; - *) - echo "Invalid value '${SUPERVISOR_SERVER}' for SUPERVISOR_SERVER. Expected 'unix', 'public_port' or 'private_port'" - exit 1 - ;; - esac - - exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf -fi