From 29aac75d80c3a813a0c4f31f46caf58ca457545c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Mon, 10 Oct 2022 12:36:25 +0200 Subject: [PATCH] build: provides earthfile for nightly build Earthly is a build tool, it uses buildkitd to create reproducable and highly cacheable builds. It is used by a new experimental nightly build to easily create storj-up images. (but can be used for any ad-hoc storj-up cluster to create the images). To make the nightly more robust, I would prefer to commit the helper files (today I do a rebase every time, but sometimes it fails). More detailed information about Earthly can be found at https://earthly.dev or https://www.youtube.com/watch?v=nChpMEdOaCQ Change-Id: I683601e0558aca53b45ed3819c46c909534f8b15 --- .earthlyignore | 4 + .gitignore | 4 + Earthfile | 155 +++++++++++++++++++++++++++++++ scripts/deploy/deploy-nightly.sh | 7 ++ 4 files changed, 170 insertions(+) create mode 100644 .earthlyignore create mode 100644 Earthfile create mode 100755 scripts/deploy/deploy-nightly.sh diff --git a/.earthlyignore b/.earthlyignore new file mode 100644 index 000000000..13da43279 --- /dev/null +++ b/.earthlyignore @@ -0,0 +1,4 @@ +build +.build +release +node_modules diff --git a/.gitignore b/.gitignore index 3e93e8f94..63429b57a 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ resource.syso # storj-up automatically generates these files and should not be commit /docker-compose.yaml /.history/ +dist + +# intellij project level runners. Add them explicitly if you would like to share. +.run diff --git a/Earthfile b/Earthfile new file mode 100644 index 000000000..8055a06c8 --- /dev/null +++ b/Earthfile @@ -0,0 +1,155 @@ +VERSION 0.6 +FROM golang:1.18 +WORKDIR /go/storj + +multinode-web: + FROM node:18 + WORKDIR /build + COPY web/multinode . + RUN ./build.sh + SAVE ARTIFACT dist AS LOCAL web/storagenode/dist + +wasm: + FROM storjlabs/ci + ENV TAG=earthly + COPY go.mod . + COPY go.sum . + COPY satellite/console/wasm satellite/console/wasm + COPY satellite/console/consolewasm satellite/console/consolewasm + COPY scripts scripts + RUN scripts/build-wasm.sh + SAVE ARTIFACT release/earthly/wasm wasm AS LOCAL web/satellite/static/wasm + +storagenode-web: + FROM node:18 + WORKDIR /build + COPY web/storagenode . + RUN ./build.sh + SAVE ARTIFACT dist AS LOCAL web/storagenode/dist + SAVE ARTIFACT static AS LOCAL web/storagenode/static + +satellite-web: + FROM node:18 + WORKDIR /build + COPY web/satellite . + RUN ./build.sh + COPY +wasm/wasm static/wasm + SAVE ARTIFACT dist AS LOCAL web/satellite/dist + SAVE ARTIFACT static AS LOCAL web/satellite/static + +storagenode-bin: + COPY go.mod go.mod + COPY go.sum go.sum + COPY private private + COPY cmd/storagenode cmd/storagenode + COPY storage storage + COPY storagenode storagenode + COPY multinode multinode + COPY web/storagenode web/storagenode + RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go build -o release/earthly/storagenode storj.io/storj/cmd/storagenode + SAVE ARTIFACT release/earthly binaries AS LOCAL release/earthly + +build-binaries: + COPY . . + RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go build -o release/earthly/satellite storj.io/storj/cmd/satellite + RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go build -o release/earthly/uplink storj.io/storj/cmd/uplink + RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go build -o release/earthly/identity storj.io/storj/cmd/identity + SAVE ARTIFACT release/earthly binaries AS LOCAL release/earthly + +build-storjup: + RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 go install storj.io/storj-up@25258f9 + SAVE ARTIFACT /go/bin binaries AS LOCAL dist/up + + +deploy-remote: + FROM ubuntu + RUN apt-get update && apt-get install -y git wget unzip + RUN cd /tmp && wget https://releases.hashicorp.com/nomad/1.3.5/nomad_1.3.5_linux_amd64.zip -O nomad.zip && unzip nomad.zip && mv nomad /usr/local/bin && rm nomad.zip + COPY +build-storjup/binaries /usr/local/bin + COPY .git .git + ARG TAG=$(git rev-parse --short HEAD) + ARG IMAGE=img.dev.storj.io/dev/storj + BUILD +build-tagged-image --TAG=$TAG --IMAGE=$IMAGE + ARG --required nomad + ARG --required ip + COPY scripts/deploy/deploy-nightly.sh . + ENV NOMAD_ADDR=$nomad + ENV IP=$ip + ENV IMAGE=$IMAGE + RUN --push ./deploy-nightly.sh + +deploy-local: + COPY +build-storjup/binaries /usr/local/bin + COPY .git .git + ARG TAG=$(git rev-parse --short HEAD) + ARG IMAGE=img.dev.storj.io/dev/storj + BUILD +build-tagged-image --TAG=$TAG --IMAGE=$IMAGE + WORKDIR /opt/storj-up + RUN storj-up init db,minimal,edge + RUN storj-up image satellite-api,storagenode $IMAGE:$TAG + SAVE ARTIFACT /opt/storj-up/docker-compose.yaml compose AS LOCAL docker-compose.yaml + +build-image: + FROM storjlabs/ci + COPY .git .git + ARG IMAGE=img.dev.storj.io/dev/storj + ARG TAG=$(git rev-parse --short HEAD) + BUILD +build-tagged-image --TAG=$TAG --IMAGE=$IMAGE + +build-tagged-image: + ARG --required TAG + ARG --required IMAGE + FROM img.dev.storj.io/storjup/base:20221011-2 + COPY +multinode-web/dist /var/lib/storj/storj/web/multinode/dist + COPY +satellite-web/dist /var/lib/storj/storj/web/satellite/dist + COPY +satellite-web/static /var/lib/storj/storj/web/satellite/static + COPY +storagenode-web/dist /var/lib/storj/storj/web/storagenode/dist + COPY +storagenode-web/static /var/lib/storj/storj/web/storagenode/static + COPY +build-binaries/binaries /var/lib/storj/go/bin/ + COPY +storagenode-bin/binaries /var/lib/storj/go/bin/ + COPY +build-storjup/binaries /var/lib/storj/go/bin/ + SAVE IMAGE --push $IMAGE:$TAG $IMAGE:latest + +run: + LOCALLY + RUN docker-compose up + +test: + COPY . . + RUN go install github.com/mfridman/tparse@36f80740879e24ba6695649290a240c5908ffcbb + RUN mkdir build + RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go test -json ./... | tee build/tests.json + SAVE ARTIFACT build/tests.json AS LOCAL build/tests.json + +integration: + COPY +build/storj-up /usr/local/bin/storj-up + COPY test/test.sh . + WITH DOCKER + RUN ./test.sh + END + +check-format: + COPY . . + RUN mkdir build + RUN bash -c '[[ $(git status --short) == "" ]] || (echo "Before formatting, please commit all your work!!! (Formatter will format only last commit)" && exit -1)' + RUN git show --name-only --pretty=format: | grep ".go" | xargs --no-run-if-empty -n1 gofmt -s -w + RUN git diff > build/format.patch + SAVE ARTIFACT build/format.patch + +format: + LOCALLY + COPY +check-format/format.patch build/format.patch + RUN git apply --allow-empty build/format.patch + RUN git status diff --git a/scripts/deploy/deploy-nightly.sh b/scripts/deploy/deploy-nightly.sh new file mode 100755 index 000000000..f9b42c5b9 --- /dev/null +++ b/scripts/deploy/deploy-nightly.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -ex +storj-up init nomad --name=core --ip=$IP minimal,gc +storj-up image satellite-api,storagenode,gc $IMAGE +storj-up persist storagenode,satellite-api,gc +storj-up env set satellite-api STORJ_DATABASE_OPTIONS_MIGRATION_UNSAFE=snapshot,testdata +nomad run storj.hcl