build: provides earthfile for nightly build
Earthly is a build tool, it uses buildkitd to create reproducable and highly cacheable builds. It is used by a new experimental nightly build to easily create storj-up images. (but can be used for any ad-hoc storj-up cluster to create the images). To make the nightly more robust, I would prefer to commit the helper files (today I do a rebase every time, but sometimes it fails). More detailed information about Earthly can be found at https://earthly.dev or https://www.youtube.com/watch?v=nChpMEdOaCQ Change-Id: I683601e0558aca53b45ed3819c46c909534f8b15
This commit is contained in:
Márton Elek
Elek, Márton
parent
8c8688ca6b
commit
29aac75d80
4
.earthlyignore
Normal file
4
.earthlyignore
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
build
|
||||||
|
.build
|
||||||
|
release
|
||||||
|
node_modules
|
||||||
4
.gitignore
vendored
4
.gitignore
vendored
@ -43,3 +43,7 @@ resource.syso
|
|||||||
# storj-up automatically generates these files and should not be commit
|
# storj-up automatically generates these files and should not be commit
|
||||||
/docker-compose.yaml
|
/docker-compose.yaml
|
||||||
/.history/
|
/.history/
|
||||||
|
dist
|
||||||
|
|
||||||
|
# intellij project level runners. Add them explicitly if you would like to share.
|
||||||
|
.run
|
||||||
|
|||||||
155
Earthfile
Normal file
155
Earthfile
Normal file
@ -0,0 +1,155 @@
|
|||||||
|
VERSION 0.6
|
||||||
|
FROM golang:1.18
|
||||||
|
WORKDIR /go/storj
|
||||||
|
|
||||||
|
multinode-web:
|
||||||
|
FROM node:18
|
||||||
|
WORKDIR /build
|
||||||
|
COPY web/multinode .
|
||||||
|
RUN ./build.sh
|
||||||
|
SAVE ARTIFACT dist AS LOCAL web/storagenode/dist
|
||||||
|
|
||||||
|
wasm:
|
||||||
|
FROM storjlabs/ci
|
||||||
|
ENV TAG=earthly
|
||||||
|
COPY go.mod .
|
||||||
|
COPY go.sum .
|
||||||
|
COPY satellite/console/wasm satellite/console/wasm
|
||||||
|
COPY satellite/console/consolewasm satellite/console/consolewasm
|
||||||
|
COPY scripts scripts
|
||||||
|
RUN scripts/build-wasm.sh
|
||||||
|
SAVE ARTIFACT release/earthly/wasm wasm AS LOCAL web/satellite/static/wasm
|
||||||
|
|
||||||
|
storagenode-web:
|
||||||
|
FROM node:18
|
||||||
|
WORKDIR /build
|
||||||
|
COPY web/storagenode .
|
||||||
|
RUN ./build.sh
|
||||||
|
SAVE ARTIFACT dist AS LOCAL web/storagenode/dist
|
||||||
|
SAVE ARTIFACT static AS LOCAL web/storagenode/static
|
||||||
|
|
||||||
|
satellite-web:
|
||||||
|
FROM node:18
|
||||||
|
WORKDIR /build
|
||||||
|
COPY web/satellite .
|
||||||
|
RUN ./build.sh
|
||||||
|
COPY +wasm/wasm static/wasm
|
||||||
|
SAVE ARTIFACT dist AS LOCAL web/satellite/dist
|
||||||
|
SAVE ARTIFACT static AS LOCAL web/satellite/static
|
||||||
|
|
||||||
|
storagenode-bin:
|
||||||
|
COPY go.mod go.mod
|
||||||
|
COPY go.sum go.sum
|
||||||
|
COPY private private
|
||||||
|
COPY cmd/storagenode cmd/storagenode
|
||||||
|
COPY storage storage
|
||||||
|
COPY storagenode storagenode
|
||||||
|
COPY multinode multinode
|
||||||
|
COPY web/storagenode web/storagenode
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
--mount=type=cache,target=/go/pkg/mod \
|
||||||
|
go build -o release/earthly/storagenode storj.io/storj/cmd/storagenode
|
||||||
|
SAVE ARTIFACT release/earthly binaries AS LOCAL release/earthly
|
||||||
|
|
||||||
|
build-binaries:
|
||||||
|
COPY . .
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
--mount=type=cache,target=/go/pkg/mod \
|
||||||
|
go build -o release/earthly/satellite storj.io/storj/cmd/satellite
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
--mount=type=cache,target=/go/pkg/mod \
|
||||||
|
go build -o release/earthly/uplink storj.io/storj/cmd/uplink
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
--mount=type=cache,target=/go/pkg/mod \
|
||||||
|
go build -o release/earthly/identity storj.io/storj/cmd/identity
|
||||||
|
SAVE ARTIFACT release/earthly binaries AS LOCAL release/earthly
|
||||||
|
|
||||||
|
build-storjup:
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
--mount=type=cache,target=/go/pkg/mod \
|
||||||
|
CGO_ENABLED=0 go install storj.io/storj-up@25258f9
|
||||||
|
SAVE ARTIFACT /go/bin binaries AS LOCAL dist/up
|
||||||
|
|
||||||
|
|
||||||
|
deploy-remote:
|
||||||
|
FROM ubuntu
|
||||||
|
RUN apt-get update && apt-get install -y git wget unzip
|
||||||
|
RUN cd /tmp && wget https://releases.hashicorp.com/nomad/1.3.5/nomad_1.3.5_linux_amd64.zip -O nomad.zip && unzip nomad.zip && mv nomad /usr/local/bin && rm nomad.zip
|
||||||
|
COPY +build-storjup/binaries /usr/local/bin
|
||||||
|
COPY .git .git
|
||||||
|
ARG TAG=$(git rev-parse --short HEAD)
|
||||||
|
ARG IMAGE=img.dev.storj.io/dev/storj
|
||||||
|
BUILD +build-tagged-image --TAG=$TAG --IMAGE=$IMAGE
|
||||||
|
ARG --required nomad
|
||||||
|
ARG --required ip
|
||||||
|
COPY scripts/deploy/deploy-nightly.sh .
|
||||||
|
ENV NOMAD_ADDR=$nomad
|
||||||
|
ENV IP=$ip
|
||||||
|
ENV IMAGE=$IMAGE
|
||||||
|
RUN --push ./deploy-nightly.sh
|
||||||
|
|
||||||
|
deploy-local:
|
||||||
|
COPY +build-storjup/binaries /usr/local/bin
|
||||||
|
COPY .git .git
|
||||||
|
ARG TAG=$(git rev-parse --short HEAD)
|
||||||
|
ARG IMAGE=img.dev.storj.io/dev/storj
|
||||||
|
BUILD +build-tagged-image --TAG=$TAG --IMAGE=$IMAGE
|
||||||
|
WORKDIR /opt/storj-up
|
||||||
|
RUN storj-up init db,minimal,edge
|
||||||
|
RUN storj-up image satellite-api,storagenode $IMAGE:$TAG
|
||||||
|
SAVE ARTIFACT /opt/storj-up/docker-compose.yaml compose AS LOCAL docker-compose.yaml
|
||||||
|
|
||||||
|
build-image:
|
||||||
|
FROM storjlabs/ci
|
||||||
|
COPY .git .git
|
||||||
|
ARG IMAGE=img.dev.storj.io/dev/storj
|
||||||
|
ARG TAG=$(git rev-parse --short HEAD)
|
||||||
|
BUILD +build-tagged-image --TAG=$TAG --IMAGE=$IMAGE
|
||||||
|
|
||||||
|
build-tagged-image:
|
||||||
|
ARG --required TAG
|
||||||
|
ARG --required IMAGE
|
||||||
|
FROM img.dev.storj.io/storjup/base:20221011-2
|
||||||
|
COPY +multinode-web/dist /var/lib/storj/storj/web/multinode/dist
|
||||||
|
COPY +satellite-web/dist /var/lib/storj/storj/web/satellite/dist
|
||||||
|
COPY +satellite-web/static /var/lib/storj/storj/web/satellite/static
|
||||||
|
COPY +storagenode-web/dist /var/lib/storj/storj/web/storagenode/dist
|
||||||
|
COPY +storagenode-web/static /var/lib/storj/storj/web/storagenode/static
|
||||||
|
COPY +build-binaries/binaries /var/lib/storj/go/bin/
|
||||||
|
COPY +storagenode-bin/binaries /var/lib/storj/go/bin/
|
||||||
|
COPY +build-storjup/binaries /var/lib/storj/go/bin/
|
||||||
|
SAVE IMAGE --push $IMAGE:$TAG $IMAGE:latest
|
||||||
|
|
||||||
|
run:
|
||||||
|
LOCALLY
|
||||||
|
RUN docker-compose up
|
||||||
|
|
||||||
|
test:
|
||||||
|
COPY . .
|
||||||
|
RUN go install github.com/mfridman/tparse@36f80740879e24ba6695649290a240c5908ffcbb
|
||||||
|
RUN mkdir build
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
--mount=type=cache,target=/go/pkg/mod \
|
||||||
|
go test -json ./... | tee build/tests.json
|
||||||
|
SAVE ARTIFACT build/tests.json AS LOCAL build/tests.json
|
||||||
|
|
||||||
|
integration:
|
||||||
|
COPY +build/storj-up /usr/local/bin/storj-up
|
||||||
|
COPY test/test.sh .
|
||||||
|
WITH DOCKER
|
||||||
|
RUN ./test.sh
|
||||||
|
END
|
||||||
|
|
||||||
|
check-format:
|
||||||
|
COPY . .
|
||||||
|
RUN mkdir build
|
||||||
|
RUN bash -c '[[ $(git status --short) == "" ]] || (echo "Before formatting, please commit all your work!!! (Formatter will format only last commit)" && exit -1)'
|
||||||
|
RUN git show --name-only --pretty=format: | grep ".go" | xargs --no-run-if-empty -n1 gofmt -s -w
|
||||||
|
RUN git diff > build/format.patch
|
||||||
|
SAVE ARTIFACT build/format.patch
|
||||||
|
|
||||||
|
format:
|
||||||
|
LOCALLY
|
||||||
|
COPY +check-format/format.patch build/format.patch
|
||||||
|
RUN git apply --allow-empty build/format.patch
|
||||||
|
RUN git status
|
||||||
7
scripts/deploy/deploy-nightly.sh
Executable file
7
scripts/deploy/deploy-nightly.sh
Executable file
@ -0,0 +1,7 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -ex
|
||||||
|
storj-up init nomad --name=core --ip=$IP minimal,gc
|
||||||
|
storj-up image satellite-api,storagenode,gc $IMAGE
|
||||||
|
storj-up persist storagenode,satellite-api,gc
|
||||||
|
storj-up env set satellite-api STORJ_DATABASE_OPTIONS_MIGRATION_UNSAFE=snapshot,testdata
|
||||||
|
nomad run storj.hcl
|
||||||
Loading…
Reference in New Issue
Block a user