From 0a78e741ec62437d97915c92e578a751b010e8ed Mon Sep 17 00:00:00 2001 From: Matt Robinson Date: Fri, 31 Aug 2018 11:21:44 -0400 Subject: [PATCH] Build and upload binaries (#296) * First pass at building binaries * Finished building images * Reworked some bits to save binaries in a better place. * First pass at uploading binaries * Builds for freebsd fail currently * Add ignore file for docker builds to not invalidate the cache as often * Docker image for 1.11-alpine exists now * Name windows binaries correctly * The makefile is used in tests. * Take a swing at updating the tests for 1.11 * Switch to something with glibc so the race detector works. * Remove unused .PHONY targets --- .dockerignore | 4 + .gitignore | 4 + Jenkinsfile | 7 ++ Makefile | 166 +++++++++++++++++++++++-------------- cmd/satellite/Dockerfile | 21 +++-- cmd/storagenode/Dockerfile | 18 ++-- cmd/uplink/Dockerfile | 23 +++-- docker-compose.yaml | 4 +- test/Dockerfile | 5 +- 9 files changed, 166 insertions(+), 86 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..5402cf0f0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +/*.swp +/satellite_* +/storagenode_* +/uplink_* diff --git a/.gitignore b/.gitignore index 46fdb6094..9ae52e3ab 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ protos/google/* *.coverprofile /release/ +*.swp +satellite_* +storagenode_* +uplink_* diff --git a/Jenkinsfile b/Jenkinsfile index 5d97f7120..944089c25 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,6 +14,12 @@ node('node') { echo "Current build result: ${currentBuild.result}" } + stage('Build Binaries') { + sh 'make binaries' + + echo "Current build result: ${currentBuild.result}" + } + stage('Push Images') { if (env.BRANCH_NAME == "master") { echo 'Push to Repo' @@ -27,6 +33,7 @@ node('node') { /* This should only deploy to staging if the branch is master */ stage('Deploy') { sh 'make deploy' + sh 'make binaries-upload' echo "Current build result: ${currentBuild.result}" } } diff --git a/Makefile b/Makefile index 452eebf78..4b819b8a3 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ -.PHONY: test lint proto check-copyrights build-dev-deps release release-osx release-windows release-linux +.PHONY: test lint proto check-copyrights build-dev-deps -GO_VERSION ?= 1.10 +GO_VERSION ?= 1.11 +GOOS ?= linux +GOARCH ?= amd64 COMPOSE_PROJECT_NAME := ${TAG}-$(shell git rev-parse --abbrev-ref HEAD) BRANCH := $(shell git rev-parse --abbrev-ref HEAD) ifeq (${BRANCH},master) @@ -9,7 +11,17 @@ TAG := $(shell git rev-parse --short HEAD)-go${GO_VERSION} else TAG := $(shell git rev-parse --short HEAD)-${BRANCH}-go${GO_VERSION} endif +CUSTOMTAG ?= +FILEEXT := +ifeq (${GOOS},windows) +FILEEXT := .exe +endif + +DOCKER_BUILD := docker build \ + --build-arg GO_VERSION=${GO_VERSION} \ + --build-arg GOOS=${GOOS} \ + --build-arg GOARCH=${GOARCH} # currently disabled linters: # gofmt # enable after switch to go1.11 @@ -83,42 +95,10 @@ test: lint gover @echo done -build-binaries: - docker build -t satellite . - -run-satellite: - docker network create test-net - - docker run -d \ - --name redis \ - --network test-net \ - -p 127.0.0.1:6379:6379 \ - redis - - docker run -d \ - --name=satellite \ - --network test-net \ - -p 127.0.0.1:8080:8080 \ - -e REDIS_ADDRESS=redis:6379 \ - -e REDIS_PASSWORD="" \ - -e REDIS_DB=1 \ - -e OVERLAY_PORT=7070 \ - satellite - test-captplanet: @echo "Running ${@}" @./scripts/test-captplanet.sh -clean-local: - # cleanup satellite - docker stop satellite || true - docker rm satellite || true - # cleanup redis - docker stop redis || true - docker rm redis || true - # cleanup docker network - docker network rm test-net || true - test-docker: docker-compose up -d --remove-orphans test docker-compose run test make test @@ -126,18 +106,18 @@ test-docker: test-docker-clean: -docker-compose down --rmi all -images: satellite-image storage-node-image uplink-image +images: satellite-image storagenode-image uplink-image echo Built version: ${TAG} .PHONY: satellite-image satellite-image: - docker build --build-arg GO_VERSION=${GO_VERSION} -t storjlabs/satellite:${TAG} -f cmd/satellite/Dockerfile . -.PHONY: storage-node-image -storage-node-image: - docker build --build-arg GO_VERSION=${GO_VERSION} -t storjlabs/storage-node:${TAG} -f cmd/storagenode/Dockerfile . + ${DOCKER_BUILD} -t storjlabs/satellite:${TAG}${CUSTOMTAG} -f cmd/satellite/Dockerfile . +.PHONY: storagenode-image +storagenode-image: + ${DOCKER_BUILD} -t storjlabs/storagenode:${TAG}${CUSTOMTAG} -f cmd/storagenode/Dockerfile . .PHONY: uplink-image uplink-image: - docker build --build-arg GO_VERSION=${GO_VERSION} -t storjlabs/uplink:${TAG} -f cmd/uplink/Dockerfile . + ${DOCKER_BUILD} -t storjlabs/uplink:${TAG}${CUSTOMTAG} -f cmd/uplink/Dockerfile . .PHONY: all-in-one all-in-one: @@ -145,17 +125,17 @@ all-in-one: $(MAKE) images -j 3 \ && export VERSION="${TAG}"; \ fi \ - && docker-compose up -d storage-node \ + && docker-compose up -d storagenode \ && scripts/fix-mock-overlay \ - && docker-compose up storage-node satellite uplink + && docker-compose up storagenode satellite uplink push-images: docker tag storjlabs/satellite:${TAG} storjlabs/satellite:latest docker push storjlabs/satellite:${TAG} docker push storjlabs/satellite:latest - docker tag storjlabs/storage-node:${TAG} storjlabs/storage-node:latest - docker push storjlabs/storage-node:${TAG} - docker push storjlabs/storage-node:latest + docker tag storjlabs/storagenode:${TAG} storjlabs/storagenode:latest + docker push storjlabs/storagenode:${TAG} + docker push storjlabs/storagenode:latest docker tag storjlabs/uplink:${TAG} storjlabs/uplink:latest docker push storjlabs/uplink:${TAG} docker push storjlabs/uplink:latest @@ -163,12 +143,12 @@ push-images: ifeq (${BRANCH},master) clean-images: -docker rmi storjlabs/satellite:${TAG} storjlabs/satellite:latest - -docker rmi storjlabs/storage-node:${TAG} storjlabs/storage-node:latest + -docker rmi storjlabs/storagenode:${TAG} storjlabs/storagenode:latest -docker rmi storjlabs/uplink:${TAG} storjlabs/uplink:latest else clean-images: -docker rmi storjlabs/satellite:${TAG} - -docker rmi storjlabs/storage-node:${TAG} + -docker rmi storjlabs/storagenode:${TAG} -docker rmi storjlabs/uplink:${TAG} endif @@ -180,22 +160,86 @@ install-deps: deploy: ./scripts/deploy.staging.sh satellite storjlabs/satellite:${TAG} for i in $(shell seq 1 60); do \ - ./scripts/deploy.staging.sh storage-node-$$i storjlabs/storage-node:${TAG}; \ + ./scripts/deploy.staging.sh storagenode-$$i storjlabs/storagenode:${TAG}; \ done -release-osx: - GOOS=darwin GOARCH=amd64 go build -o release/uplink-osx-amd64/uplink ./cmd/uplink - cd release; tar czvf uplink-osx-amd64.tar.gz uplink-osx-amd64 - rm -rf release/uplink-osx-amd64 +.PHONY: binary +binary: CUSTOMTAG = -${GOOS}-${GOARCH} +binary: + mkdir -p release/${TAG} + CUSTOMTAG=$(CUSTOMTAG) $(MAKE) $(COMPONENT)-image + cid=$$(docker create storjlabs/$(COMPONENT):${TAG}${CUSTOMTAG}) \ + && docker cp $$cid:/app/$(COMPONENT) release/${TAG}/$(COMPONENT)_${GOOS}_${GOARCH}${FILEEXT} \ + && docker rm $$cid + docker rmi storjlabs/$(COMPONENT):${TAG}${CUSTOMTAG} + rm -f release/${TAG}/${COMPONENT}_${GOOS}_${GOARCH}.zip + cd release/${TAG}; zip ${COMPONENT}_${GOOS}_${GOARCH}.zip ${COMPONENT}_${GOOS}_${GOARCH}${FILEEXT} + rm -f release/${TAG}/${COMPONENT}_${GOOS}_${GOARCH}${FILEEXT} -release-linux: - GOOS=linux GOARCH=amd64 go build -o release/uplink-linux-amd64/uplink ./cmd/uplink - cd release; tar czvf uplink-linux-amd64.tar.gz uplink-linux-amd64 - rm -rf release/uplink-linux-amd64 +# To update this section, modify and run the following: +# for c in satellite storagenode uplink; do \ +# for oa in "darwin amd64" "linux 386" \ +# "linux amd64" "windows 386" "windows amd64"; do \ +# echo "$c $oa"; done; done | while read -r c o a; do; \ +# printf ".PHONY: ${c}_${o}_${a}\n${c}_${o}_${a}:\n\tGOOS=${o} GOARCH=${a} COMPONENT=${c} \$(MAKE) binary\n"; \ +# done +.PHONY: satellite_darwin_amd64 +satellite_darwin_amd64: + GOOS=darwin GOARCH=amd64 COMPONENT=satellite $(MAKE) binary +.PHONY: satellite_linux_386 +satellite_linux_386: + GOOS=linux GOARCH=386 COMPONENT=satellite $(MAKE) binary +.PHONY: satellite_linux_amd64 +satellite_linux_amd64: + GOOS=linux GOARCH=amd64 COMPONENT=satellite $(MAKE) binary +.PHONY: satellite_windows_386 +satellite_windows_386: + GOOS=windows GOARCH=386 COMPONENT=satellite $(MAKE) binary +.PHONY: satellite_windows_amd64 +satellite_windows_amd64: + GOOS=windows GOARCH=amd64 COMPONENT=satellite $(MAKE) binary +.PHONY: storagenode_darwin_amd64 +storagenode_darwin_amd64: + GOOS=darwin GOARCH=amd64 COMPONENT=storagenode $(MAKE) binary +.PHONY: storagenode_linux_386 +storagenode_linux_386: + GOOS=linux GOARCH=386 COMPONENT=storagenode $(MAKE) binary +.PHONY: storagenode_linux_amd64 +storagenode_linux_amd64: + GOOS=linux GOARCH=amd64 COMPONENT=storagenode $(MAKE) binary +.PHONY: storagenode_windows_386 +storagenode_windows_386: + GOOS=windows GOARCH=386 COMPONENT=storagenode $(MAKE) binary +.PHONY: storagenode_windows_amd64 +storagenode_windows_amd64: + GOOS=windows GOARCH=amd64 COMPONENT=storagenode $(MAKE) binary +.PHONY: uplink_darwin_amd64 +uplink_darwin_amd64: + GOOS=darwin GOARCH=amd64 COMPONENT=uplink $(MAKE) binary +.PHONY: uplink_linux_386 +uplink_linux_386: + GOOS=linux GOARCH=386 COMPONENT=uplink $(MAKE) binary +.PHONY: uplink_linux_amd64 +uplink_linux_amd64: + GOOS=linux GOARCH=amd64 COMPONENT=uplink $(MAKE) binary +.PHONY: uplink_windows_386 +uplink_windows_386: + GOOS=windows GOARCH=386 COMPONENT=uplink $(MAKE) binary +.PHONY: uplink_windows_amd64 +uplink_windows_amd64: + GOOS=windows GOARCH=amd64 COMPONENT=uplink $(MAKE) binary -release-windows: - GOOS=windows GOARCH=amd64 go build -o release/uplink-windows-amd64/uplink ./cmd/uplink - cd release; zip uplink-windows-amd64.zip uplink-windows-amd64 - rm -rf release/uplink-windows-amd64 +# To update this section, modify and run the following: +# grep -Eo '^[a-z]*_[a-z]*_[a-z0-9]*' Makefile | tr '\n' ' ' +.PHONY: binaries +binaries: satellite_darwin_amd64 satellite_linux_386 satellite_linux_amd64 satellite_windows_386 satellite_windows_amd64 storagenode_darwin_amd64 storagenode_linux_386 storagenode_linux_amd64 storagenode_windows_386 storagenode_windows_amd64 uplink_darwin_amd64 uplink_linux_386 uplink_linux_amd64 uplink_windows_386 uplink_windows_amd64 -release: release-osx release-linux release-windows +.PHONY: binaries-upload +binaries-upload: + cd release; gsutil -m cp -r . gs://storj-v3-alpha-builds + +.PHONY: binaries-clean +binaries-clean: + rm -rf release + +clean: test-docker-clean binaries-clean clean-images diff --git a/cmd/satellite/Dockerfile b/cmd/satellite/Dockerfile index 48b26de40..70244ec0f 100644 --- a/cmd/satellite/Dockerfile +++ b/cmd/satellite/Dockerfile @@ -1,13 +1,18 @@ -ARG GO_VERSION=1.10 +ARG GO_VERSION=1.11 FROM golang:${GO_VERSION}-alpine AS build-env -RUN apk update && \ - apk upgrade && \ - apk add curl && \ - apk add git -RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh +ARG GOOS=linux +ARG GOARCH=amd64 +ARG LDFLAGS= +RUN apk add -U curl git musl-dev gcc COPY . /go/src/storj.io/storj -RUN cd /go/src/storj.io/storj && dep ensure -vendor-only -RUN cd /go/src/storj.io/storj/cmd/satellite && go build -o satellite +ENV GOOS=${GOOS} \ + GOARCH=${GOARCH} \ + LDFLAGS=${LDFLAGS} \ + GOARM=6 \ + CGO_ENABLED=0 \ + GO111MODULE=on +RUN cd /go/src/storj.io/storj/cmd/satellite \ + && go build -o satellite -ldflags "-s -w $LDFLAGS" # final stage diff --git a/cmd/storagenode/Dockerfile b/cmd/storagenode/Dockerfile index b93cb54dc..a92480688 100644 --- a/cmd/storagenode/Dockerfile +++ b/cmd/storagenode/Dockerfile @@ -1,11 +1,19 @@ -# build -ARG GO_VERSION=1.10 +ARG GO_VERSION=1.11 FROM golang:${GO_VERSION}-alpine AS build-env +ARG GOOS=linux +ARG GOARCH=amd64 +ARG LDFLAGS= RUN apk add -U curl git musl-dev gcc -RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh COPY . /go/src/storj.io/storj -RUN cd /go/src/storj.io/storj && dep ensure -vendor-only -RUN cd /go/src/storj.io/storj/cmd/storagenode && go build -o storagenode +ENV GOOS=${GOOS} \ + GOARCH=${GOARCH} \ + LDFLAGS=${LDFLAGS} \ + GOARM=6 \ + CGO_ENABLED=0 \ + GO111MODULE=on +RUN cd /go/src/storj.io/storj/cmd/storagenode \ + && go build -o storagenode -ldflags "-s -w $LDFLAGS" + # final stage FROM alpine diff --git a/cmd/uplink/Dockerfile b/cmd/uplink/Dockerfile index 02bce5f79..8a72066b2 100644 --- a/cmd/uplink/Dockerfile +++ b/cmd/uplink/Dockerfile @@ -1,12 +1,19 @@ -# build -ARG GO_VERSION=1.10 +ARG GO_VERSION=1.11 FROM golang:${GO_VERSION}-alpine AS build-env +ARG GOOS=linux +ARG GOARCH=amd64 +ARG LDFLAGS= RUN apk add -U curl git musl-dev gcc -RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh -WORKDIR /go/src/storj.io/storj -COPY . . -RUN dep ensure -vendor-only -RUN go build -o uplink cmd/uplink/*.go +COPY . /go/src/storj.io/storj +ENV GOOS=${GOOS} \ + GOARCH=${GOARCH} \ + LDFLAGS=${LDFLAGS} \ + GOARM=6 \ + CGO_ENABLED=0 \ + GO111MODULE=on +RUN cd /go/src/storj.io/storj/cmd/uplink \ + && go build -o uplink -ldflags "-s -w $LDFLAGS" + # final stage FROM alpine @@ -16,6 +23,6 @@ EXPOSE 7777 WORKDIR /app -COPY --from=build-env /go/src/storj.io/storj/uplink /app/ +COPY --from=build-env /go/src/storj.io/storj/cmd/uplink/uplink /app/ COPY cmd/uplink/entrypoint /entrypoint ENTRYPOINT ["/entrypoint"] diff --git a/docker-compose.yaml b/docker-compose.yaml index ecc1c409c..cfc4062f6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,8 +8,8 @@ services: test-redis: image: redis - storage-node: - image: storjlabs/storage-node:${VERSION} + storagenode: + image: storjlabs/storagenode:${VERSION} environment: - SATELLITE_ADDR=satellite:7777 links: diff --git a/test/Dockerfile b/test/Dockerfile index c7bcb3e94..b12bcaa6d 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,6 +1,7 @@ -FROM golang:1.10-alpine +FROM golang:1.10 -RUN apk -U add make git bash gcc musl-dev +RUN apt-get update \ + && apt-get install -y build-essential RUN cd / \ && rm -rf /go \