Build and upload binaries (#296)

* First pass at building binaries

* Finished building images

* Reworked some bits to save binaries in a better place.

* First pass at uploading binaries

* Builds for freebsd fail currently

* Add ignore file for docker builds to not invalidate the cache as often

* Docker image for 1.11-alpine exists now

* Name windows binaries correctly

* The makefile is used in tests.

* Take a swing at updating the tests for 1.11

* Switch to something with glibc so the race detector works.

* Remove unused .PHONY targets

.dockerignore

@@ -0,0 +1,4 @@
+/*.swp
+/satellite_*
+/storagenode_*
+/uplink_*

.gitignore

@@ -37,3 +37,7 @@ protos/google/*
 *.coverprofile
 
 /release/
+*.swp
+satellite_*
+storagenode_*
+uplink_*

Jenkinsfile

@@ -14,6 +14,12 @@ node('node') {
       echo "Current build result: ${currentBuild.result}"
     }
 
+    stage('Build Binaries') {
+      sh 'make binaries'
+
+      echo "Current build result: ${currentBuild.result}"
+    }
+
     stage('Push Images') {
       if (env.BRANCH_NAME == "master") {
         echo 'Push to Repo'
@@ -27,6 +33,7 @@ node('node') {
       /* This should only deploy to staging if the branch is master */
       stage('Deploy') {
         sh 'make deploy'
+        sh 'make binaries-upload'
         echo "Current build result: ${currentBuild.result}"
       }
     }

Makefile

@@ -1,7 +1,9 @@
-.PHONY: test lint proto check-copyrights build-dev-deps release release-osx release-windows release-linux
+.PHONY: test lint proto check-copyrights build-dev-deps
 
 
-GO_VERSION ?= 1.10
+GO_VERSION ?= 1.11
+GOOS ?= linux
+GOARCH ?= amd64
 COMPOSE_PROJECT_NAME := ${TAG}-$(shell git rev-parse --abbrev-ref HEAD)
 BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
 ifeq (${BRANCH},master)
@@ -9,7 +11,17 @@ TAG    	:= $(shell git rev-parse --short HEAD)-go${GO_VERSION}
 else
 TAG    	:= $(shell git rev-parse --short HEAD)-${BRANCH}-go${GO_VERSION}
 endif
+CUSTOMTAG ?=
 
+FILEEXT :=
+ifeq (${GOOS},windows)
+FILEEXT := .exe
+endif
+
+DOCKER_BUILD := docker build \
+	--build-arg GO_VERSION=${GO_VERSION} \
+	--build-arg GOOS=${GOOS} \
+	--build-arg GOARCH=${GOARCH}
 
 # currently disabled linters:
 #   gofmt               # enable after switch to go1.11
@@ -83,42 +95,10 @@ test: lint
 	gover
 	@echo done
 
-build-binaries:
-	docker build -t satellite .
-
-run-satellite:
-	docker network create test-net
-
-	docker run -d \
-		--name redis \
-		--network test-net \
-		-p 127.0.0.1:6379:6379 \
-		redis
-
-	docker run -d \
-		--name=satellite \
-		--network test-net \
-		-p 127.0.0.1:8080:8080 \
-		-e REDIS_ADDRESS=redis:6379 \
-		-e REDIS_PASSWORD="" \
-		-e REDIS_DB=1 \
-		-e OVERLAY_PORT=7070 \
-		satellite
-
 test-captplanet:
 	@echo "Running ${@}"
 	@./scripts/test-captplanet.sh
 
-clean-local:
-	# cleanup satellite
-	docker stop satellite || true
-	docker rm satellite || true
-	# cleanup redis
-	docker stop redis || true
-	docker rm redis || true
-	# cleanup docker network
-	docker network rm test-net || true
-
 test-docker:
 	docker-compose up -d --remove-orphans test
 	docker-compose run test make test
@@ -126,18 +106,18 @@ test-docker:
 test-docker-clean:
 	-docker-compose down --rmi all
 
-images: satellite-image storage-node-image uplink-image
+images: satellite-image storagenode-image uplink-image
 	echo Built version: ${TAG}
 
 .PHONY: satellite-image
 satellite-image:
-	docker build --build-arg GO_VERSION=${GO_VERSION} -t storjlabs/satellite:${TAG} -f cmd/satellite/Dockerfile .
+	${DOCKER_BUILD} -t storjlabs/satellite:${TAG}${CUSTOMTAG} -f cmd/satellite/Dockerfile .
-.PHONY: storage-node-image
+.PHONY: storagenode-image
-storage-node-image:
+storagenode-image:
-	docker build --build-arg GO_VERSION=${GO_VERSION} -t storjlabs/storage-node:${TAG} -f cmd/storagenode/Dockerfile .
+	${DOCKER_BUILD} -t storjlabs/storagenode:${TAG}${CUSTOMTAG} -f cmd/storagenode/Dockerfile .
 .PHONY: uplink-image
 uplink-image:
-	docker build --build-arg GO_VERSION=${GO_VERSION} -t storjlabs/uplink:${TAG} -f cmd/uplink/Dockerfile .
+	${DOCKER_BUILD} -t storjlabs/uplink:${TAG}${CUSTOMTAG} -f cmd/uplink/Dockerfile .
 
 .PHONY: all-in-one
 all-in-one:
@@ -145,17 +125,17 @@ all-in-one:
 		$(MAKE) images -j 3 \
 		&& export VERSION="${TAG}"; \
 	fi \
-	&& docker-compose up -d storage-node \
+	&& docker-compose up -d storagenode \
 	&& scripts/fix-mock-overlay \
-	&& docker-compose up storage-node satellite uplink
+	&& docker-compose up storagenode satellite uplink
 
 push-images:
 	docker tag storjlabs/satellite:${TAG} storjlabs/satellite:latest
 	docker push storjlabs/satellite:${TAG}
 	docker push storjlabs/satellite:latest
-	docker tag storjlabs/storage-node:${TAG} storjlabs/storage-node:latest
+	docker tag storjlabs/storagenode:${TAG} storjlabs/storagenode:latest
-	docker push storjlabs/storage-node:${TAG}
+	docker push storjlabs/storagenode:${TAG}
-	docker push storjlabs/storage-node:latest
+	docker push storjlabs/storagenode:latest
 	docker tag storjlabs/uplink:${TAG} storjlabs/uplink:latest
 	docker push storjlabs/uplink:${TAG}
 	docker push storjlabs/uplink:latest
@@ -163,12 +143,12 @@ push-images:
 ifeq (${BRANCH},master)
 clean-images:
 	-docker rmi storjlabs/satellite:${TAG} storjlabs/satellite:latest
-	-docker rmi storjlabs/storage-node:${TAG} storjlabs/storage-node:latest
+	-docker rmi storjlabs/storagenode:${TAG} storjlabs/storagenode:latest
 	-docker rmi storjlabs/uplink:${TAG} storjlabs/uplink:latest
 else
 clean-images:
 	-docker rmi storjlabs/satellite:${TAG}
-	-docker rmi storjlabs/storage-node:${TAG}
+	-docker rmi storjlabs/storagenode:${TAG}
 	-docker rmi storjlabs/uplink:${TAG}
 endif
 
@@ -180,22 +160,86 @@ install-deps:
 deploy:
 	./scripts/deploy.staging.sh satellite storjlabs/satellite:${TAG}
 	for i in $(shell seq 1 60); do \
-		./scripts/deploy.staging.sh storage-node-$$i storjlabs/storage-node:${TAG}; \
+		./scripts/deploy.staging.sh storagenode-$$i storjlabs/storagenode:${TAG}; \
 	done
 
-release-osx:
+.PHONY: binary
-	GOOS=darwin GOARCH=amd64 go build -o release/uplink-osx-amd64/uplink ./cmd/uplink
+binary: CUSTOMTAG = -${GOOS}-${GOARCH}
-	cd release; tar czvf uplink-osx-amd64.tar.gz uplink-osx-amd64
+binary:
-	rm -rf release/uplink-osx-amd64
+	mkdir -p release/${TAG}
+	CUSTOMTAG=$(CUSTOMTAG) $(MAKE) $(COMPONENT)-image
+	cid=$$(docker create storjlabs/$(COMPONENT):${TAG}${CUSTOMTAG}) \
+	&& docker cp $$cid:/app/$(COMPONENT) release/${TAG}/$(COMPONENT)_${GOOS}_${GOARCH}${FILEEXT} \
+    && docker rm $$cid
+	docker rmi storjlabs/$(COMPONENT):${TAG}${CUSTOMTAG}
+	rm -f release/${TAG}/${COMPONENT}_${GOOS}_${GOARCH}.zip
+	cd release/${TAG}; zip ${COMPONENT}_${GOOS}_${GOARCH}.zip ${COMPONENT}_${GOOS}_${GOARCH}${FILEEXT}
+	rm -f release/${TAG}/${COMPONENT}_${GOOS}_${GOARCH}${FILEEXT}
 
-release-linux:
+# To update this section, modify and run the following:
-	GOOS=linux GOARCH=amd64 go build -o release/uplink-linux-amd64/uplink ./cmd/uplink
+# for c in satellite storagenode uplink; do \
-	cd release; tar czvf uplink-linux-amd64.tar.gz uplink-linux-amd64
+# for oa in "darwin amd64" "linux 386" \
-	rm -rf release/uplink-linux-amd64
+# "linux amd64" "windows 386" "windows amd64"; do \
+# echo "$c $oa"; done; done | while read -r c o a; do; \
+# printf ".PHONY: ${c}_${o}_${a}\n${c}_${o}_${a}:\n\tGOOS=${o} GOARCH=${a} COMPONENT=${c} \$(MAKE) binary\n"; \
+# done
+.PHONY: satellite_darwin_amd64
+satellite_darwin_amd64:
+	GOOS=darwin GOARCH=amd64 COMPONENT=satellite $(MAKE) binary
+.PHONY: satellite_linux_386
+satellite_linux_386:
+	GOOS=linux GOARCH=386 COMPONENT=satellite $(MAKE) binary
+.PHONY: satellite_linux_amd64
+satellite_linux_amd64:
+	GOOS=linux GOARCH=amd64 COMPONENT=satellite $(MAKE) binary
+.PHONY: satellite_windows_386
+satellite_windows_386:
+	GOOS=windows GOARCH=386 COMPONENT=satellite $(MAKE) binary
+.PHONY: satellite_windows_amd64
+satellite_windows_amd64:
+	GOOS=windows GOARCH=amd64 COMPONENT=satellite $(MAKE) binary
+.PHONY: storagenode_darwin_amd64
+storagenode_darwin_amd64:
+	GOOS=darwin GOARCH=amd64 COMPONENT=storagenode $(MAKE) binary
+.PHONY: storagenode_linux_386
+storagenode_linux_386:
+	GOOS=linux GOARCH=386 COMPONENT=storagenode $(MAKE) binary
+.PHONY: storagenode_linux_amd64
+storagenode_linux_amd64:
+	GOOS=linux GOARCH=amd64 COMPONENT=storagenode $(MAKE) binary
+.PHONY: storagenode_windows_386
+storagenode_windows_386:
+	GOOS=windows GOARCH=386 COMPONENT=storagenode $(MAKE) binary
+.PHONY: storagenode_windows_amd64
+storagenode_windows_amd64:
+	GOOS=windows GOARCH=amd64 COMPONENT=storagenode $(MAKE) binary
+.PHONY: uplink_darwin_amd64
+uplink_darwin_amd64:
+	GOOS=darwin GOARCH=amd64 COMPONENT=uplink $(MAKE) binary
+.PHONY: uplink_linux_386
+uplink_linux_386:
+	GOOS=linux GOARCH=386 COMPONENT=uplink $(MAKE) binary
+.PHONY: uplink_linux_amd64
+uplink_linux_amd64:
+	GOOS=linux GOARCH=amd64 COMPONENT=uplink $(MAKE) binary
+.PHONY: uplink_windows_386
+uplink_windows_386:
+	GOOS=windows GOARCH=386 COMPONENT=uplink $(MAKE) binary
+.PHONY: uplink_windows_amd64
+uplink_windows_amd64:
+	GOOS=windows GOARCH=amd64 COMPONENT=uplink $(MAKE) binary
 
-release-windows:
+# To update this section, modify and run the following:
-	GOOS=windows GOARCH=amd64 go build -o release/uplink-windows-amd64/uplink ./cmd/uplink
+# grep -Eo '^[a-z]*_[a-z]*_[a-z0-9]*' Makefile | tr '\n' ' '
-	cd release; zip uplink-windows-amd64.zip uplink-windows-amd64
+.PHONY: binaries
-	rm -rf release/uplink-windows-amd64
+binaries: satellite_darwin_amd64 satellite_linux_386 satellite_linux_amd64 satellite_windows_386 satellite_windows_amd64 storagenode_darwin_amd64 storagenode_linux_386 storagenode_linux_amd64 storagenode_windows_386 storagenode_windows_amd64 uplink_darwin_amd64 uplink_linux_386 uplink_linux_amd64 uplink_windows_386 uplink_windows_amd64
 
-release: release-osx release-linux release-windows
+.PHONY: binaries-upload
+binaries-upload:
+	cd release; gsutil -m cp -r . gs://storj-v3-alpha-builds
+
+.PHONY: binaries-clean
+binaries-clean:
+	rm -rf release
+
+clean: test-docker-clean binaries-clean clean-images

cmd/satellite/Dockerfile

@@ -1,13 +1,18 @@
-ARG GO_VERSION=1.10
+ARG GO_VERSION=1.11
 FROM golang:${GO_VERSION}-alpine AS build-env
-RUN apk update && \
+ARG GOOS=linux
-    apk upgrade && \
+ARG GOARCH=amd64
-    apk add curl && \
+ARG LDFLAGS=
-    apk add git
+RUN apk add -U curl git musl-dev gcc
-RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
 COPY . /go/src/storj.io/storj
-RUN cd /go/src/storj.io/storj && dep ensure -vendor-only
+ENV GOOS=${GOOS} \
-RUN cd /go/src/storj.io/storj/cmd/satellite && go build -o satellite
+    GOARCH=${GOARCH} \
+    LDFLAGS=${LDFLAGS} \
+    GOARM=6 \
+    CGO_ENABLED=0 \
+    GO111MODULE=on
+RUN cd /go/src/storj.io/storj/cmd/satellite \
+ && go build -o satellite -ldflags "-s -w $LDFLAGS"
 
 
 # final stage

cmd/storagenode/Dockerfile

@@ -1,11 +1,19 @@
-# build
+ARG GO_VERSION=1.11
-ARG GO_VERSION=1.10
 FROM golang:${GO_VERSION}-alpine AS build-env
+ARG GOOS=linux
+ARG GOARCH=amd64
+ARG LDFLAGS=
 RUN apk add -U curl git musl-dev gcc
-RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
 COPY . /go/src/storj.io/storj
-RUN cd /go/src/storj.io/storj && dep ensure -vendor-only
+ENV GOOS=${GOOS} \
-RUN cd /go/src/storj.io/storj/cmd/storagenode && go build -o storagenode
+    GOARCH=${GOARCH} \
+    LDFLAGS=${LDFLAGS} \
+    GOARM=6 \
+    CGO_ENABLED=0 \
+    GO111MODULE=on
+RUN cd /go/src/storj.io/storj/cmd/storagenode \
+ && go build -o storagenode -ldflags "-s -w $LDFLAGS"
+
 
 # final stage
 FROM alpine

cmd/uplink/Dockerfile

@@ -1,12 +1,19 @@
-# build
+ARG GO_VERSION=1.11
-ARG GO_VERSION=1.10
 FROM golang:${GO_VERSION}-alpine AS build-env
+ARG GOOS=linux
+ARG GOARCH=amd64
+ARG LDFLAGS=
 RUN apk add -U curl git musl-dev gcc
-RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+COPY . /go/src/storj.io/storj
-WORKDIR /go/src/storj.io/storj
+ENV GOOS=${GOOS} \
-COPY . .
+    GOARCH=${GOARCH} \
-RUN dep ensure -vendor-only
+    LDFLAGS=${LDFLAGS} \
-RUN go build -o uplink cmd/uplink/*.go
+    GOARM=6 \
+    CGO_ENABLED=0 \
+    GO111MODULE=on
+RUN cd /go/src/storj.io/storj/cmd/uplink \
+ && go build -o uplink -ldflags "-s -w $LDFLAGS"
+
 
 # final stage
 FROM alpine
@@ -16,6 +23,6 @@ EXPOSE 7777
 
 
 WORKDIR /app
-COPY --from=build-env /go/src/storj.io/storj/uplink /app/
+COPY --from=build-env /go/src/storj.io/storj/cmd/uplink/uplink /app/
 COPY cmd/uplink/entrypoint /entrypoint
 ENTRYPOINT ["/entrypoint"]

docker-compose.yaml

@@ -8,8 +8,8 @@ services:
   test-redis:
     image: redis
 
-  storage-node:
+  storagenode:
-    image: storjlabs/storage-node:${VERSION}
+    image: storjlabs/storagenode:${VERSION}
     environment:
     - SATELLITE_ADDR=satellite:7777
     links:

test/Dockerfile

@@ -1,6 +1,7 @@
-FROM golang:1.10-alpine
+FROM golang:1.10
 
-RUN apk -U add make git bash gcc musl-dev
+RUN apt-get update \
+ && apt-get install -y build-essential
 
 RUN cd / \
  && rm -rf /go \