From f8df46124952db90c67436ba7f3965a5bc625ca8 Mon Sep 17 00:00:00 2001 From: Ivan Fraixedes Date: Tue, 7 May 2019 23:20:04 +0200 Subject: [PATCH] v3-1660: Create a test that breaks when satellite config file has changed (#1898) * scripts: check for satellite configuration changes Create a simple script that checks if the satellite configuration has differed with the last changes. * Makefile: add target to check satellite cfg changes Add a Makefile target which executes the script that checks if the last changes has made a change in the satellite configuration. * ci: add satellite cfg check on integration stage * FIXUP: use releae defaults rather than development ones * FIXUP: show the message when config differs & some cleanups * scripts: add script to update the satellite cfg lock Add a script for allowing to update the satellite configuration lock file and add Makefile target to run it. * scripts/testsdata: update satellite cfg lock Update the satellite configuration lock file with the last changes that satellite has suffered upstream. --- Jenkinsfile.public | 1 + Makefile | 9 ++ scripts/test-satellite-cfg-change.sh | 24 +++ scripts/testdata/satellite-config.yaml.lock | 162 ++++++++++++++++++++ scripts/update-satellite-cfg-lock.sh | 28 ++++ 5 files changed, 224 insertions(+) create mode 100755 scripts/test-satellite-cfg-change.sh create mode 100644 scripts/testdata/satellite-config.yaml.lock create mode 100755 scripts/update-satellite-cfg-lock.sh diff --git a/Jenkinsfile.public b/Jenkinsfile.public index e3a890936..dc3f7293d 100644 --- a/Jenkinsfile.public +++ b/Jenkinsfile.public @@ -91,6 +91,7 @@ pipeline { steps { sh 'make test-sim' + sh 'make test-satellite-cfg-change' } } } diff --git a/Makefile b/Makefile index 8bc60b5d2..3cc5bb879 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,15 @@ test-sim: ## Test source with storj-sim (jenkins) @echo "Running ${@}" @./scripts/test-sim.sh +.PHONY: test-satellite-cfg-change +test-satellite-cfg-change: ## Test if the satellite config file has changed + @echo "Running ${@}" + @cd scripts; ./test-satellite-cfg-change.sh + +.PHONY: test-update-satellite-cfg-lock +test-update-satellite-cfg-lock: ## Update the satellite config lock file + @cd scripts; ./update-satellite-cfg-lock.sh + .PHONY: test-certificate-signing test-certificate-signing: ## Test certificate signing service and storagenode setup (jenkins) @echo "Running ${@}" diff --git a/scripts/test-satellite-cfg-change.sh b/scripts/test-satellite-cfg-change.sh new file mode 100755 index 000000000..f2fbf6f40 --- /dev/null +++ b/scripts/test-satellite-cfg-change.sh @@ -0,0 +1,24 @@ +#!/bin/bash - + +# NOTE this script MUST BE EXECUTED from the same directory where it's located +# to always obtain the same paths in the satellite configuration file. + +set -uo pipefail + +TESTDATA_DIR="./testdata" +cleanup(){ + rm "$TESTDATA_DIR/config.yaml" +} +trap cleanup EXIT + +satellite --config-dir "$TESTDATA_DIR" --defaults release setup > /dev/null + + +diff "$TESTDATA_DIR/satellite-config.yaml.lock" "$TESTDATA_DIR/config.yaml" +if [[ $? != 0 ]]; then + echo + echo "NOTIFY the Devops and PM when this test fails so they can plan for changing it in the release process before fixing it to merge your PR." + echo "Once you have notified them you can update the lock file through another Makefile target" + echo + exit 1 +fi diff --git a/scripts/testdata/satellite-config.yaml.lock b/scripts/testdata/satellite-config.yaml.lock new file mode 100644 index 000000000..e74e51650 --- /dev/null +++ b/scripts/testdata/satellite-config.yaml.lock @@ -0,0 +1,162 @@ +# how frequently segments are audited +audit.interval: 30s +# max number of times to attempt updating a statdb batch +audit.max-retries-stat-db: 3 +# the minimum acceptable bytes that storage nodes can transfer per second to the satellite +audit.min-bytes-per-second: 128 B +# how frequently checker should audit segments +checker.interval: 30s +# server address of the graphql api gateway and frontend app +console.address: "127.0.0.1:8081" +# auth token needed for access to registration token creation endpoint +console.auth-token: "" +# external endpoint of the satellite if hosted +console.external-address: "" +# path to static resources +console.static-dir: "" +# satellite database connection string +database: "postgres://" +# address to listen on for debug endpoints +debug.addr: "127.0.0.1:0" +# If set, a path to write a process trace SVG to +debug.trace-out: "" +# determines which set of configuration defaults to use. can either be 'dev' or 'release' +defaults: "release" +# the interval at which the satellite attempts to find new nodes via random node ID lookups +discovery.discovery-interval: 1s +# the interval at which the the graveyard tries to resurrect nodes +discovery.graveyard-interval: 30s +# the interval at which the cache refreshes itself in seconds +discovery.refresh-interval: 1s +# the amount of nodes refreshed at each interval +discovery.refresh-limit: 100 +# help for setup +help: false +# path to the certificate chain for this identity +identity.cert-path: "/root/.local/share/storj/identity/satellite/identity.cert" +# path to the private key for this identity +identity.key-path: "/root/.local/share/storj/identity/satellite/identity.key" +# alpha is a system wide concurrency parameter +kademlia.alpha: 5 +# the Kademlia node to bootstrap against +kademlia.bootstrap-addr: "bootstrap.storj.io:8888" +# the base interval to wait when retrying bootstrap +kademlia.bootstrap-backoff-base: 1s +# the maximum amount of time to wait when retrying bootstrap +kademlia.bootstrap-backoff-max: 30s +# size of each Kademlia bucket +kademlia.bucket-size: 20 +# the path for storage node db services to be created on +kademlia.db-path: "testdata/kademlia" +# the public address of the Kademlia node, useful for nodes behind NAT +kademlia.external-address: "" +# operator email address +kademlia.operator.email: "" +# operator wallet adress +kademlia.operator.wallet: "" +# size of Kademlia replacement cache +kademlia.replacement-cache-size: 5 +# if true, log function filename and line number +log.caller: false +# if true, set logging to development mode +log.development: false +# configures log encoding. can either be 'console' or 'json' +log.encoding: "console" +# the minimum log level to log +log.level: warn +# can be stdout, stderr, or a filename +log.output: "stderr" +# if true, log stack traces +log.stack: false +# smtp authentication type +mail.auth-type: "login" +# oauth2 app's client id +mail.client-id: "" +# oauth2 app's client secret +mail.client-secret: "" +# sender email address +mail.from: "" +# plain/login auth user login +mail.login: "" +# plain/login auth user password +mail.password: "" +# refresh token used to retrieve new access token +mail.refresh-token: "" +# smtp server address +mail.smtp-server-address: "" +# path to email templates source +mail.template-path: "" +# uri which is used when retrieving new access token +mail.token-uri: "" +# lifespan of bandwidth agreements in days +metainfo.bw-expiration: 45 +# the database connection string to use +metainfo.database-url: "postgres://" +# maximum inline segment size +metainfo.max-inline-segment-size: 8.0 KB +# minimum remote segment size +metainfo.min-remote-segment-size: 1.2 KiB +# toggle flag if overlay is enabled +metainfo.overlay: true +# address to send telemetry to +metrics.addr: "collectora.storj.io:9000" +# application name for telemetry identification +metrics.app: "satellite" +# application suffix +metrics.app-suffix: "-dev" +# how frequently to send up telemetry +metrics.interval: 1m0s +# path to log for oom notices +monkit.hw.oomlog: "/var/log/kern.log" +# the number of times a node has been audited to not be considered a New Node +overlay.node.audit-count: 500 +# a node's ratio of successful audits +overlay.node.audit-success-ratio: 0.4 +# the minimum node software version for node selection queries +overlay.node.minimum-version: "" +# the percentage of new nodes allowed per request +overlay.node.new-node-percentage: 0.05 +# the amount of time without seeing a node before its considered offline +overlay.node.online-window: 1h0m0s +# the number of times a node's uptime has been checked to not be considered a New Node +overlay.node.uptime-count: 500 +# a node's ratio of being up/online vs. down/offline +overlay.node.uptime-ratio: 0.9 +# how frequently checker should audit segments +repairer.interval: 1h0m0s +# maximum buffer memory (in bytes) to be allocated for read buffers +repairer.max-buffer-mem: 4.0 MB +# maximum segments that can be repaired concurrently +repairer.max-repair: 5 +# time limit for uploading repaired pieces to new storage nodes +repairer.timeout: 10m0s +# option for deleting tallies after they are rolled up +rollup.delete-tallies: false +# how frequently rollup should run +rollup.interval: 24h0m0s +# the bandwidth and storage usage limit for the alpha release +rollup.max-alpha-usage: 25.0 GB +# public address to listen on +server.address: ":7777" +# if true, client leaves may contain the most recent certificate revocation for the current certificate +server.extensions.revocation: true +# if true, client leaves must contain a valid "signed certificate extension" (NB: verified against certs in the peer ca whitelist; i.e. if true, a whitelist must be provided) +server.extensions.whitelist-signed-leaf: false +# path to the CA cert whitelist (peer identities must be signed by one these to be verified). this will override the default peer whitelist +server.peer-ca-whitelist-path: "" +# identity version(s) the server will be allowed to talk to +server.peer-id-versions: "latest" +# private address to listen on +server.private-address: "127.0.0.1:7778" +# url for revocation database (e.g. bolt://some.db OR redis://127.0.0.1:6378?db=2&password=abc123) +server.revocation-dburl: "bolt://testdata/revocations.db" +# if true, uses peer ca whitelist checking +server.use-peer-ca-whitelist: false +# how frequently the tally service should run +tally.interval: 1h0m0s +# Interval to check the version +version.check-interval: 15m0s +# Request timeout for version checks +version.request-timeout: 1m0s +# server address to check its version against +version.server-address: "https://version.alpha.storj.io" diff --git a/scripts/update-satellite-cfg-lock.sh b/scripts/update-satellite-cfg-lock.sh new file mode 100755 index 000000000..1a4b7aba7 --- /dev/null +++ b/scripts/update-satellite-cfg-lock.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# NOTE this script MUST BE EXECUTED from the same directory where it's located +# to always obtain the same paths in the satellite configuration file. + +set -ueo pipefail + +read -p "Have you warned the DevOps Team before updating this file? " -n 1 -r +echo # (optional) move to a new line +if [[ ! $REPLY =~ ^[Yy]$ ]] +then + echo operation aborted!!! + exit 1 +fi + +#setup tmpdir for testfiles and cleanup +TMP_DIR=$(mktemp -d -t update-satellite-cfg-lock-XXXXX) +cleanup(){ + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +go build -o "$TMP_DIR/satellite" "../cmd/satellite" + +PATH=$TMP_DIR:$PATH +TESTDATA_DIR="./testdata" +satellite --config-dir "$TESTDATA_DIR" --defaults release setup > /dev/null +mv "$TESTDATA_DIR/config.yaml" "$TESTDATA_DIR/satellite-config.yaml.lock"