From 3b2a3645f8ec58189f04f3617d093c4ad3bb7f62 Mon Sep 17 00:00:00 2001 From: Derek Smith Date: Tue, 24 May 2022 14:19:26 -0400 Subject: [PATCH] Control and formatting updates. --- deploy.sh | 449 +++++++++++++++++++++++---------- deployment/backup_btcpay.sh | 2 +- deployment/backup_www.sh | 2 +- deployment/domain_init.sh | 27 +- deployment/generate_certs.sh | 3 +- deployment/go_btcpay.sh | 2 +- deployment/go_umbrel.sh | 4 +- deployment/prepare_vps_host.sh | 2 +- deployment/provision_lxc.sh | 2 +- deployment/restore_btcpay.sh | 2 +- deployment/run_btcpay_setup.sh | 4 +- deployment/stub_docker_yml.sh | 3 +- deployment/stub_nginxconf.sh | 3 +- management/registry_mirror.yml | 54 ++++ 14 files changed, 386 insertions(+), 173 deletions(-) create mode 100644 management/registry_mirror.yml diff --git a/deploy.sh b/deploy.sh index d28e088..5124943 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -exu +set -eux cd "$(dirname "$0")" check_dependencies () { @@ -13,13 +13,13 @@ check_dependencies () { } # Check system's dependencies -check_dependencies wait-for-it dig rsync sshfs lxc docker-machine +check_dependencies wait-for-it dig rsync sshfs lxc docker-machine duplicity # TODO remove dependency on Docker-machine. That's what we use to provision VM on 3rd party vendors. Looking for LXD endpoint. MIGRATE_VPS=false DOMAIN_NAME= -VPS_HOSTING_TARGET= +VPS_HOSTING_TARGET=lxd RUN_CERT_RENEWAL=true USER_NO_BACKUP=false USER_RUN_RESTORE=false @@ -32,7 +32,7 @@ DEPLOY_BTCPAY_SERVER=false REDEPLOY_STACK=false MACVLAN_INTERFACE= - +# grab any modifications from the command line. for i in "$@"; do case $i in --aws) @@ -41,6 +41,8 @@ for i in "$@"; do ;; --restore) USER_RUN_RESTORE=true + RUN_CERT_RENEWAL=false + USER_NO_BACKUP=true shift ;; --update-btcpay) @@ -55,10 +57,6 @@ for i in "$@"; do MIGRATE_VPS=true shift ;; - --storage-backend=*) - LXD_DISK_TO_USE="${i#*=}" - shift - ;; --no-cert-renew) RUN_CERT_RENEWAL=false shift @@ -71,6 +69,10 @@ for i in "$@"; do BTC_CHAIN=testnet shift ;; + --regtest) + BTC_CHAIN=regtest + shift + ;; --reconfigure-btcpay) RECONFIGURE_BTCPAY_SERVER=true shift @@ -81,146 +83,313 @@ for i in "$@"; do esac done +# set up our default paths. +export CLUSTERS_DIR="$HOME/ss-clusters" +export CACHES_DIR="$HOME/ss-cache" +export SSH_HOME="$HOME/.ssh" + +export REGISTRY_DOCKER_IMAGE="registry:2" +CURRENT_REMOTE="$(lxc remote get-default)" +export LXD_REMOTE_PATH="$CLUSTERS_DIR/$CURRENT_REMOTE" +export CURRENT_REMOTE="$CURRENT_REMOTE" + +mkdir -p "$CACHES_DIR" "$LXD_REMOTE_PATH" + +CLUSTER_DEFINTION="$LXD_REMOTE_PATH/cluster_definition" +export CLUSTER_DEFINTION="$CLUSTER_DEFINTION" +if [ ! -f "$CLUSTER_DEFINTION" ]; then + # stub out a cluster_definition. + cat >"$CLUSTER_DEFINTION" < +export REGISTRY_PASSWORD= + +export MACVLAN_INTERFACE="\$MACVLAN_INTERFACE" +export LXD_DISK_TO_USE="\$LXD_DISK_TO_USE" +export SITE_LIST="\$SITE_LIST" + +EOL + + chmod 0744 "$CLUSTER_DEFINTION" + echo "We stubbed out a '$CLUSTER_DEFINTION' file for you." + echo "Use this file to customize your cluster deployment;" + echo "Check out 'https://www.sovereign-stack.org/cluster-definition' for an example." + exit 1 +fi + + +# if an authorized_keys file does not exist, we'll stub one out with the current user. +# add additional id_rsa.pub entries manually for more administrative logins. +if [ ! -f "$LXD_REMOTE_PATH/authorized_keys" ]; then + cat "$SSH_HOME/id_rsa.pub" >> "$LXD_REMOTE_PATH/authorized_keys" + echo "INFO: Sovereign Stack just stubbed out '$LXD_REMOTE_PATH/authorized_keys'. Go update it." + echo " Add ssh pubkeys for your various management machines, if any. We've stubbed it out" + echo " with your ssh pubkey at '$HOME/.ssh/id_rsa.pub'." + exit 1 +fi + + +######################################### # check for the env file. Source it if there. -if [ -f "$(pwd)/env" ]; then - source "$(pwd)/env"; +if [ -f "$CLUSTER_DEFINTION" ]; then + source "$CLUSTER_DEFINTION" + + ###########################3 + # # This section is done to the management machine. We deploy a registry pull through cache on port 5000 + # if ! docker volume list | grep -q registry_data; then + # docker volume create registry_data + # fi + + # if the registry URL isn't defined, then we just use the upstream dockerhub. + # recommended to run a registry cache on your management machine though. + if [ ! -z "$REGISTRY_URL" ]; then + +cat > "$LXD_REMOTE_PATH/registry.yml" <"$SITE_DEFINITION_PATH" <> "$SITE_PATH/authorized_keys" -fi - ## This is a weird if clause since we need to LEFT-ALIGN the statement below. SSH_STRING="Host ${FQDN}" if ! grep -q "$SSH_STRING" "$SSH_HOME/config"; then @@ -43,25 +37,14 @@ if [ "$VPS_HOSTING_TARGET" = aws ]; then elif [ "$VPS_HOSTING_TARGET" = lxd ]; then ssh-keygen -f "$SSH_HOME/known_hosts" -R "$FQDN" - #check to ensure the MACVLAN interface has been set by the user - if [ -z "$MACVLAN_INTERFACE" ]; then - echo "ERROR: MACVLAN_INTERFACE has not been defined. Use '--macvlan-interface=eno1' for example." - exit 1 - fi - - # let's first check to ensure there's a cert.tar.gz. We need a valid cert for testing. - if [ ! -f "$SITE_PATH/certs.tar.gz" ]; then - echo "ERROR: We need a valid cert for testing." - exit 1 - fi - # if the machine doesn't exist, we create it. if ! lxc list --format csv | grep -q "$LXD_VM_NAME"; then export RUN_BACKUP=false # create a base image if needed and instantiate a VM. if [ -z "$MAC_ADDRESS_TO_PROVISION" ]; then - echo "ERROR: You MUST define a MAC Address for all your machines." + echo "ERROR: You MUST define a MAC Address for all your machines by setting WWW_MAC_ADDRESS, BTCPAY_MAC_ADDRESS, UMBREL_MAC_ADDRESS, in your site defintion." + echo "INFO: IMPORTANT! You MUST have DHCP Reservations for these MAC addresses. You also need static DNS entries." exit 1 fi @@ -72,10 +55,16 @@ elif [ "$VPS_HOSTING_TARGET" = lxd ]; then ./prepare_vps_host.sh fi +# if the local docker client isn't logged in, do so; +# this helps prevent docker pull errors since they throttle. +if [ ! -f "$HOME/.docker/config.json" ]; then + echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USERNAME" --password-stdin +fi # this tells our local docker client to target the remote endpoint via SSH export DOCKER_HOST="ssh://ubuntu@$FQDN" + # the following scripts take responsibility for the rest of the provisioning depending on the app you're deploying. if [ "$APP_TO_DEPLOY" = www ]; then ./go_www.sh diff --git a/deployment/generate_certs.sh b/deployment/generate_certs.sh index d1ad1cd..fcc256c 100755 --- a/deployment/generate_certs.sh +++ b/deployment/generate_certs.sh @@ -1,7 +1,6 @@ #!/bin/bash -set -exu -cd "$(dirname "$0")" +set -e if [ "$VPS_HOSTING_TARGET" = aws ]; then diff --git a/deployment/go_btcpay.sh b/deployment/go_btcpay.sh index 3c8bd6c..c6c4d9f 100755 --- a/deployment/go_btcpay.sh +++ b/deployment/go_btcpay.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -exu +set -eu cd "$(dirname "$0")" if [ "$RUN_BACKUP" = true ]; then diff --git a/deployment/go_umbrel.sh b/deployment/go_umbrel.sh index 7c2af8d..b66faa3 100755 --- a/deployment/go_umbrel.sh +++ b/deployment/go_umbrel.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -exu +set -eu ssh "$FQDN" " set -x @@ -32,7 +32,7 @@ ssh "$FQDN" " # Start services back up. ssh "$FQDN" " - set -ex + set -e cd /home/ubuntu/umbrel git config pull.rebase true diff --git a/deployment/prepare_vps_host.sh b/deployment/prepare_vps_host.sh index d05c763..df3c2c7 100755 --- a/deployment/prepare_vps_host.sh +++ b/deployment/prepare_vps_host.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -exu +set -eu # scan the remote machine and install it's identity in our SSH known_hosts file. ssh-keyscan -H -t ecdsa "$FQDN" >> "$SSH_HOME/known_hosts" diff --git a/deployment/provision_lxc.sh b/deployment/provision_lxc.sh index 75b64f1..8eb73aa 100755 --- a/deployment/provision_lxc.sh +++ b/deployment/provision_lxc.sh @@ -148,4 +148,4 @@ wait_for_lxc_ip "$LXD_VM_NAME" run_ddns # remove any existing SSH identities for the host, then add it back. -ssh-keygen -R "$IP_V4_ADDRESS" \ No newline at end of file +ssh-keygen -R "$IP_V4_ADDRESS" diff --git a/deployment/restore_btcpay.sh b/deployment/restore_btcpay.sh index 74e5d29..c04c514 100755 --- a/deployment/restore_btcpay.sh +++ b/deployment/restore_btcpay.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -exu +u # this scripts ASSUMES services have already been taken down. diff --git a/deployment/run_btcpay_setup.sh b/deployment/run_btcpay_setup.sh index ec71977..0a956f6 100755 --- a/deployment/run_btcpay_setup.sh +++ b/deployment/run_btcpay_setup.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -e # export BTCPAY_FASTSYNC_ARCHIVE_FILENAME="utxo-snapshot-bitcoin-testnet-1445586.tar" @@ -11,7 +11,7 @@ set -ex cat > "$SITE_PATH/btcpay.sh" <>"$DOCKER_YAML_PATH" <>"$NGINX_CONF_PATH" <