From c8a7088b762fe56b462257bcf78da4016bf9eaef Mon Sep 17 00:00:00 2001 From: Derek Smith Date: Fri, 27 May 2022 23:07:17 -0400 Subject: [PATCH] Various updates Signed-off-by: Derek Smith --- defaults.sh | 36 ++--- deploy.sh | 283 +++++++++++++++++++-------------- deployment/backup_www.sh | 2 +- deployment/domain_init.sh | 2 +- deployment/generate_certs.sh | 2 +- deployment/prepare_vps_host.sh | 2 +- deployment/provision_lxc.sh | 10 +- deployment/provision_vps.sh | 2 - deployment/run_ddns.sh | 2 +- install.sh | 5 +- shared.sh | 20 +-- 11 files changed, 193 insertions(+), 173 deletions(-) diff --git a/defaults.sh b/defaults.sh index 42497db..b52cf6b 100644 --- a/defaults.sh +++ b/defaults.sh @@ -25,6 +25,7 @@ export DDNS_PASSWORD= # this is where the html is sourced from. export SITE_HTML_PATH= +export BTCPAY_ADDITIONAL_HOSTNAMES= # enter your AWS Access Key and Secret Access Key here. export AWS_ACCESS_KEY= @@ -78,29 +79,20 @@ export NEXTCLOUD_SPACE_GB=10 DEV_LXD_REMOTE="$(lxc remote get-default)" export DEV_LXD_REMOTE="$DEV_LXD_REMOTE" -#export SITE_TITLE= - -# we use this later when we create a VM, we annotate what git commit (from a tag) we used. -LATEST_GIT_TAG="$(git describe --abbrev=0)" -export LATEST_GIT_TAG="$LATEST_GIT_TAG" - +# first of all, if there are uncommited changes, we quit. You better stash or commit! +# Remote VPS instances are tagged with your current git HEAD so we know which code revision +# used when provisioning the VPS. LATEST_GIT_COMMIT="$(cat ./.git/refs/heads/master)" export LATEST_GIT_COMMIT="$LATEST_GIT_COMMIT" +# check if there are any uncommited changes. It's dangerous to instantiate VMs using +# code that hasn't been committed. +# if git update-index --refresh | grep -q "needs update"; then +# echo "ERROR: You have uncommited changes! Better stash your work with 'git stash'." +# exit 1 +# fi -# let's ensure all the tools are installed -if [ ! -f "$(which rsync)" ]; then - echo "ERROR: rsync is not installed. You may want to install your dependencies." - exit 1 -fi - -# shellcheck disable=1091 - -export LXD_DISK_TO_USE= - - -ENABLE_NGINX_CACHING=false - +ENABLE_NGINX_CACHING=true # TODO @@ -119,12 +111,6 @@ fi export SITE_PATH="$SITE_PATH" export BTC_CHAIN="$BTC_CHAIN" -# if we're running aws/public, we enable nginx caching since it's a public site. -if [ "$VPS_HOSTING_TARGET" = aws ]; then - # TODO the correct behavior is to be =true, but cookies aren't working right now. - ENABLE_NGINX_CACHING=true -fi - DEFAULT_DB_IMAGE="mariadb:10.6.5" export ENABLE_NGINX_CACHING="$ENABLE_NGINX_CACHING" diff --git a/deploy.sh b/deploy.sh index 5124943..01c1a28 100755 --- a/deploy.sh +++ b/deploy.sh @@ -6,7 +6,7 @@ cd "$(dirname "$0")" check_dependencies () { for cmd in "$@"; do if ! command -v "$cmd" >/dev/null 2>&1; then - echo "This script requires \"${cmd}\" to be installed. Please run 'sudo ~/sovereign-stack/install.sh'" + echo "This script requires \"${cmd}\" to be installed. Please run 'install.sh'." exit 1 fi done @@ -16,7 +16,6 @@ check_dependencies () { check_dependencies wait-for-it dig rsync sshfs lxc docker-machine duplicity # TODO remove dependency on Docker-machine. That's what we use to provision VM on 3rd party vendors. Looking for LXD endpoint. - MIGRATE_VPS=false DOMAIN_NAME= VPS_HOSTING_TARGET=lxd @@ -26,11 +25,9 @@ USER_RUN_RESTORE=false BTC_CHAIN=regtest UPDATE_BTCPAY=false RECONFIGURE_BTCPAY_SERVER=false -BTCPAY_ADDITIONAL_HOSTNAMES= -LXD_DISK_TO_USE= DEPLOY_BTCPAY_SERVER=false -REDEPLOY_STACK=false MACVLAN_INTERFACE= +LXD_DISK_TO_USE= # grab any modifications from the command line. for i in "$@"; do @@ -45,6 +42,10 @@ for i in "$@"; do USER_NO_BACKUP=true shift ;; + --domain=*) + DOMAIN_NAME="${i#*=}" + shift + ;; --update-btcpay) UPDATE_BTCPAY=true shift @@ -87,18 +88,37 @@ done export CLUSTERS_DIR="$HOME/ss-clusters" export CACHES_DIR="$HOME/ss-cache" export SSH_HOME="$HOME/.ssh" - +export DOMAIN_NAME="$DOMAIN_NAME" export REGISTRY_DOCKER_IMAGE="registry:2" -CURRENT_REMOTE="$(lxc remote get-default)" + +if [ "$VPS_HOSTING_TARGET" = lxd ]; then + CURRENT_REMOTE="$(lxc remote get-default)" +elif [ "$VPS_HOSTING_TARGET" = aws ]; then + CURRENT_REMOTE="docker-machine" +fi + export LXD_REMOTE_PATH="$CLUSTERS_DIR/$CURRENT_REMOTE" export CURRENT_REMOTE="$CURRENT_REMOTE" -mkdir -p "$CACHES_DIR" "$LXD_REMOTE_PATH" -CLUSTER_DEFINTION="$LXD_REMOTE_PATH/cluster_definition" -export CLUSTER_DEFINTION="$CLUSTER_DEFINTION" -if [ ! -f "$CLUSTER_DEFINTION" ]; then - # stub out a cluster_definition. +# if an authorized_keys file does not exist, we'll stub one out with the current user. +# add additional id_rsa.pub entries manually for more administrative logins. +if [ ! -f "$LXD_REMOTE_PATH/authorized_keys" ]; then + mkdir -p "u" + cat "$SSH_HOME/id_rsa.pub" >> "$LXD_REMOTE_PATH/authorized_keys" + echo "INFO: Sovereign Stack just stubbed out '$LXD_REMOTE_PATH/authorized_keys'. Go update it." + echo " Add ssh pubkeys for your various management machines, if any. We've stubbed it out" + echo " with your ssh pubkey at '$HOME/.ssh/id_rsa.pub'." + exit 1 +fi + +if [ "$VPS_HOSTING_TARGET" = lxd ]; then + mkdir -p "$CACHES_DIR" "$LXD_REMOTE_PATH" + CLUSTER_DEFINTION="$LXD_REMOTE_PATH/cluster_definition" + export CLUSTER_DEFINTION="$CLUSTER_DEFINTION" + + if [ ! -f "$CLUSTER_DEFINTION" ]; then + # stub out a cluster_definition. cat >"$CLUSTER_DEFINTION" <> "$LXD_REMOTE_PATH/authorized_keys" - echo "INFO: Sovereign Stack just stubbed out '$LXD_REMOTE_PATH/authorized_keys'. Go update it." - echo " Add ssh pubkeys for your various management machines, if any. We've stubbed it out" - echo " with your ssh pubkey at '$HOME/.ssh/id_rsa.pub'." - exit 1 -fi - - -######################################### -# check for the env file. Source it if there. -if [ -f "$CLUSTER_DEFINTION" ]; then + ######################################### + if [ ! -f "$CLUSTER_DEFINTION" ]; then + echo "ERROR: CLUSTER DEFINITION NOT PRESENT." + exit 1 + fi + source "$CLUSTER_DEFINTION" ###########################3 @@ -159,7 +170,7 @@ if [ -f "$CLUSTER_DEFINTION" ]; then # if the registry URL isn't defined, then we just use the upstream dockerhub. # recommended to run a registry cache on your management machine though. - if [ ! -z "$REGISTRY_URL" ]; then + if [ -n "$REGISTRY_URL" ]; then cat > "$LXD_REMOTE_PATH/registry.yml" <"$SITE_DEFINITION_PATH" <"$SITE_DEFINITION_PATH" <> "$SITE_PATH/debug.log" +echo "Successfull deployed '$DOMAIN_NAME' with git commit '$(cat ./.git/refs/heads/master)' VPS_HOSTING_TARGET=$VPS_HOSTING_TARGET;" >> "$SITE_PATH/debug.log" diff --git a/deployment/generate_certs.sh b/deployment/generate_certs.sh index d5b7590..92d82bc 100755 --- a/deployment/generate_certs.sh +++ b/deployment/generate_certs.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex # let's do a refresh of the certificates. Let's Encrypt will not run if it's not time. docker pull certbot/certbot:latest diff --git a/deployment/prepare_vps_host.sh b/deployment/prepare_vps_host.sh index df3c2c7..d05c763 100755 --- a/deployment/prepare_vps_host.sh +++ b/deployment/prepare_vps_host.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eu +set -exu # scan the remote machine and install it's identity in our SSH known_hosts file. ssh-keyscan -H -t ecdsa "$FQDN" >> "$SSH_HOME/known_hosts" diff --git a/deployment/provision_lxc.sh b/deployment/provision_lxc.sh index 12ae50f..f8c6fb5 100755 --- a/deployment/provision_lxc.sh +++ b/deployment/provision_lxc.sh @@ -58,7 +58,7 @@ function run_ddns { echo "" echo "SUCCESS: The DNS appears to be configured correctly." - echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow stale DNS records to expire." + echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow cached DNS records to expire." sleep "$DDNS_SLEEP_SECONDS"; break; fi @@ -69,11 +69,13 @@ function run_ddns { } # create the default storage pool if necessary -if ! lxc storage list --format csv | grep -q default; then +if ! lxc storage list --format csv | grep -q "sovereign-stack"; then if [ -n "$LXD_DISK_TO_USE" ]; then - lxc storage create default zfs source="$LXD_DISK_TO_USE" size="${ROOT_DISK_SIZE_GB}GB" + lxc storage create "sovereign-stack" zfs source="$LXD_DISK_TO_USE" + # size="${ROOT_DISK_SIZE_GB}GB" else - lxc storage create default zfs size="${ROOT_DISK_SIZE_GB}GB" + lxc storage create "sovereign-stack" zfs + # size="${ROOT_DISK_SIZE_GB}GB" fi fi diff --git a/deployment/provision_vps.sh b/deployment/provision_vps.sh index c4068bd..8c14817 100755 --- a/deployment/provision_vps.sh +++ b/deployment/provision_vps.sh @@ -47,7 +47,6 @@ if [ "$APP_TO_DEPLOY" = www ] || [ "$APP_TO_DEPLOY" = certonly ]; then --amazonec2-ami "$AWS_AMI_ID" \ --amazonec2-root-size "$ROOT_DISK_SIZE_GB" \ --amazonec2-instance-type "$WWW_INSTANCE_TYPE" \ - --engine-label tag="$LATEST_GIT_TAG" \ --engine-label commit="$LATEST_GIT_COMMIT" \ "$FQDN" @@ -63,7 +62,6 @@ elif [ "$APP_TO_DEPLOY" = btcpay ]; then --amazonec2-ami "$AWS_AMI_ID" \ --amazonec2-root-size "$ROOT_DISK_SIZE_GB" \ --amazonec2-instance-type "$BTCPAY_INSTANCE_TYPE" \ - --engine-label tag="$LATEST_GIT_TAG" \ --engine-label commit="$LATEST_GIT_COMMIT" \ "$FQDN" diff --git a/deployment/run_ddns.sh b/deployment/run_ddns.sh index 9d3b391..81852cc 100755 --- a/deployment/run_ddns.sh +++ b/deployment/run_ddns.sh @@ -29,7 +29,7 @@ while true; do echo "" echo "SUCCESS: The DNS appears to be configured correctly." - echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow stale DNS records to expire." + echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow cached DNS records to expire." sleep "$DDNS_SLEEP_SECONDS"; break; fi diff --git a/install.sh b/install.sh index e9cd17a..6b2e029 100755 --- a/install.sh +++ b/install.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex cd "$(dirname "$0")" cat ./certs/docker.gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg @@ -21,7 +21,7 @@ if ! snap list | grep -q lxd; then fi # let's ensure docker-machine is available. This is only temporary though. -curl -L https://github.com/docker/machine/releases/download/v0.16.2/docker-machine-$(uname -s)-$(uname -m) >/tmp/docker-machine && +curl -L "https://github.com/docker/machine/releases/download/v0.16.2/docker-machine-$(uname -s)-$(uname -m)" >/tmp/docker-machine && chmod +x /tmp/docker-machine && sudo cp /tmp/docker-machine /usr/local/bin/docker-machine @@ -50,6 +50,5 @@ fi sudo groupadd docker sudo usermod -aG docker "$USER" - # TODO CHECK IF EXISTS echo "alias ss-deploy='/home/$USER/sovereign-stack/deploy.sh \$@'" >> "$HOME/.bashrc" diff --git a/shared.sh b/shared.sh index b6e0ce3..eaefd4c 100755 --- a/shared.sh +++ b/shared.sh @@ -1,6 +1,12 @@ #!/bin/bash -set -eu +set -ex + +VALUE=${SITE_PATH:-} +if [ -z "$VALUE" ]; then + echo "ERROR: Your SITE_PATH is undefined. Did you specify the domain correctly?" + exit 1 +fi # check to see if the enf file exists. exist if not. if [ ! -d "$SITE_PATH" ]; then @@ -27,13 +33,7 @@ export UNIX_BACKUP_TIMESTAMP="$UNIX_BACKUP_TIMESTAMP" REMOTE_BACKUP_PATH="$REMOTE_HOME/backups/$APP_TO_DEPLOY/$BACKUP_TIMESTAMP" LOCAL_BACKUP_PATH="$SITE_PATH/backups/$APP_TO_DEPLOY/$BACKUP_TIMESTAMP" export LOCAL_BACKUP_PATH="$LOCAL_BACKUP_PATH" -BACKUP_PATH_CREATED=false -if [ ! -d "$LOCAL_BACKUP_PATH" ]; then - mkdir -p "$LOCAL_BACKUP_PATH" - BACKUP_PATH_CREATED=true -fi -export BACKUP_PATH_CREATED="$BACKUP_PATH_CREATED" mkdir -p "$SSHFS_PATH" # VALIDATE THE INPUT from the ENVFILE @@ -49,7 +49,6 @@ export NOSTR_FQDN="$NOSTR_HOSTNAME.$DOMAIN_NAME" export ADMIN_ACCOUNT_USERNAME="info" export CERTIFICATE_EMAIL_ADDRESS="$ADMIN_ACCOUNT_USERNAME@$DOMAIN_NAME" -#export MAIL_FROM="$SITE_TITLE <$CERTIFICATE_EMAIL_ADDRESS>" export REMOTE_CERT_BASE_DIR="$REMOTE_HOME/.certs" export REMOTE_CERT_DIR="$REMOTE_CERT_BASE_DIR/$FQDN" @@ -171,11 +170,6 @@ if [ -z "$DOMAIN_NAME" ]; then exit 1 fi -#if [ -z "$SITE_TITLE" ]; then -# echo "ERROR: Ensure SITE_TITLE is configured in your site_definition." -# exit 1 -#fi - if [ -z "$DEPLOY_BTCPPAY_SERVER" ]; then echo "ERROR: Ensure DEPLOY_BTCPPAY_SERVER is configured in your site_definition." exit 1