Various updates

Signed-off-by: Derek Smith <derek@farscapian.com>
This commit is contained in:
Derek Smith 2022-05-27 23:07:17 -04:00
parent 806794c308
commit c8a7088b76
Signed by: farscapian
GPG Key ID: 8F1CD799CCA516CC
11 changed files with 193 additions and 173 deletions

View File

@ -25,6 +25,7 @@ export DDNS_PASSWORD=
# this is where the html is sourced from.
export SITE_HTML_PATH=
export BTCPAY_ADDITIONAL_HOSTNAMES=
# enter your AWS Access Key and Secret Access Key here.
export AWS_ACCESS_KEY=
@ -78,29 +79,20 @@ export NEXTCLOUD_SPACE_GB=10
DEV_LXD_REMOTE="$(lxc remote get-default)"
export DEV_LXD_REMOTE="$DEV_LXD_REMOTE"
#export SITE_TITLE=
# we use this later when we create a VM, we annotate what git commit (from a tag) we used.
LATEST_GIT_TAG="$(git describe --abbrev=0)"
export LATEST_GIT_TAG="$LATEST_GIT_TAG"
# first of all, if there are uncommited changes, we quit. You better stash or commit!
# Remote VPS instances are tagged with your current git HEAD so we know which code revision
# used when provisioning the VPS.
LATEST_GIT_COMMIT="$(cat ./.git/refs/heads/master)"
export LATEST_GIT_COMMIT="$LATEST_GIT_COMMIT"
# check if there are any uncommited changes. It's dangerous to instantiate VMs using
# code that hasn't been committed.
# if git update-index --refresh | grep -q "needs update"; then
# echo "ERROR: You have uncommited changes! Better stash your work with 'git stash'."
# exit 1
# fi
# let's ensure all the tools are installed
if [ ! -f "$(which rsync)" ]; then
echo "ERROR: rsync is not installed. You may want to install your dependencies."
exit 1
fi
# shellcheck disable=1091
export LXD_DISK_TO_USE=
ENABLE_NGINX_CACHING=false
ENABLE_NGINX_CACHING=true
# TODO
@ -119,12 +111,6 @@ fi
export SITE_PATH="$SITE_PATH"
export BTC_CHAIN="$BTC_CHAIN"
# if we're running aws/public, we enable nginx caching since it's a public site.
if [ "$VPS_HOSTING_TARGET" = aws ]; then
# TODO the correct behavior is to be =true, but cookies aren't working right now.
ENABLE_NGINX_CACHING=true
fi
DEFAULT_DB_IMAGE="mariadb:10.6.5"
export ENABLE_NGINX_CACHING="$ENABLE_NGINX_CACHING"

249
deploy.sh
View File

@ -6,7 +6,7 @@ cd "$(dirname "$0")"
check_dependencies () {
for cmd in "$@"; do
if ! command -v "$cmd" >/dev/null 2>&1; then
echo "This script requires \"${cmd}\" to be installed. Please run 'sudo ~/sovereign-stack/install.sh'"
echo "This script requires \"${cmd}\" to be installed. Please run 'install.sh'."
exit 1
fi
done
@ -16,7 +16,6 @@ check_dependencies () {
check_dependencies wait-for-it dig rsync sshfs lxc docker-machine duplicity
# TODO remove dependency on Docker-machine. That's what we use to provision VM on 3rd party vendors. Looking for LXD endpoint.
MIGRATE_VPS=false
DOMAIN_NAME=
VPS_HOSTING_TARGET=lxd
@ -26,11 +25,9 @@ USER_RUN_RESTORE=false
BTC_CHAIN=regtest
UPDATE_BTCPAY=false
RECONFIGURE_BTCPAY_SERVER=false
BTCPAY_ADDITIONAL_HOSTNAMES=
LXD_DISK_TO_USE=
DEPLOY_BTCPAY_SERVER=false
REDEPLOY_STACK=false
MACVLAN_INTERFACE=
LXD_DISK_TO_USE=
# grab any modifications from the command line.
for i in "$@"; do
@ -45,6 +42,10 @@ for i in "$@"; do
USER_NO_BACKUP=true
shift
;;
--domain=*)
DOMAIN_NAME="${i#*=}"
shift
;;
--update-btcpay)
UPDATE_BTCPAY=true
shift
@ -87,16 +88,35 @@ done
export CLUSTERS_DIR="$HOME/ss-clusters"
export CACHES_DIR="$HOME/ss-cache"
export SSH_HOME="$HOME/.ssh"
export DOMAIN_NAME="$DOMAIN_NAME"
export REGISTRY_DOCKER_IMAGE="registry:2"
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
CURRENT_REMOTE="$(lxc remote get-default)"
elif [ "$VPS_HOSTING_TARGET" = aws ]; then
CURRENT_REMOTE="docker-machine"
fi
export LXD_REMOTE_PATH="$CLUSTERS_DIR/$CURRENT_REMOTE"
export CURRENT_REMOTE="$CURRENT_REMOTE"
mkdir -p "$CACHES_DIR" "$LXD_REMOTE_PATH"
# if an authorized_keys file does not exist, we'll stub one out with the current user.
# add additional id_rsa.pub entries manually for more administrative logins.
if [ ! -f "$LXD_REMOTE_PATH/authorized_keys" ]; then
mkdir -p "u"
cat "$SSH_HOME/id_rsa.pub" >> "$LXD_REMOTE_PATH/authorized_keys"
echo "INFO: Sovereign Stack just stubbed out '$LXD_REMOTE_PATH/authorized_keys'. Go update it."
echo " Add ssh pubkeys for your various management machines, if any. We've stubbed it out"
echo " with your ssh pubkey at '$HOME/.ssh/id_rsa.pub'."
exit 1
fi
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
mkdir -p "$CACHES_DIR" "$LXD_REMOTE_PATH"
CLUSTER_DEFINTION="$LXD_REMOTE_PATH/cluster_definition"
export CLUSTER_DEFINTION="$CLUSTER_DEFINTION"
if [ ! -f "$CLUSTER_DEFINTION" ]; then
# stub out a cluster_definition.
cat >"$CLUSTER_DEFINTION" <<EOL
@ -111,7 +131,7 @@ SITE_LIST="domain1.tld"
# REQUIRED - change the MACVLAN_INTERFACE to the host adapter that attaches to the SERVERS LAN segment/VLAN/subnet.
MACVLAN_INTERFACE="REQUIRED_CHANGE_ME"
#LXD_DISK_TO_USE=""
LXD_DISK_TO_USE=""
# Deploy a registry cache on your management machine.
DEPLOY_REGISTRY=true
@ -134,21 +154,12 @@ EOL
exit 1
fi
# if an authorized_keys file does not exist, we'll stub one out with the current user.
# add additional id_rsa.pub entries manually for more administrative logins.
if [ ! -f "$LXD_REMOTE_PATH/authorized_keys" ]; then
cat "$SSH_HOME/id_rsa.pub" >> "$LXD_REMOTE_PATH/authorized_keys"
echo "INFO: Sovereign Stack just stubbed out '$LXD_REMOTE_PATH/authorized_keys'. Go update it."
echo " Add ssh pubkeys for your various management machines, if any. We've stubbed it out"
echo " with your ssh pubkey at '$HOME/.ssh/id_rsa.pub'."
#########################################
if [ ! -f "$CLUSTER_DEFINTION" ]; then
echo "ERROR: CLUSTER DEFINITION NOT PRESENT."
exit 1
fi
#########################################
# check for the env file. Source it if there.
if [ -f "$CLUSTER_DEFINTION" ]; then
source "$CLUSTER_DEFINTION"
###########################3
@ -159,7 +170,7 @@ if [ -f "$CLUSTER_DEFINTION" ]; then
# if the registry URL isn't defined, then we just use the upstream dockerhub.
# recommended to run a registry cache on your management machine though.
if [ ! -z "$REGISTRY_URL" ]; then
if [ -n "$REGISTRY_URL" ]; then
cat > "$LXD_REMOTE_PATH/registry.yml" <<EOL
version: 0.1
@ -185,90 +196,14 @@ EOL
docker stack deploy -c management/registry_mirror.yml registry
fi
fi
else
echo "ERROR: CLUSTER DEFINITION NOT PRESENT."
exit 1
fi
# iterate through our site list as provided by operator from cluster_definition
for i in ${SITE_LIST//,/ }; do
export DOMAIN_NAME="$i"
source ./defaults.sh
if [ -f "$SITE_PATH/site_definition" ]; then
source "$SITE_PATH/site_definition"
else
# check to see if the enf file exists. exist if not.
SITE_DEFINITION_PATH="$SITE_PATH/site_definition"
if [ ! -f "$SITE_DEFINITION_PATH" ]; then
function new_pass {
apg -a 1 -M nc -n 3 -m 26 -E GHIJKLMNOPQRSTUVWXYZ | head -n1 | awk '{print $1;}'
}
# stub out a site_definition with new passwords.
cat >"$SITE_DEFINITION_PATH" <<EOL
#!/bin/bash
#export SITE_TITLE="Short Title of Project"
export DOMAIN_NAME="domain.tld"
# duplicitiy backup archive password
export DUPLICITY_BACKUP_PASSPHRASE="$(new_pass)"
# AWS only
#export DDNS_PASSWORD=
#export SMTP_PASSWORD=
## WWW
export DEPLOY_WWW_SERVER=true
# REQUIRED - CHANGE ME - RESERVE ME IN DHCP
export WWW_MAC_ADDRESS="CHANGE_ME"
# Deploy APPS to www
export DEPLOY_GHOST=true
export DEPLOY_NOSTR=false
export DEPLOY_NEXTCLOUD=true
export DEPLOY_ONION_SITE=false
export NOSTR_ACCOUNT_PUBKEY="CHANGE_ME"
# passwords for WWW apps
export GHOST_MYSQL_PASSWORD="$(new_pass)"
export GHOST_MYSQL_ROOT_PASSWORD="$(new_pass)"
export NEXTCLOUD_MYSQL_PASSWORD="$(new_pass)"
export GITEA_MYSQL_PASSWORD="$(new_pass)"
export NEXTCLOUD_MYSQL_ROOT_PASSWORD="$(new_pass)"
export GITEA_MYSQL_ROOT_PASSWORD="$(new_pass)"
## BTCPAY SERVER
export DEPLOY_BTCPAY_SERVER=false
# REQUIRED if DEPLOY_BTCPAY_SERVER=true
#export BTCPAY_MAC_ADDRESS="CHANGE_ME"
## BTCPAY Server
export DEPLOY_UMBREL_VPS=false
# REQUIRED if DEPLOY_UMBREL_VPS=true
#export UMBREL_MAC_ADDRESS="CHANGE_ME"
# CHAIN to DEPLOY; valid are 'testnet' and 'mainnet'
export BTC_CHAIN=regtest
EOL
chmod 0744 "$SITE_DEFINITION_PATH"
echo "INFO: we stubbed a new site_defintion for you at '$SITE_DEFINITION_PATH'. Go update it yo!"
exit 1
fi
exit 1
fi
function run_domain {
export VPS_HOSTING_TARGET="$VPS_HOSTING_TARGET"
export LXD_DISK_TO_USE="$LXD_DISK_TO_USE"
@ -279,19 +214,16 @@ fi
export MIGRATE_VPS="$MIGRATE_VPS"
export RECONFIGURE_BTCPAY_SERVER="$RECONFIGURE_BTCPAY_SERVER"
export MACVLAN_INTERFACE="$MACVLAN_INTERFACE"
export LXD_DISK_TO_USE="$LXD_DISK_TO_USE"
# # first of all, if there are uncommited changes, we quit. You better stash your work yo!
# if git update-index --refresh| grep -q "needs update"; then
# echo "ERROR: You have uncommited changes! Better stash your work with 'git stash'."
# exit 1
# fi
source ./defaults.sh
# iterate over all our server endpoints and provision them if needed.
# www
VPS_HOSTNAME=
for APP_TO_DEPLOY in btcpay www umbrel; do
FQDN=
export APP_TO_DEPLOY="$APP_TO_DEPLOY"
# shellcheck disable=SC1091
source ./shared.sh
@ -322,6 +254,14 @@ fi
fi
fi
# create the local packup path if it's not there!
BACKUP_PATH_CREATED=false
if [ ! -d "$LOCAL_BACKUP_PATH" ]; then
mkdir -p "$LOCAL_BACKUP_PATH"
BACKUP_PATH_CREATED=true
fi
export BACKUP_PATH_CREATED="$BACKUP_PATH_CREATED"
export MAC_ADDRESS_TO_PROVISION="$MAC_ADDRESS_TO_PROVISION"
export VPS_HOSTNAME="$VPS_HOSTNAME"
export FQDN="$VPS_HOSTNAME.$DOMAIN_NAME"
@ -391,5 +331,106 @@ fi
fi
done
}
function stub_site_definition {
source ./defaults.sh
if [ -f "$SITE_PATH/site_definition" ]; then
source "$SITE_PATH/site_definition"
else
# check to see if the enf file exists. exist if not.
SITE_DEFINITION_PATH="$SITE_PATH/site_definition"
if [ ! -f "$SITE_DEFINITION_PATH" ]; then
# stub out a site_definition with new passwords.
cat >"$SITE_DEFINITION_PATH" <<EOL
#!/bin/bash
# Set the domain name for the identity site.
export DOMAIN_NAME="domain.tld"
# duplicitiy backup archive password
export DUPLICITY_BACKUP_PASSPHRASE="$(new_pass)"
# AWS only
#export DDNS_PASSWORD=
#export SMTP_PASSWORD=
## WWW
export DEPLOY_WWW_SERVER=true
# REQUIRED - CHANGE ME - RESERVE ME IN DHCP
export WWW_MAC_ADDRESS="CHANGE_ME"
# Deploy APPS to www
export DEPLOY_GHOST=true
export DEPLOY_NEXTCLOUD=true
export DEPLOY_NOSTR=false
# set if NOSTR_ACCOUNT_PUBKEY=true
export NOSTR_ACCOUNT_PUBKEY="CHANGE_ME"
export DEPLOY_GITEA=false
export DEPLOY_ONION_SITE=false
# passwords for WWW apps
## GHOST
export GHOST_MYSQL_PASSWORD="$(new_pass)"
export GHOST_MYSQL_ROOT_PASSWORD="$(new_pass)"
## NEXTCLOUD
export NEXTCLOUD_MYSQL_PASSWORD="$(new_pass)"
export NEXTCLOUD_MYSQL_ROOT_PASSWORD="$(new_pass)"
## GITEA
export GITEA_MYSQL_PASSWORD="$(new_pass)"
export GITEA_MYSQL_ROOT_PASSWORD="$(new_pass)"
## BTCPAY SERVER; if true, then a BTCPay server is deployed.
export DEPLOY_BTCPAY_SERVER=false
# REQUIRED if DEPLOY_BTCPAY_SERVER=true
#export BTCPAY_MAC_ADDRESS="CHANGE_ME"
## BTCPAY Server
export DEPLOY_UMBREL_VPS=false
# REQUIRED if DEPLOY_UMBREL_VPS=true
#export UMBREL_MAC_ADDRESS="CHANGE_ME"
# CHAIN to DEPLOY; valid are 'testnet' and 'mainnet'
export BTC_CHAIN=regtest
EOL
chmod 0744 "$SITE_DEFINITION_PATH"
echo "INFO: we stubbed a new site_defintion for you at '$SITE_DEFINITION_PATH'. Go update it yo!"
exit 1
fi
fi
}
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
# iterate through our site list as provided by operator from cluster_definition
for i in ${SITE_LIST//,/ }; do
export DOMAIN_NAME="$i"
stub_site_definition
# run the logic for a domain deployment.
run_domain
done
elif [ "$VPS_HOSTING_TARGET" = aws ]; then
stub_site_definition
# if we're on AWS, we can just provision each system separately.
run_domain
fi

View File

@ -9,7 +9,7 @@ cd "$(dirname "$0")"
# maybe something like https://superuser.com/questions/616182/how-to-mount-local-directory-to-remote-like-sshfs
# step 1: run duplicity on the remote system to backup all files to the remote system.
ssh "$FQDN" sudo PASSPHRASE="$DUPLICITY_BACKUP_PASSPHRASE" duplicity --exclude "$REMOTE_HOME/backups" "$REMOTE_HOME" "file://$REMOTE_BACKUP_PATH"
ssh "$FQDN" sudo PASSPHRASE="$DUPLICITY_BACKUP_PASSPHRASE" duplicity --allow-source-mismatch --exclude "$REMOTE_HOME/backups" "$REMOTE_HOME" "file://$REMOTE_BACKUP_PATH"
ssh "$FQDN" sudo chown -R ubuntu:ubuntu "$REMOTE_BACKUP_PATH"
# now let's pull down the latest files from the backup directory.

View File

@ -85,4 +85,4 @@ else
exit
fi
echo "Successfull deployed '$DOMAIN_NAME' with git commit '$(cat ./.git/refs/heads/master)' VPS_HOSTING_TARGET=$VPS_HOSTING_TARGET; Latest git tag is $LATEST_GIT_TAG" >> "$SITE_PATH/debug.log"
echo "Successfull deployed '$DOMAIN_NAME' with git commit '$(cat ./.git/refs/heads/master)' VPS_HOSTING_TARGET=$VPS_HOSTING_TARGET;" >> "$SITE_PATH/debug.log"

View File

@ -1,6 +1,6 @@
#!/bin/bash
set -e
set -ex
# let's do a refresh of the certificates. Let's Encrypt will not run if it's not time.
docker pull certbot/certbot:latest

View File

@ -1,6 +1,6 @@
#!/bin/bash
set -eu
set -exu
# scan the remote machine and install it's identity in our SSH known_hosts file.
ssh-keyscan -H -t ecdsa "$FQDN" >> "$SSH_HOME/known_hosts"

View File

@ -58,7 +58,7 @@ function run_ddns {
echo ""
echo "SUCCESS: The DNS appears to be configured correctly."
echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow stale DNS records to expire."
echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow cached DNS records to expire."
sleep "$DDNS_SLEEP_SECONDS";
break;
fi
@ -69,11 +69,13 @@ function run_ddns {
}
# create the default storage pool if necessary
if ! lxc storage list --format csv | grep -q default; then
if ! lxc storage list --format csv | grep -q "sovereign-stack"; then
if [ -n "$LXD_DISK_TO_USE" ]; then
lxc storage create default zfs source="$LXD_DISK_TO_USE" size="${ROOT_DISK_SIZE_GB}GB"
lxc storage create "sovereign-stack" zfs source="$LXD_DISK_TO_USE"
# size="${ROOT_DISK_SIZE_GB}GB"
else
lxc storage create default zfs size="${ROOT_DISK_SIZE_GB}GB"
lxc storage create "sovereign-stack" zfs
# size="${ROOT_DISK_SIZE_GB}GB"
fi
fi

View File

@ -47,7 +47,6 @@ if [ "$APP_TO_DEPLOY" = www ] || [ "$APP_TO_DEPLOY" = certonly ]; then
--amazonec2-ami "$AWS_AMI_ID" \
--amazonec2-root-size "$ROOT_DISK_SIZE_GB" \
--amazonec2-instance-type "$WWW_INSTANCE_TYPE" \
--engine-label tag="$LATEST_GIT_TAG" \
--engine-label commit="$LATEST_GIT_COMMIT" \
"$FQDN"
@ -63,7 +62,6 @@ elif [ "$APP_TO_DEPLOY" = btcpay ]; then
--amazonec2-ami "$AWS_AMI_ID" \
--amazonec2-root-size "$ROOT_DISK_SIZE_GB" \
--amazonec2-instance-type "$BTCPAY_INSTANCE_TYPE" \
--engine-label tag="$LATEST_GIT_TAG" \
--engine-label commit="$LATEST_GIT_COMMIT" \
"$FQDN"

View File

@ -29,7 +29,7 @@ while true; do
echo ""
echo "SUCCESS: The DNS appears to be configured correctly."
echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow stale DNS records to expire."
echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow cached DNS records to expire."
sleep "$DDNS_SLEEP_SECONDS";
break;
fi

View File

@ -1,6 +1,6 @@
#!/bin/bash
set -e
set -ex
cd "$(dirname "$0")"
cat ./certs/docker.gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
@ -21,7 +21,7 @@ if ! snap list | grep -q lxd; then
fi
# let's ensure docker-machine is available. This is only temporary though.
curl -L https://github.com/docker/machine/releases/download/v0.16.2/docker-machine-$(uname -s)-$(uname -m) >/tmp/docker-machine &&
curl -L "https://github.com/docker/machine/releases/download/v0.16.2/docker-machine-$(uname -s)-$(uname -m)" >/tmp/docker-machine &&
chmod +x /tmp/docker-machine &&
sudo cp /tmp/docker-machine /usr/local/bin/docker-machine
@ -50,6 +50,5 @@ fi
sudo groupadd docker
sudo usermod -aG docker "$USER"
# TODO CHECK IF EXISTS
echo "alias ss-deploy='/home/$USER/sovereign-stack/deploy.sh \$@'" >> "$HOME/.bashrc"

View File

@ -1,6 +1,12 @@
#!/bin/bash
set -eu
set -ex
VALUE=${SITE_PATH:-}
if [ -z "$VALUE" ]; then
echo "ERROR: Your SITE_PATH is undefined. Did you specify the domain correctly?"
exit 1
fi
# check to see if the enf file exists. exist if not.
if [ ! -d "$SITE_PATH" ]; then
@ -27,13 +33,7 @@ export UNIX_BACKUP_TIMESTAMP="$UNIX_BACKUP_TIMESTAMP"
REMOTE_BACKUP_PATH="$REMOTE_HOME/backups/$APP_TO_DEPLOY/$BACKUP_TIMESTAMP"
LOCAL_BACKUP_PATH="$SITE_PATH/backups/$APP_TO_DEPLOY/$BACKUP_TIMESTAMP"
export LOCAL_BACKUP_PATH="$LOCAL_BACKUP_PATH"
BACKUP_PATH_CREATED=false
if [ ! -d "$LOCAL_BACKUP_PATH" ]; then
mkdir -p "$LOCAL_BACKUP_PATH"
BACKUP_PATH_CREATED=true
fi
export BACKUP_PATH_CREATED="$BACKUP_PATH_CREATED"
mkdir -p "$SSHFS_PATH"
# VALIDATE THE INPUT from the ENVFILE
@ -49,7 +49,6 @@ export NOSTR_FQDN="$NOSTR_HOSTNAME.$DOMAIN_NAME"
export ADMIN_ACCOUNT_USERNAME="info"
export CERTIFICATE_EMAIL_ADDRESS="$ADMIN_ACCOUNT_USERNAME@$DOMAIN_NAME"
#export MAIL_FROM="$SITE_TITLE <$CERTIFICATE_EMAIL_ADDRESS>"
export REMOTE_CERT_BASE_DIR="$REMOTE_HOME/.certs"
export REMOTE_CERT_DIR="$REMOTE_CERT_BASE_DIR/$FQDN"
@ -171,11 +170,6 @@ if [ -z "$DOMAIN_NAME" ]; then
exit 1
fi
#if [ -z "$SITE_TITLE" ]; then
# echo "ERROR: Ensure SITE_TITLE is configured in your site_definition."
# exit 1
#fi
if [ -z "$DEPLOY_BTCPPAY_SERVER" ]; then
echo "ERROR: Ensure DEPLOY_BTCPPAY_SERVER is configured in your site_definition."
exit 1