1
1
Fork 1

Various refactors. Separting out VMs.

This commit is contained in:
Derek Smith 2022-08-24 10:11:50 -04:00
parent c95265817f
commit 07c1c13bba
Signed by: farscapian
GPG Key ID: 8F1CD799CCA516CC
17 changed files with 332 additions and 304 deletions

2
.gitignore vendored
View File

@ -1 +1 @@
reset.sh
./reset.sh

View File

@ -74,6 +74,7 @@ export VM_NAME="sovereign-stack-base"
export DEV_MEMORY_MB="4096"
export DEV_CPU_COUNT="4"
export SSHFS_PATH="/tmp/sshfs_temp"
mkdir -p "$SSHFS_PATH"
export DOCKER_IMAGE_CACHE_FQDN="registry-1.docker.io"
export NEXTCLOUD_SPACE_GB=10
@ -136,3 +137,7 @@ export DEPLOY_MGMT_REGISTRY=true
export REMOTE_HOME="/home/ubuntu"
export BTCPAY_SERVER_APPPATH="$REMOTE_HOME/btcpayserver-docker"
export REMOTE_CERT_BASE_DIR="$REMOTE_HOME/.certs"
# this space is for OS, docker images, etc. DOES NOT INCLUDE USER DATA.
export ROOT_DISK_SIZE_GB=20

269
deploy.sh
View File

@ -23,22 +23,25 @@ if ! lsb_release -d | grep -q "Ubuntu 22.04"; then
exit 1
fi
MIGRATE_VPS=false
DOMAIN_NAME=
RESTORE_ARCHIVE=
VPS_HOSTING_TARGET=lxd
RUN_CERT_RENEWAL=true
USER_NO_BACKUP=false
USER_RUN_RESTORE=false
RESTORE_WWW_USERDATA=true
RESTORE_WWW=false
BACKUP_WWW=true
RESTORE_BTCPAY=false
BACKUP_BTCPAY=true
MIGRATE_WWW=false
MIGRATE_BTCPAY=false
USER_SKIP_WWW=false
USER_SKIP_BTCPAY=false
UPDATE_BTCPAY=false
RECONFIGURE_BTCPAY_SERVER=false
DEPLOY_BTCPAY_SERVER=false
CLUSTER_NAME="$(lxc remote get-default)"
RUN_BACKUP=true
# grab any modifications from the command line.
for i in "$@"; do
@ -48,13 +51,14 @@ for i in "$@"; do
shift
;;
--restore-www)
USER_RUN_RESTORE=true
RESTORE_WWW=true
BACKUP_WWW=false
RUN_CERT_RENEWAL=false
USER_NO_BACKUP=true
shift
;;
--restore-btcpay)
RESTORE_BTCPAY=true
BACKUP_BTCPAY=false
shift
;;
--archive=*)
@ -77,12 +81,21 @@ for i in "$@"; do
USER_SKIP_BTCPAY=true
shift
;;
--no-backup)
USER_NO_BACKUP=true
--no-backup-www)
BACKUP_WWW=false
shift
;;
--migrate)
MIGRATE_VPS=true
--no-backup-btcpay)
BACKUP_BTCPAY=false
shift
;;
--migrate-www)
MIGRATE_WWW=true
RUN_CERT_RENEWAL=false
shift
;;
--migrate-btcpay)
MIGRATE_BTCPAY=true
RUN_CERT_RENEWAL=false
shift
;;
@ -109,7 +122,12 @@ export SSH_HOME="$HOME/.ssh"
export DOMAIN_NAME="$DOMAIN_NAME"
export REGISTRY_DOCKER_IMAGE="registry:2"
export RESTORE_ARCHIVE="$RESTORE_ARCHIVE"
export RESTORE_WWW="$RESTORE_WWW"
export BACKUP_WWW="$BACKUP_WWW"
export RESTORE_BTCPAY="$RESTORE_BTCPAY"
export BACKUP_BTCPAY="$RESTORE_BTCPAY"
export MIGRATE_WWW="$MIGRATE_WWW"
export MIGRATE_BTCPAY="$MIGRATE_BTCPAY"
if [ "$VPS_HOSTING_TARGET" = aws ]; then
@ -172,7 +190,7 @@ proxy:
EOL
# enable docker swarm mode so we can support docker stacks.
if ! docker info | grep -q "Swarm: active"; then
if docker info | grep -q "Swarm: inactive"; then
docker swarm init
fi
@ -192,41 +210,23 @@ function new_pass {
gpg --gen-random --armor 1 25
}
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
# first let's get the DISK_TO_USE and DATA_PLANE_MACVLAN_INTERFACE from the ss-config
# which is set up during LXD cluster creation ss-cluster.
LXD_SS_CONFIG_LINE="$(lxc network list --format csv | grep ss-config)"
CONFIG_ITEMS="$(echo "$LXD_SS_CONFIG_LINE" | awk -F'"' '{print $2}')"
DATA_PLANE_MACVLAN_INTERFACE="$(echo "$CONFIG_ITEMS" | cut -d ',' -f2)"
DISK_TO_USE="$(echo "$CONFIG_ITEMS" | cut -d ',' -f3)"
export DATA_PLANE_MACVLAN_INTERFACE="$DATA_PLANE_MACVLAN_INTERFACE"
export DISK_TO_USE="$DISK_TO_USE"
./deployment/create_lxc_base.sh
fi
function run_domain {
function instantiate_vms {
export VPS_HOSTING_TARGET="$VPS_HOSTING_TARGET"
export RUN_CERT_RENEWAL="$RUN_CERT_RENEWAL"
export BTC_CHAIN="$BTC_CHAIN"
export UPDATE_BTCPAY="$UPDATE_BTCPAY"
export MIGRATE_VPS="$MIGRATE_VPS"
export RECONFIGURE_BTCPAY_SERVER="$RECONFIGURE_BTCPAY_SERVER"
# iterate over all our server endpoints and provision them if needed.
# www
VPS_HOSTNAME=
for VIRTUAL_MACHINE in www btcpayserver; do
FQDN=
# shellcheck disable=SC1091
source ./shared.sh
export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
if [ ! -f "$SITE_PATH/site_definition" ]; then
echo "ERROR: Something went wrong. Your site_definition is missing."
exit 1
@ -234,6 +234,51 @@ function run_domain {
source "$SITE_PATH/site_definition"
# VALIDATE THE INPUT from the ENVFILE
if [ -z "$DOMAIN_NAME" ]; then
echo "ERROR: DOMAIN_NAME not specified. Use the --domain-name= option."
exit 1
fi
# TODO, ensure VPS_HOSTING_TARGET is in range.
export NEXTCLOUD_FQDN="$NEXTCLOUD_HOSTNAME.$DOMAIN_NAME"
export BTCPAY_FQDN="$BTCPAY_HOSTNAME.$DOMAIN_NAME"
export BTCPAY_USER_FQDN="$BTCPAY_HOSTNAME_IN_CERT.$DOMAIN_NAME"
export WWW_FQDN="$WWW_HOSTNAME.$DOMAIN_NAME"
export GITEA_FQDN="$GITEA_HOSTNAME.$DOMAIN_NAME"
export NOSTR_FQDN="$NOSTR_HOSTNAME.$DOMAIN_NAME"
export ADMIN_ACCOUNT_USERNAME="info"
export CERTIFICATE_EMAIL_ADDRESS="$ADMIN_ACCOUNT_USERNAME@$DOMAIN_NAME"
export REMOTE_NEXTCLOUD_PATH="$REMOTE_HOME/nextcloud"
export REMOTE_GITEA_PATH="$REMOTE_HOME/gitea"
export BTC_CHAIN="$BTC_CHAIN"
export WWW_INSTANCE_TYPE="$WWW_INSTANCE_TYPE"
export BTCPAY_ADDITIONAL_HOSTNAMES="$BTCPAY_ADDITIONAL_HOSTNAMES"
# ensure the
if [ ! -f "$PROJECT_PATH/project_definition" ]; then
echo "ERROR: Your project_definition is not set."
exit 1
fi
source "$PROJECT_PATH/project_definition"
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
# first let's get the DISK_TO_USE and DATA_PLANE_MACVLAN_INTERFACE from the ss-config
# which is set up during LXD cluster creation ss-cluster.
LXD_SS_CONFIG_LINE="$(lxc network list --format csv | grep ss-config)"
CONFIG_ITEMS="$(echo "$LXD_SS_CONFIG_LINE" | awk -F'"' '{print $2}')"
DATA_PLANE_MACVLAN_INTERFACE="$(echo "$CONFIG_ITEMS" | cut -d ',' -f2)"
DISK_TO_USE="$(echo "$CONFIG_ITEMS" | cut -d ',' -f3)"
export DATA_PLANE_MACVLAN_INTERFACE="$DATA_PLANE_MACVLAN_INTERFACE"
export DISK_TO_USE="$DISK_TO_USE"
./deployment/create_lxc_base.sh
fi
# create the local packup path if it's not there!
BACKUP_PATH_CREATED=false
@ -247,19 +292,25 @@ function run_domain {
export REMOTE_BACKUP_PATH="$REMOTE_HOME/backups/$VIRTUAL_MACHINE/$BACKUP_TIMESTAMP"
LOCAL_BACKUP_PATH="$SITE_PATH/backups/$VIRTUAL_MACHINE/$BACKUP_TIMESTAMP"
export LOCAL_BACKUP_PATH="$LOCAL_BACKUP_PATH"
export BACKUP_TIMESTAMP="$BACKUP_TIMESTAMP"
export UNIX_BACKUP_TIMESTAMP="$UNIX_BACKUP_TIMESTAMP"
export REMOTE_CERT_DIR="$REMOTE_CERT_BASE_DIR/$FQDN"
# ensure the admin has set the MAC address for the base image.
if [ -z "$SOVEREIGN_STACK_MAC_ADDRESS" ]; then
echo "ERROR: SOVEREIGN_STACK_MAC_ADDRESS is undefined. Check your project definition."
exit 1
fi
if [ ! -d "$LOCAL_BACKUP_PATH" ]; then
mkdir -p "$LOCAL_BACKUP_PATH"
BACKUP_PATH_CREATED=true
RESTORE_WWW_USERDATA=false
fi
DDNS_HOST=
MIGRATE_VPS=false
if [ "$VIRTUAL_MACHINE" = www ]; then
if [ "$DEPLOY_WWW_SERVER" = false ] || [ "$USER_SKIP_WWW" = true ]; then
continue
@ -269,6 +320,9 @@ function run_domain {
MAC_ADDRESS_TO_PROVISION="$WWW_MAC_ADDRESS"
DDNS_HOST="$WWW_HOSTNAME"
ROOT_DISK_SIZE_GB="$((ROOT_DISK_SIZE_GB + NEXTCLOUD_SPACE_GB))"
if [ "$MIGRATE_WWW" = true ]; then
MIGRATE_VPS=true
fi
elif [ "$VIRTUAL_MACHINE" = btcpayserver ] || [ "$USER_SKIP_BTCPAY" = true ]; then
if [ "$DEPLOY_BTCPAY_SERVER" = false ]; then
continue
@ -282,10 +336,14 @@ function run_domain {
elif [ "$BTC_CHAIN" = testnet ]; then
ROOT_DISK_SIZE_GB=70
fi
if [ "$MIGRATE_BTCPAY" = true ]; then
MIGRATE_VPS=true
fi
elif [ "$VIRTUAL_MACHINE" = "sovereign-stack" ]; then
DDNS_HOST="sovereign-stack-base"
ROOT_DISK_SIZE_GB=8
MAC_ADDRESS_TO_PROVISION="$SOVEREIGN_STACK_MAC_ADDRESS"
else
echo "ERROR: VIRTUAL_MACHINE not within allowable bounds."
exit
@ -294,8 +352,8 @@ function run_domain {
export DDNS_HOST="$DDNS_HOST"
export FQDN="$DDNS_HOST.$DOMAIN_NAME"
export LXD_VM_NAME="${FQDN//./-}"
#${PROJECT_NAME//./-}-
export REMOTE_BACKUP_PATH="$REMOTE_BACKUP_PATH"
export RESTORE_WWW_USERDATA="$RESTORE_WWW_USERDATA"
# This next section of if statements is our sanity checking area.
if [ "$VPS_HOSTING_TARGET" = aws ]; then
@ -306,74 +364,6 @@ function run_domain {
fi
fi
if [ "$DEPLOY_GHOST" = true ]; then
if [ -z "$GHOST_MYSQL_PASSWORD" ]; then
echo "ERROR: Ensure GHOST_MYSQL_PASSWORD is configured in your site_definition."
exit 1
fi
if [ -z "$GHOST_MYSQL_ROOT_PASSWORD" ]; then
echo "ERROR: Ensure GHOST_MYSQL_ROOT_PASSWORD is configured in your site_definition."
exit 1
fi
fi
if [ "$DEPLOY_GITEA" = true ]; then
if [ -z "$GITEA_MYSQL_PASSWORD" ]; then
echo "ERROR: Ensure GITEA_MYSQL_PASSWORD is configured in your site_definition."
exit 1
fi
if [ -z "$GITEA_MYSQL_ROOT_PASSWORD" ]; then
echo "ERROR: Ensure GITEA_MYSQL_ROOT_PASSWORD is configured in your site_definition."
exit 1
fi
fi
if [ "$DEPLOY_NEXTCLOUD" = true ]; then
if [ -z "$NEXTCLOUD_MYSQL_ROOT_PASSWORD" ]; then
echo "ERROR: Ensure NEXTCLOUD_MYSQL_ROOT_PASSWORD is configured in your site_definition."
exit 1
fi
if [ -z "$NEXTCLOUD_MYSQL_PASSWORD" ]; then
echo "ERROR: Ensure NEXTCLOUD_MYSQL_PASSWORD is configured in your site_definition."
exit 1
fi
fi
if [ "$DEPLOY_NOSTR" = true ]; then
if [ -z "$NOSTR_ACCOUNT_PUBKEY" ]; then
echo "ERROR: Ensure NOSTR_ACCOUNT_PUBKEY is configured in your site_definition."
exit 1
fi
if [ -z "$NOSTR_ACCOUNT_PUBKEY" ]; then
echo "ERROR: Ensure NOSTR_ACCOUNT_PUBKEY is configured in your site_definition."
exit 1
fi
fi
if [ -z "$DUPLICITY_BACKUP_PASSPHRASE" ]; then
echo "ERROR: Ensure DUPLICITY_BACKUP_PASSPHRASE is configured in your site_definition."
exit 1
fi
if [ -z "$DOMAIN_NAME" ]; then
echo "ERROR: Ensure DOMAIN_NAME is configured in your site_definition."
exit 1
fi
if [ -z "$DEPLOY_BTCPPAY_SERVER" ]; then
echo "ERROR: Ensure DEPLOY_BTCPPAY_SERVER is configured in your site_definition."
exit 1
fi
if [ -z "$NOSTR_ACCOUNT_PUBKEY" ]; then
echo "ERROR: You MUST specify a Nostr public key. This is how you get all your social features."
echo "INFO: Go to your site_definition file and set the NOSTR_ACCOUNT_PUBKEY variable."
exit 1
fi
MACHINE_EXISTS=false
if [ "$VPS_HOSTING_TARGET" = aws ] && docker-machine ls -q | grep -q "$FQDN"; then
MACHINE_EXISTS=true
@ -383,10 +373,6 @@ function run_domain {
MACHINE_EXISTS=true
fi
if [ "$USER_NO_BACKUP" = true ]; then
RUN_BACKUP=false
fi
if [ "$MACHINE_EXISTS" = true ]; then
# we delete the machine if the user has directed us to
if [ "$MIGRATE_VPS" = true ]; then
@ -398,7 +384,7 @@ function run_domain {
# get a backup of the machine. This is what we restore to the new VPS.
echo "INFO: Machine exists. Since we're going to delete it, let's grab a backup. We don't need to restore services since we're deleting it."
RESTORE_BTCPAY=false UPDATE_BTCPAY=false RUN_RESTORE=false RUN_BACKUP=true RUN_SERVICES=false ./deployment/domain_init.sh
./deployment/deploy_vms.sh
# delete the remote VPS.
if [ "$VPS_HOSTING_TARGET" = aws ]; then
@ -412,17 +398,16 @@ function run_domain {
fi
elif [ "$VPS_HOSTING_TARGET" = lxd ]; then
lxc delete --force "$LXD_VM_NAME"
USER_RUN_RESTORE=true
fi
# Then we run the script again to re-instantiate a new VPS, restoring all user data
# if restore directory doesn't exist, then we end up with a new site.
echo "INFO: Recreating the remote VPS then restoring user data."
sleep 5
RESTORE_BTCPAY=true UPDATE_BTCPAY=false RUN_RESTORE=true RUN_BACKUP=false RUN_SERVICES=true RUN_CERT_RENEWAL=false RESTORE_ARCHIVE="$RESTORE_ARCHIVE" ./deployment/domain_init.sh
sleep 5
sleep 2
./deployment/deploy_vms.sh
else
RESTORE_BTCPAY="$RESTORE_BTCPAY" UPDATE_BTCPAY="$UPDATE_BTCPAY" RUN_RESTORE="$USER_RUN_RESTORE" RUN_BACKUP="$RUN_BACKUP" RUN_SERVICES=true ./deployment/domain_init.sh
./deployment/deploy_vms.sh
fi
else
if [ "$MIGRATE_VPS" = true ]; then
@ -430,22 +415,54 @@ function run_domain {
fi
# The machine does not exist. Let's bring it into existence, restoring from latest backup.
echo "Machine does not exist. RUN_RESTORE=$USER_RUN_RESTORE RUN_BACKUP=false"
RESTORE_BTCPAY="$RESTORE_BTCPAY" UPDATE_BTCPAY="$UPDATE_BTCPAY" RUN_RESTORE="$USER_RUN_RESTORE" RUN_BACKUP=false RUN_SERVICES=true ./deployment/domain_init.sh
echo "Machine does not exist."
./deployment/deploy_vms.sh
fi
done
}
function run_domain {
# if the local docker client isn't logged in, do so;
# this helps prevent docker pull errors since they throttle.
if [ ! -f "$HOME/.docker/config.json" ]; then
echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USERNAME" --password-stdin
fi
# this tells our local docker client to target the remote endpoint via SSH
export DOCKER_HOST="ssh://ubuntu@$WWW_FQDN"
# enable docker swarm mode so we can support docker stacks.
if docker info | grep -q "Swarm: inactive"; then
docker swarm init --advertise-addr enp6s0
fi
bash -c "./deployment/www/go.sh"
export DOCKER_HOST="ssh://ubuntu@$BTCPAY_FQDN"
# enable docker swarm mode so we can support docker stacks.
if docker info | grep -q "Swarm: inactive"; then
docker swarm init --advertise-addr enp6s0
fi
bash -c "./deployment/btcpayserver/go.sh"
echo "Successfully deployed '$DOMAIN_NAME' with git commit '$(cat ./.git/refs/heads/master)' VPS_HOSTING_TARGET=$VPS_HOSTING_TARGET;"
}
function stub_site_definition {
mkdir -p "$SITE_PATH" "$PROJECT_PATH/sites"
export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
mkdir -p "$SITE_PATH"
if [ -f "$SITE_PATH/site_definition" ]; then
source ./shared.sh
else
# create a symlink from the CLUSTERPATH/sites/DOMAIN_NAME to the ss-sites/domain name
if [ ! -d "$PROJECT_PATH/sites/$DOMAIN_NAME" ]; then
ln -s "$SITE_PATH" "$PROJECT_PATH/sites/$DOMAIN_NAME"
fi
if [ ! -f "$SITE_PATH/site_definition" ]; then
# check to see if the enf file exists. exist if not.
SITE_DEFINITION_PATH="$SITE_PATH/site_definition"
if [ ! -f "$SITE_DEFINITION_PATH" ]; then
@ -463,10 +480,6 @@ export DUPLICITY_BACKUP_PASSPHRASE="$(new_pass)"
# AWS only
#export DDNS_PASSWORD=
## WWW
export DEPLOY_WWW_SERVER=true
export WWW_SERVER_MAC_ADDRESS="CHANGE_ME_REQUIRED"
# Deploy APPS to www
export DEPLOY_GHOST=true
export DEPLOY_NEXTCLOUD=true

View File

@ -23,12 +23,11 @@ elif [ "$RESTORE_BTCPAY" = true ]; then
./restore.sh
RUN_BACKUP=false
RUN_SERVICES=true
OPEN_URL=true
# if this is a new VM from a migration procedure, then we can now run setup.
if [ "$MIGRATE_VPS" = true ]; then
if [ "$MIGRATE_BTCPAY" = true ]; then
./stub_btcpay_setup.sh
fi
@ -37,13 +36,12 @@ elif [ "$RECONFIGURE_BTCPAY_SERVER" == true ]; then
# if so, we re-run setup script.
./stub_btcpay_setup.sh
RUN_BACKUP=false
RUN_SERVICES=true
OPEN_URL=true
fi
# if the script gets this far, then we grab a regular backup.
if [ "$RUN_BACKUP" = true ]; then
if [ "$BACKUP_BTCPAY" = true ]; then
# we just grab a regular backup
./backup.sh "$UNIX_BACKUP_TIMESTAMP"
fi
@ -60,8 +58,8 @@ fi
if [ "$OPEN_URL" = true ]; then
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
if wait-for-it -t 5 "$WWW_FQDN:443"; then
xdg-open "https://$WWW_FQDN" > /dev/null 2>&1
if wait-for-it -t 5 "$WWW_FQDN:80"; then
xdg-open "http://$WWW_FQDN" > /dev/null 2>&1
fi
else
if wait-for-it -t 5 "$FQDN:443"; then

View File

@ -19,9 +19,6 @@ while [ ! -f /var/lib/cloud/instance/boot-finished ]; do
sleep 1
done
#curl -SL https://github.com/docker/compose/releases/download/v2.6.1/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
#chmod 0777 /usr/local/bin/docker-compose
if [ ! -d "btcpayserver-docker" ]; then
echo "cloning btcpayserver-docker";
git clone -b master https://github.com/btcpayserver/btcpayserver-docker btcpayserver-docker;
@ -78,5 +75,6 @@ EOL
# send the setup script to the remote machine.
scp "$SITE_PATH/btcpay.sh" "ubuntu@$FQDN:$REMOTE_HOME/btcpay_setup.sh"
ssh "$FQDN" "chmod 0744 $REMOTE_HOME/btcpay_setup.sh"
ssh "$FQDN" "sudo bash -c ./btcpay_setup.sh"
ssh "$BTCPAY_FQDN" "chmod 0744 $REMOTE_HOME/btcpay_setup.sh"
ssh "$BTCPAY_FQDN" "sudo bash -c ./btcpay_setup.sh"
ssh "$BTCPAY_FQDN" "touch $REMOTE_HOME/btcpay.complete"

View File

@ -33,9 +33,12 @@ function prepare_host {
# create a directory to store backup archives. This is on all new vms.
ssh "$FQDN" mkdir -p "$REMOTE_HOME/backups"
# if this execution is for btcpayserver, then we run the stub/btcpay setup script
# but only if it hasn't been executed before.
if [ "$VIRTUAL_MACHINE" = btcpayserver ]; then
echo "INFO: new machine detected. Provisioning BTCPay server scripts."
./btcpayserver/stub_btcpay_setup.sh
if [ "$(ssh "$BTCPAY_FQDN" [[ ! -f "$REMOTE_HOME/btcpay.complete" ]]; echo $?)" -eq 0 ]; then
./btcpayserver/stub_btcpay_setup.sh
fi
fi
}
@ -44,7 +47,6 @@ function prepare_host {
if [ "$VPS_HOSTING_TARGET" = aws ]; then
# let's create the remote VPS if needed.
if ! docker-machine ls -q --filter name="$FQDN" | grep -q "$FQDN"; then
RUN_BACKUP=false
./provision_vps.sh
@ -55,7 +57,6 @@ elif [ "$VPS_HOSTING_TARGET" = lxd ]; then
# if the machine doesn't exist, we create it.
if ! lxc list --format csv | grep -q "$LXD_VM_NAME"; then
export RUN_BACKUP=false
# create a base image if needed and instantiate a VM.
if [ -z "$MAC_ADDRESS_TO_PROVISION" ]; then
@ -68,18 +69,5 @@ elif [ "$VPS_HOSTING_TARGET" = lxd ]; then
fi
prepare_host
fi
# if the local docker client isn't logged in, do so;
# this helps prevent docker pull errors since they throttle.
if [ ! -f "$HOME/.docker/config.json" ]; then
echo "$REGISTRY_PASSWORD" | docker login --username "$REGISTRY_USERNAME" --password-stdin
fi
# this tells our local docker client to target the remote endpoint via SSH
export DOCKER_HOST="ssh://ubuntu@$FQDN"
# the following scripts take responsibility for the rest of the provisioning depending on the app you're deploying.
bash -c "./$VIRTUAL_MACHINE/go.sh"
echo "Successfully deployed '$DOMAIN_NAME' with git commit '$(cat ./.git/refs/heads/master)' VPS_HOSTING_TARGET=$VPS_HOSTING_TARGET;"

View File

@ -26,13 +26,9 @@ function run_ddns {
# we're waiting here to allow dns records to stale out.
# this is necessary for certificate renewal; letsencrypt might have stale records
# and cert renew won't succeed. HOWEVER, if we're running a restore operation, we SHOULD NOT
# do a certificate renewal (we're restoring old certs). Thus it is not necessary to sleep here.
if [ "$RUN_RESTORE" = false ]; then
echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow cached DNS records to expire."
sleep "$DDNS_SLEEP_SECONDS";
fi
echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow cached DNS records to expire."
sleep "$DDNS_SLEEP_SECONDS";
break;
fi

View File

@ -194,16 +194,16 @@ else
dhcp4: false
EOF
if [[ "$LXD_HOSTNAME" = www-* ]]; then
if [[ "$LXD_HOSTNAME" = $WWW_HOSTNAME-* ]]; then
cat >> "$YAML_PATH" <<EOF
addresses: [10.139.144.5/24]
nameservers:
addresses: [10.139.144.1]
EOF
fi
if [[ "$LXD_HOSTNAME" = btcpay-* ]]; then
if [[ "$LXD_HOSTNAME" = $BTCPAY_HOSTNAME-* ]]; then
cat >> "$YAML_PATH" <<EOF
addresses: [10.139.144.10/24]
nameservers:
@ -226,9 +226,7 @@ devices:
type: disk
EOF
# TODO get the sovereign-stack lxc profile OFF the lxdbrSS bridge network.
echo "DATA_PLANE_MACVLAN_INTERFACE: $DATA_PLANE_MACVLAN_INTERFACE"
# Stub out the network piece for the base image.
if [ "$LXD_HOSTNAME" = sovereign-stack ] ; then
# If we are deploying the www, we attach the vm to the underlay via macvlan.

View File

@ -9,12 +9,12 @@ cd "$(dirname "$0")"
# maybe something like https://superuser.com/questions/616182/how-to-mount-local-directory-to-remote-like-sshfs
# step 1: run duplicity on the remote system to backup all files to the remote system.
ssh "$FQDN" sudo PASSPHRASE="$DUPLICITY_BACKUP_PASSPHRASE" duplicity --allow-source-mismatch --exclude "$REMOTE_HOME/backups" "$REMOTE_HOME" "file://$REMOTE_BACKUP_PATH"
ssh "$FQDN" sudo chown -R ubuntu:ubuntu "$REMOTE_BACKUP_PATH"
ssh "$WWW_FQDN" sudo PASSPHRASE="$DUPLICITY_BACKUP_PASSPHRASE" duplicity --allow-source-mismatch --exclude "$REMOTE_HOME/backups" "$REMOTE_HOME" "file://$REMOTE_BACKUP_PATH"
ssh "$WWW_FQDN" sudo chown -R ubuntu:ubuntu "$REMOTE_BACKUP_PATH"
# now let's pull down the latest files from the backup directory.
# create a temp directory to serve as the mountpoint for the remote machine backups directory
sshfs "$FQDN:$REMOTE_BACKUP_PATH" "$SSHFS_PATH"
sshfs "$WWW_FQDN:$REMOTE_BACKUP_PATH" "$SSHFS_PATH"
# rsync the files from the remote server to our local backup path.
rsync -av "$SSHFS_PATH/" "$LOCAL_BACKUP_PATH/"

View File

@ -18,8 +18,8 @@ if [ "$VPS_HOSTING_TARGET" = aws ]; then
certbot/certbot certonly -v --noninteractive --agree-tos --key-type ecdsa --standalone --expand -d "$DOMAIN_NAME" -d "$FQDN" -d "$NEXTCLOUD_FQDN" -d "$GITEA_FQDN" --email "$CERTIFICATE_EMAIL_ADDRESS"
elif [ "$VPS_HOSTING_TARGET" = lxd ]; then
# with the lxd side, we are trying to expose ALL OUR services from one IP address, which terminates
# at a cachehing reverse proxy that runs nginx.
# with the lxd side, we are trying to expose ALL OUR services from one IP address, which terminates
# at a cachehing reverse proxy that runs nginx.
docker run -it --rm \
--name certbot \
-p 80:80 \

View File

@ -1,65 +1,115 @@
#!/bin/bash
set -exu
set -exuo
cd "$(dirname "$0")"
if [ "$DEPLOY_GHOST" = true ]; then
if [ -z "$GHOST_MYSQL_PASSWORD" ]; then
echo "ERROR: Ensure GHOST_MYSQL_PASSWORD is configured in your site_definition."
exit 1
fi
if [ -z "$GHOST_MYSQL_ROOT_PASSWORD" ]; then
echo "ERROR: Ensure GHOST_MYSQL_ROOT_PASSWORD is configured in your site_definition."
exit 1
fi
fi
if [ "$DEPLOY_GITEA" = true ]; then
if [ -z "$GITEA_MYSQL_PASSWORD" ]; then
echo "ERROR: Ensure GITEA_MYSQL_PASSWORD is configured in your site_definition."
exit 1
fi
if [ -z "$GITEA_MYSQL_ROOT_PASSWORD" ]; then
echo "ERROR: Ensure GITEA_MYSQL_ROOT_PASSWORD is configured in your site_definition."
exit 1
fi
fi
if [ "$DEPLOY_NEXTCLOUD" = true ]; then
if [ -z "$NEXTCLOUD_MYSQL_ROOT_PASSWORD" ]; then
echo "ERROR: Ensure NEXTCLOUD_MYSQL_ROOT_PASSWORD is configured in your site_definition."
exit 1
fi
if [ -z "$NEXTCLOUD_MYSQL_PASSWORD" ]; then
echo "ERROR: Ensure NEXTCLOUD_MYSQL_PASSWORD is configured in your site_definition."
exit 1
fi
fi
if [ "$DEPLOY_NOSTR" = true ]; then
if [ -z "$NOSTR_ACCOUNT_PUBKEY" ]; then
echo "ERROR: Ensure NOSTR_ACCOUNT_PUBKEY is configured in your site_definition."
exit 1
fi
if [ -z "$NOSTR_ACCOUNT_PUBKEY" ]; then
echo "ERROR: Ensure NOSTR_ACCOUNT_PUBKEY is configured in your site_definition."
exit 1
fi
fi
if [ -z "$DUPLICITY_BACKUP_PASSPHRASE" ]; then
echo "ERROR: Ensure DUPLICITY_BACKUP_PASSPHRASE is configured in your site_definition."
exit 1
fi
if [ -z "$DOMAIN_NAME" ]; then
echo "ERROR: Ensure DOMAIN_NAME is configured in your site_definition."
exit 1
fi
if [ -z "$NOSTR_ACCOUNT_PUBKEY" ]; then
echo "ERROR: You MUST specify a Nostr public key. This is how you get all your social features."
echo "INFO: Go to your site_definition file and set the NOSTR_ACCOUNT_PUBKEY variable."
exit 1
fi
bash -c ./stub_nginxconf.sh
TOR_CONFIG_PATH=
ssh "$FQDN" mkdir -p "$REMOTE_HOME/ghost_site" "$REMOTE_HOME/ghost_db"
ssh "$WWW_FQDN" mkdir -p "$REMOTE_HOME/ghost_site" "$REMOTE_HOME/ghost_db"
if [ "$DEPLOY_NEXTCLOUD" = true ]; then
ssh "$FQDN" "mkdir -p $REMOTE_NEXTCLOUD_PATH/db/data"
ssh "$FQDN" "mkdir -p $REMOTE_NEXTCLOUD_PATH/db/logs"
ssh "$FQDN" "mkdir -p $REMOTE_NEXTCLOUD_PATH/html"
ssh "$WWW_FQDN" "mkdir -p $REMOTE_NEXTCLOUD_PATH/db/data"
ssh "$WWW_FQDN" "mkdir -p $REMOTE_NEXTCLOUD_PATH/db/logs"
ssh "$WWW_FQDN" "mkdir -p $REMOTE_NEXTCLOUD_PATH/html"
fi
if [ "$DEPLOY_GITEA" = true ]; then
ssh "$FQDN" "mkdir -p $REMOTE_GITEA_PATH/data $REMOTE_GITEA_PATH/db"
fi
# enable docker swarm mode so we can support docker stacks.
if ! docker info | grep -q "Swarm: active"; then
docker swarm init --advertise-addr enp6s0
fi
# stop services.
if docker stack list --format "{{.Name}}" | grep -q webstack; then
docker stack rm webstack
sleep 15
fi
# this will generate letsencrypt certs and pull them down locally.
# if [ "$VPS_HOSTING_TARGET" != lxd ]; then
# really we should change this if clause to some thing like
# "if the perimeter firewall allows port 80/443, then go ahead."
if [ "$RUN_CERT_RENEWAL" = true ]; then
./generate_certs.sh
fi
if [ "$RUN_BACKUP" = true ]; then
if [ "$BACKUP_WWW" = true ]; then
./backup.sh
fi
if [ "$RUN_RESTORE" = true ]; then
if [ "$RESTORE_WWW" = true ]; then
# Generally speaking we try to restore data. But if the BACKUP directory was
# just created, we know that we'll deploy fresh.
if [ "$RESTORE_WWW_USERDATA" = true ]; then
./restore.sh
fi
./restore.sh
else
./generate_certs.sh
fi
if [ "$DEPLOY_ONION_SITE" = true ]; then
# ensure the tor image is built
docker build -t tor:latest ./tor
# if the tor folder doesn't exist, we provision a new one. Otherwise you need to restore.
# this is how we generate a new torv3 endpoint.
if ! ssh "$FQDN" "[ -d $REMOTE_HOME/tor/www ]"; then
ssh "$FQDN" "mkdir -p $REMOTE_HOME/tor"
if ! ssh "$WWW_FQDN" "[ -d $REMOTE_HOME/tor/www ]"; then
ssh "$WWW_FQDN" "mkdir -p $REMOTE_HOME/tor"
TOR_CONFIG_PATH="$(pwd)/tor/torrc-init"
export TOR_CONFIG_PATH="$TOR_CONFIG_PATH"
docker stack deploy -c ./tor.yml torstack
@ -68,37 +118,37 @@ if [ "$DEPLOY_ONION_SITE" = true ]; then
sleep 20
fi
ONION_ADDRESS="$(ssh "$FQDN" sudo cat "${REMOTE_HOME}"/tor/www/hostname)"
ONION_ADDRESS="$(ssh "$WWW_FQDN" sudo cat "${REMOTE_HOME}"/tor/www/hostname)"
export ONION_ADDRESS="$ONION_ADDRESS"
# # Since we run a separate ghost process, we create a new directory and symlink it to the original
# if ! ssh "$FQDN" "[ -L $REMOTE_HOME/tor_ghost ]"; then
# ssh "$FQDN" ln -s "$REMOTE_HOME/ghost_site/themes $REMOTE_HOME/tor_ghost/themes"
# if ! ssh "$WWW_FQDN" "[ -L $REMOTE_HOME/tor_ghost ]"; then
# ssh "$WWW_FQDN" ln -s "$REMOTE_HOME/ghost_site/themes $REMOTE_HOME/tor_ghost/themes"
# fi
fi
if [ "$RUN_SERVICES" = true ]; then
mkdir -p "$SITE_PATH/stacks"
DOCKER_YAML_PATH="$SITE_PATH/stacks/www.yml"
export DOCKER_YAML_PATH="$DOCKER_YAML_PATH"
bash -c ./stub_docker_yml.sh
#if [ "$RUN_SERVICES" = true ]; then
mkdir -p "$SITE_PATH/stacks"
DOCKER_YAML_PATH="$SITE_PATH/stacks/www.yml"
export DOCKER_YAML_PATH="$DOCKER_YAML_PATH"
bash -c ./stub_docker_yml.sh
docker stack deploy -c "$DOCKER_YAML_PATH" webstack
docker stack deploy -c "$DOCKER_YAML_PATH" webstack
# start a browser session; point it to port 80 to ensure HTTPS redirect.
wait-for-it -t 320 "$FQDN:80"
wait-for-it -t 320 "$FQDN:443"
# start a browser session; point it to port 80 to ensure HTTPS redirect.
wait-for-it -t 320 "$WWW_FQDN:80"
wait-for-it -t 320 "$WWW_FQDN:443"
# open bowser tabs.
if [ "$DEPLOY_GHOST" = true ]; then
xdg-open "http://$FQDN" > /dev/null 2>&1
fi
if [ "$DEPLOY_NEXTCLOUD" = true ]; then
xdg-open "http://$NEXTCLOUD_FQDN" > /dev/null 2>&1
fi
if [ "$DEPLOY_GITEA" = true ]; then
xdg-open "http://$GITEA_FQDN" > /dev/null 2>&1
fi
# open bowser tabs.
if [ "$DEPLOY_GHOST" = true ]; then
xdg-open "http://$WWW_FQDN" > /dev/null 2>&1
fi
if [ "$DEPLOY_NEXTCLOUD" = true ]; then
xdg-open "http://$NEXTCLOUD_FQDN" > /dev/null 2>&1
fi
if [ "$DEPLOY_GITEA" = true ]; then
xdg-open "http://$GITEA_FQDN" > /dev/null 2>&1
fi
#fi

View File

@ -7,13 +7,13 @@ set -exu
# indeed, our first step is the delete the home directory on the remote server.
# delete the home directory so we know we are restoring all files from the duplicity archive.
ssh "$FQDN" sudo rm -rf "$REMOTE_HOME/*"
ssh "$WWW_FQDN" sudo rm -rf "$REMOTE_HOME/*"
# scp our local backup directory to the remote machine
ssh "$FQDN" mkdir -p "$REMOTE_BACKUP_PATH"
ssh "$WWW_FQDN" mkdir -p "$REMOTE_BACKUP_PATH"
# TODO instead of scp the files up there, lets' mount the local backup folder to a remote folder then just run a duplicity restore.
scp -r "$LOCAL_BACKUP_PATH/" "$FQDN:$REMOTE_HOME/backups/$VIRTUAL_MACHINE"
scp -r "$LOCAL_BACKUP_PATH/" "$WWW_FQDN:$REMOTE_HOME/backups/$VIRTUAL_MACHINE"
# now we run duplicity to restore the archive.
ssh "$FQDN" sudo PASSPHRASE="$DUPLICITY_BACKUP_PASSPHRASE" duplicity --force restore "file://$REMOTE_BACKUP_PATH/" "$REMOTE_HOME/"
ssh "$WWW_FQDN" sudo PASSPHRASE="$DUPLICITY_BACKUP_PASSPHRASE" duplicity --force restore "file://$REMOTE_BACKUP_PATH/" "$REMOTE_HOME/"

View File

@ -32,7 +32,7 @@ cat >>"$DOCKER_YAML_PATH" <<EOL
volumes:
- ${REMOTE_HOME}/ghost_site:/var/lib/ghost/content
environment:
- url=https://${FQDN}
- url=https://${WWW_FQDN}
- database__client=mysql
- database__connection__host=ghostdb
- database__connection__user=ghost
@ -296,7 +296,7 @@ cat >>"$DOCKER_YAML_PATH" <<EOL
configs:
nginx-config:
file: ${SITE_PATH}/nginx.conf
file: ${PROJECT_PATH}/nginx.conf
EOL
if [ "$DEPLOY_ONION_SITE" = true ]; then

View File

@ -51,7 +51,7 @@ EOL
# ghost http to https redirects.
cat >>"$NGINX_CONF_PATH" <<EOL
# http://${DOMAIN_NAME} redirect to https://${FQDN}
# http://${DOMAIN_NAME} redirect to https://${WWW_FQDN}
server {
listen 80;
listen [::]:80;
@ -63,15 +63,16 @@ cat >>"$NGINX_CONF_PATH" <<EOL
return 301 https://${DOMAIN_NAME}\$request_uri;
}
}
EOL
cat >>"$NGINX_CONF_PATH" <<EOL
# http://${FQDN} redirect to https://${FQDN}
# http://${WWW_FQDN} redirect to https://${WWW_FQDN}
server {
listen 80;
listen [::]:80;
server_name ${FQDN};
return 301 https://${FQDN}\$request_uri;
server_name ${WWW_FQDN};
return 301 https://${WWW_FQDN}\$request_uri;
}
EOL
@ -128,9 +129,6 @@ cat >>"$NGINX_CONF_PATH" <<EOL
# global TLS settings
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1.3;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
ssl_session_tickets off;
@ -142,19 +140,29 @@ cat >>"$NGINX_CONF_PATH" <<EOL
# default server if hostname not specified.
#server {
# listen 443 default_server;
# return 403;
#}
server {
listen 443 default_server;
# map \$http_user_agent \$og_prefix {
# ~*(googlebot|twitterbot)/ /open-graph;
# }
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
return 403;
}
# maybe helps with Twitter cards.
map \$http_user_agent \$og_prefix {
~*(googlebot|twitterbot)/ /open-graph;
}
# https://${DOMAIN_NAME} redirect to https://${FQDN}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
server_name ${DOMAIN_NAME};
@ -176,9 +184,9 @@ EOL
fi
cat >>"$NGINX_CONF_PATH" <<EOL
# catch all; send request to ${FQDN}
# catch all; send request to ${WWW_FQDN}
location / {
return 301 https://${FQDN}\$request_uri;
return 301 https://${WWW_FQDN}\$request_uri;
}
EOL
#####################################################
@ -209,6 +217,11 @@ if [ "$VPS_HOSTING_TARGET" = lxd ]; then
server {
listen 443 ssl http2;
ssl on;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
server_name ${BTCPAY_USER_FQDN};
# Route everything to the real BTCPay server
@ -246,7 +259,11 @@ cat >>"$NGINX_CONF_PATH" <<EOL
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name ${FQDN};
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
server_name ${WWW_FQDN};
EOL
# add the Onion-Location header if specifed.
@ -376,6 +393,11 @@ cat >>"$NGINX_CONF_PATH" <<EOL
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
ssl_trusted_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
server_name ${NEXTCLOUD_FQDN};
location / {

View File

@ -2,10 +2,10 @@
set -x
CLUSTER_NAME=""
SSH_ENDPOINT_HOSTNAME=""
SSH_ENDPOINT_DOMAIN_NAME=""
TEST_DOMAIN=""
CLUSTER_NAME="development"
SSH_ENDPOINT_HOSTNAME="atlantis"
SSH_ENDPOINT_DOMAIN_NAME="ancapistan.io"
TEST_DOMAIN="ancapistan.casa"
export LXD_VM_NAME="${TEST_DOMAIN//./-}"
@ -21,6 +21,8 @@ lxc profile delete sovereign-stack
lxc image rm sovereign-stack-base
lxc image rm ubuntu-base
lxc network delete lxdbrSS
lxc storage delete sovereign-stack
lxc remote switch "local"
@ -28,6 +30,6 @@ lxc remote remove "$CLUSTER_NAME"
source "$HOME/.bashrc"
./cluster.sh create "$CLUSTER_NAME" "$SSH_ENDPOINT.$DOMAIN_NAME"
./cluster.sh create "$CLUSTER_NAME" "$SSH_ENDPOINT_HOSTNAME.$SSH_ENDPOINT_DOMAIN_NAME" --data-plane-interface=enp89s0
./deploy.sh
#./deploy.sh

View File

@ -1,49 +0,0 @@
#!/bin/bash
set -ex
VALUE=${SITE_PATH:-}
if [ -z "$VALUE" ]; then
echo "ERROR: Your SITE_PATH is undefined. Did you specify the domain correctly?"
exit 1
fi
# check to see if the enf file exists. exist if not.
if [ ! -d "$SITE_PATH" ]; then
echo "ERROR: '$SITE_PATH' does not exist."
exit 1
fi
mkdir -p "$SSHFS_PATH"
# VALIDATE THE INPUT from the ENVFILE
if [ -z "$DOMAIN_NAME" ]; then
echo "ERROR: DOMAIN_NAME not specified. Use the --domain-name= option."
exit 1
fi
# TODO, ensure VPS_HOSTING_TARGET is in range.
export NEXTCLOUD_FQDN="$NEXTCLOUD_HOSTNAME.$DOMAIN_NAME"
export BTCPAY_FQDN="$BTCPAY_HOSTNAME.$DOMAIN_NAME"
export BTCPAY_USER_FQDN="$BTCPAY_HOSTNAME_IN_CERT.$DOMAIN_NAME"
export WWW_FQDN="$WWW_HOSTNAME.$DOMAIN_NAME"
export GITEA_FQDN="$GITEA_HOSTNAME.$DOMAIN_NAME"
export NOSTR_FQDN="$NOSTR_HOSTNAME.$DOMAIN_NAME"
export ADMIN_ACCOUNT_USERNAME="info"
export CERTIFICATE_EMAIL_ADDRESS="$ADMIN_ACCOUNT_USERNAME@$DOMAIN_NAME"
export REMOTE_CERT_BASE_DIR="$REMOTE_HOME/.certs"
export REMOTE_NEXTCLOUD_PATH="$REMOTE_HOME/nextcloud"
export REMOTE_GITEA_PATH="$REMOTE_HOME/gitea"
# this space is for OS, docker images, etc. DOES NOT INCLUDE USER DATA.
export ROOT_DISK_SIZE_GB=20
export BTC_CHAIN="$BTC_CHAIN"
export ROOT_DISK_SIZE_GB=$ROOT_DISK_SIZE_GB
export WWW_INSTANCE_TYPE="$WWW_INSTANCE_TYPE"
export BTCPAY_ADDITIONAL_HOSTNAMES="$BTCPAY_ADDITIONAL_HOSTNAMES"

7
show_lxc.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
lxc list
lxc network list
lxc profile list
lxc storage list
lxc image list