From d6eb1f30316280e3561906e86c409e806a226f01 Mon Sep 17 00:00:00 2001 From: Derek Smith Date: Fri, 7 Apr 2023 10:23:04 -0400 Subject: [PATCH] Refactorings. --- .gitignore | 1 + defaults.sh | 129 --------- deployment/base.sh | 9 + deployment/create_lxc_base.sh | 92 +++++++ deployment/deploy.sh | 432 ++++++++++++++++++++++++++++++ deployment/deploy_vm.sh | 115 ++++++++ deployment/deployment_defaults.sh | 36 +++ deployment/down.sh | 2 +- deployment/remote.sh | 2 +- deployment/remote_env.sh | 4 +- deployment/stub_lxc_profile.sh | 286 ++++++++++++++++++++ deployment/update.sh | 18 +- install.sh | 35 +-- management/bash_aliases | 2 +- management/provision.sh | 1 + uninstall.sh | 4 +- update.sh | 1 + 17 files changed, 1000 insertions(+), 169 deletions(-) create mode 100644 .gitignore delete mode 100755 defaults.sh create mode 100644 deployment/base.sh create mode 100755 deployment/create_lxc_base.sh create mode 100755 deployment/deploy.sh create mode 100755 deployment/deploy_vm.sh create mode 100644 deployment/deployment_defaults.sh create mode 100755 deployment/stub_lxc_profile.sh create mode 100644 update.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8ff54e8 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +archive \ No newline at end of file diff --git a/defaults.sh b/defaults.sh deleted file mode 100755 index d013979..0000000 --- a/defaults.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - -set -e - -export DEPLOY_GHOST=true -export DEPLOY_CLAMS=false -export DEPLOY_NOSTR=false -export DEPLOY_NEXTCLOUD=false -export DEPLOY_GITEA=false - -export WWW_HOSTNAME="www" -export BTCPAY_HOSTNAME="btcpayserver" -export BTCPAY_HOSTNAME_IN_CERT="btcpay" -export NEXTCLOUD_HOSTNAME="nextcloud" -export GITEA_HOSTNAME="git" -export NOSTR_HOSTNAME="relay" -export CLAMS_HOSTNAME="clams" - -export SITE_LANGUAGE_CODES="en" -export LANGUAGE_CODE="en" -export NOSTR_ACCOUNT_PUBKEY= - -# this is where the html is sourced from. -export SITE_HTML_PATH= -export BTCPAY_ADDITIONAL_HOSTNAMES= - -export GHOST_MYSQL_PASSWORD= -export GHOST_MYSQL_ROOT_PASSWORD= -export NEXTCLOUD_MYSQL_PASSWORD= -export GITEA_MYSQL_PASSWORD= -export NEXTCLOUD_MYSQL_ROOT_PASSWORD= -export GITEA_MYSQL_ROOT_PASSWORD= -export DUPLICITY_BACKUP_PASSPHRASE= -#opt-add-fireflyiii;opt-add-zammad - - -export SSH_HOME="$HOME/.ssh" -export PASS_HOME="$HOME/.password-store" - - -export BTCPAY_SERVER_CPU_COUNT="4" -export BTCPAY_SERVER_MEMORY_MB="4096" -export WWW_SERVER_CPU_COUNT="4" -export WWW_SERVER_MEMORY_MB="4096" - -export DOCKER_IMAGE_CACHE_FQDN="registry-1.docker.io" - -export NEXTCLOUD_SPACE_GB=10 - -DEFAULT_DB_IMAGE="mariadb:10.11.2-jammy" - - -# run the docker stack. -export GHOST_IMAGE="ghost:5.38.0" - -# TODO switch to mysql. May require intricate export work for existing sites. -# THIS MUST BE COMPLETED BEFORE v1 RELEASE -#https://forum.ghost.org/t/how-to-migrate-from-mariadb-10-to-mysql-8/29575 -export GHOST_DB_IMAGE="mysql:8.0.32" - - -export NGINX_IMAGE="nginx:1.23.3" - -# version of backup is 24.0.3 -export NEXTCLOUD_IMAGE="nextcloud:25.0.4" -export NEXTCLOUD_DB_IMAGE="$DEFAULT_DB_IMAGE" - -# TODO PIN the gitea version number. -export GITEA_IMAGE="gitea/gitea:latest" -export GITEA_DB_IMAGE="$DEFAULT_DB_IMAGE" - -export NOSTR_RELAY_IMAGE="scsibug/nostr-rs-relay" - -export WWW_SERVER_MAC_ADDRESS= -export BTCPAYSERVER_MAC_ADDRESS= - -export SS_ROOT_PATH="$HOME/ss" - -export REMOTES_PATH="$SS_ROOT_PATH/remotes" -export PROJECTS_PATH="$SS_ROOT_PATH/projects" -export SITES_PATH="$SS_ROOT_PATH/sites" - -# mount into ss-mgmt/home/ubuntu/snap/lxd/common/config -export LXD_CONFIG_PATH="$SS_ROOT_PATH/lxd" - -# The base VM image. -export LXD_UBUNTU_BASE_VERSION="jammy" -export BASE_IMAGE_VM_NAME="ss-base-${LXD_UBUNTU_BASE_VERSION//./-}" -export BASE_LXC_IMAGE="ubuntu/$LXD_UBUNTU_BASE_VERSION/cloud" -WEEK_NUMBER=$(date +%U) -export UBUNTU_BASE_IMAGE_NAME="ss-ubuntu-${LXD_UBUNTU_BASE_VERSION//./-}" - -export DOCKER_BASE_IMAGE_NAME="ss-docker-${LXD_UBUNTU_BASE_VERSION//./-}-$WEEK_NUMBER" - -export OTHER_SITES_LIST= -export BTCPAY_ALT_NAMES= -export BITCOIN_CHAIN=regtest -export REMOTE_HOME="/home/ubuntu" -export REMOTE_DATA_PATH="$REMOTE_HOME/ss-data" -export REMOTE_DATA_PATH_LETSENCRYPT="$REMOTE_DATA_PATH/letsencrypt" -export REMOTE_BACKUP_PATH="$REMOTE_HOME/backups" -export BTCPAY_SERVER_APPPATH="$REMOTE_DATA_PATH/btcpayserver-docker" - -# this space is for OS, docker images, etc -# values here are fine for regtest generally. Later scripts adjust -# these values based on testnet/mainnet -export WWW_SSDATA_DISK_SIZE_GB=20 -export WWW_BACKUP_DISK_SIZE_GB=50 -export WWW_DOCKER_DISK_SIZE_GB=30 - -export BTCPAYSERVER_SSDATA_DISK_SIZE_GB=20 -export BTCPAYSERVER_BACKUP_DISK_SIZE_GB=5 -export BTCPAYSERVER_DOCKER_DISK_SIZE_GB=30 - -export REGISTRY_URL="https://index.docker.io/v1" - -# this is the git commit of the project/ sub git repo. -# used in the migration script to switch into past for backup -# then back to present (TARGET_PROJECT_GIT_COMMIT) for restore. -export TARGET_PROJECT_GIT_COMMIT=e1754181484007b79ac10aa2f0dd1cd5035b5763 - -# -export TESTNET_BLOCK_HASH=00000000d8277ba1ca66b40b3e3476629e6f0f97c5b8cfaeabfe402e55db223a -export MAINNET_BLOCK_HASH=000000000000000000047941e3a6102e8896a4ae66b962599568eb25abd6b405 - - - -export SS_CACHE_PATH="$SS_ROOT_PATH/cache" -export SS_JAMMY_PATH="$SS_CACHE_PATH/$UBUNTU_BASE_IMAGE_NAME" \ No newline at end of file diff --git a/deployment/base.sh b/deployment/base.sh new file mode 100644 index 0000000..61cae1e --- /dev/null +++ b/deployment/base.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# The base VM image. +export LXD_UBUNTU_BASE_VERSION="jammy" +export BASE_IMAGE_VM_NAME="ss-base-${LXD_UBUNTU_BASE_VERSION//./-}" +export BASE_LXC_IMAGE="ubuntu/$LXD_UBUNTU_BASE_VERSION/cloud" +WEEK_NUMBER=$(date +%U) +export UBUNTU_BASE_IMAGE_NAME="ss-ubuntu-${LXD_UBUNTU_BASE_VERSION//./-}" +export DOCKER_BASE_IMAGE_NAME="ss-docker-${LXD_UBUNTU_BASE_VERSION//./-}-$WEEK_NUMBER" diff --git a/deployment/create_lxc_base.sh b/deployment/create_lxc_base.sh new file mode 100755 index 0000000..549eb78 --- /dev/null +++ b/deployment/create_lxc_base.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -exu +cd "$(dirname "$0")" + +. ./base.sh + +bash -c "./stub_lxc_profile.sh --lxd-hostname=$BASE_IMAGE_VM_NAME" + +if lxc list -q --project default | grep -q "$BASE_IMAGE_VM_NAME" ; then + lxc delete -f "$BASE_IMAGE_VM_NAME" --project=default +fi + +# let's download our base image. +if ! lxc image list --format csv --columns l | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then + # copy the image down from canonical. + lxc image copy "images:$BASE_LXC_IMAGE" "$REMOTE_NAME": --alias "$UBUNTU_BASE_IMAGE_NAME" --public --vm --auto-update +fi + +# If the lxc VM does exist, then we will delete it (so we can start fresh) +if lxc list --format csv -q | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then + # if there's no snapshot, we dispense with the old image and try again. + if ! lxc info "$BASE_IMAGE_VM_NAME" | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then + lxc delete "$BASE_IMAGE_VM_NAME" --force + ssh-keygen -f "$SSH_HOME/known_hosts" -R "$BASE_IMAGE_VM_NAME" + fi +else + # the base image is ubuntu:22.04. + lxc init --profile="$BASE_IMAGE_VM_NAME" "$UBUNTU_BASE_IMAGE_NAME" "$BASE_IMAGE_VM_NAME" --vm --project=default + + # TODO move this sovereign-stack-base construction VM to separate dedicated IP + lxc config set "$BASE_IMAGE_VM_NAME" --project=default + + # for CHAIN in mainnet testnet; do + # for DATA in blocks chainstate; do + # lxc storage volume attach ss-base "$CHAIN-$DATA" "$BASE_IMAGE_VM_NAME" "/home/ubuntu/bitcoin/$DATA" + # done + # done + + lxc start "$BASE_IMAGE_VM_NAME" --project=default + + sleep 15 + while lxc exec "$BASE_IMAGE_VM_NAME" --project=default -- [ ! -f /var/lib/cloud/instance/boot-finished ]; do + sleep 1 + done + + # ensure the ssh service is listening at localhost + lxc exec "$BASE_IMAGE_VM_NAME" --project=default -- wait-for-it -t 100 127.0.0.1:22 + + # # If we have any chaninstate or blocks in our SSME, let's push them to the + # # remote host as a zfs volume that way deployments can share a common history + # # of chainstate/blocks. + # for CHAIN in testnet mainnet; do + # for DATA in blocks chainstate; do + # # if the storage snapshot doesn't yet exist, create it. + # if ! lxc storage volume list ss-base -q --format csv -c n | grep -q "$CHAIN-$DATA/snap0"; then + # DATA_PATH="/home/ubuntu/.ss/cache/bitcoin/$CHAIN/$DATA" + # if [ -d "$DATA_PATH" ]; then + # COMPLETE_FILE_PATH="$DATA_PATH/complete" + # if lxc exec "$BASE_IMAGE_VM_NAME" -- [ ! -f "$COMPLETE_FILE_PATH" ]; then + # lxc file push --recursive --project=default "$DATA_PATH/" "$BASE_IMAGE_VM_NAME""$DATA_PATH/" + # lxc exec "$BASE_IMAGE_VM_NAME" -- su ubuntu - bash -c "echo $(date) > $COMPLETE_FILE_PATH" + # lxc exec "$BASE_IMAGE_VM_NAME" -- chown -R 999:999 "$DATA_PATH/$DATA" + # else + # echo "INFO: it appears as though $CHAIN/$DATA has already been initialized. Continuing." + # fi + # fi + # fi + # done + # done + + # stop the VM and get a snapshot. + lxc stop "$BASE_IMAGE_VM_NAME" --project=default + lxc snapshot "$BASE_IMAGE_VM_NAME" "$UBUNTU_BASE_IMAGE_NAME" --project=default + +fi + +echo "INFO: Publishing '$BASE_IMAGE_VM_NAME' as image '$DOCKER_BASE_IMAGE_NAME'. Please wait." +lxc publish --public "$BASE_IMAGE_VM_NAME/$UBUNTU_BASE_IMAGE_NAME" --project=default --alias="$DOCKER_BASE_IMAGE_NAME" --compression none + +echo "INFO: Success creating the base image. Deleting artifacts from the build process." +lxc delete -f "$BASE_IMAGE_VM_NAME" --project=default + +# # now let's get a snapshot of each of the blocks/chainstate directories. +# for CHAIN in testnet mainnet; do +# for DATA in blocks chainstate; do +# if ! lxc storage volume list ss-base -q --format csv -c n | grep -q "$CHAIN-$DATA/snap0"; then +# echo "INFO: Creating a snapshot 'ss-base/$CHAIN-$DATA/snap0'." +# lxc storage volume snapshot ss-base --project=default "$CHAIN-$DATA" +# fi +# done +# done diff --git a/deployment/deploy.sh b/deployment/deploy.sh new file mode 100755 index 0000000..92b446f --- /dev/null +++ b/deployment/deploy.sh @@ -0,0 +1,432 @@ +#!/bin/bash + +set -exu +cd "$(dirname "$0")" + +LATEST_GIT_COMMIT="$(cat ./project/.git/refs/heads/main)" +export LATEST_GIT_COMMIT="$LATEST_GIT_COMMIT" + +# check to ensure dependencies are met. +for cmd in wait-for-it dig rsync sshfs lxc; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "This script requires \"${cmd}\" to be installed. Please run 'install.sh'." + exit 1 + fi +done + +# do a spot check; if we are on production warn. +if lxc remote get-default | grep -q "production"; then + echo "WARNING: You are running command against a production system!" + echo "" + + # check if there are any uncommited changes. It's dangerous to + # alter production systems when you have commits to make or changes to stash. + if git update-index --refresh | grep -q "needs update"; then + echo "ERROR: You have uncommited changes! You MUST commit or stash all changes to continue." + exit 1 + fi + + RESPONSE= + read -r -p " Are you sure you want to continue (y) ": RESPONSE + if [ "$RESPONSE" != "y" ]; then + echo "STOPPING." + exit 1 + fi + +fi + +PRIMARY_DOMAIN= +RUN_CERT_RENEWAL=true +SKIP_WWW=false +RESTORE_WWW=false +RESTORE_CERTS=false +BACKUP_CERTS=false +BACKUP_BTCPAY=false +BACKUP_CERTS=false +BACKUP_APPS=false +BACKUP_BTCPAY=false +BACKUP_BTCPAY_ARCHIVE_PATH= +RESTORE_BTCPAY=false +SKIP_BTCPAY=false +UPDATE_BTCPAY=false +REMOTE_NAME="$(lxc remote get-default)" +STOP_SERVICES=false +USER_SAYS_YES=false +RESTART_FRONT_END=true + +# grab any modifications from the command line. +for i in "$@"; do + case $i in + --restore-certs) + RESTORE_CERTS=true + shift + ;; + --restore-www) + RESTORE_WWW=true + RESTORE_CERTS=true + + shift + ;; + --restore-btcpay) + RESTORE_BTCPAY=true + shift + ;; + --backup-www) + BACKUP_CERTS=true + BACKUP_APPS=true + shift + ;; + --backup-btcpayserver) + BACKUP_BTCPAY=true + shift + ;; + --stop) + STOP_SERVICES=true + RESTART_FRONT_END=false + shift + ;; + --backup-archive-path=*) + BACKUP_BTCPAY_ARCHIVE_PATH="${i#*=}" + shift + ;; + --update-btcpay) + UPDATE_BTCPAY=true + shift + ;; + --skip-www) + SKIP_WWW=true + shift + ;; + --skip-btcpayserver) + SKIP_BTCPAY=true + shift + ;; + --no-cert-renew) + RUN_CERT_RENEWAL=false + shift + ;; + -y) + USER_SAYS_YES=true + shift + ;; + *) + echo "Unexpected option: $1" + exit 1 + ;; + esac +done + +if [ "$RESTORE_BTCPAY" = true ] && [ -z "$BACKUP_BTCPAY_ARCHIVE_PATH" ]; then + echo "ERROR: Use the '--backup-archive-path=/path/to/btcpay/archive.tar.gz' option when restoring btcpay server." + exit 1 +fi + +if [ "$RESTORE_BTCPAY" = true ] && [ ! -f "$BACKUP_BTCPAY_ARCHIVE_PATH" ]; then + echo "ERROR: The backup archive path you specified DOES NOT exist!" + exit 1 +fi + +. ./remote_env.sh + +export REGISTRY_DOCKER_IMAGE="registry:2" +export RESTORE_WWW="$RESTORE_WWW" +export STOP_SERVICES="$STOP_SERVICES" +export BACKUP_CERTS="$BACKUP_CERTS" +export BACKUP_APPS="$BACKUP_APPS" +export RESTORE_BTCPAY="$RESTORE_BTCPAY" +export BACKUP_BTCPAY="$BACKUP_BTCPAY" +export RUN_CERT_RENEWAL="$RUN_CERT_RENEWAL" +export REMOTE_NAME="$REMOTE_NAME" +export REMOTE_PATH="$REMOTES_PATH/$REMOTE_NAME" +export USER_SAYS_YES="$USER_SAYS_YES" +export BACKUP_BTCPAY_ARCHIVE_PATH="$BACKUP_BTCPAY_ARCHIVE_PATH" +export RESTART_FRONT_END="$RESTART_FRONT_END" +export RESTORE_CERTS="$RESTORE_CERTS" + +# todo convert this to Trezor-T +SSH_PUBKEY_PATH="$SSH_HOME/id_rsa.pub" +export SSH_PUBKEY_PATH="$SSH_PUBKEY_PATH" +if [ ! -f "$SSH_PUBKEY_PATH" ]; then + # generate a new SSH key for the base vm image. + ssh-keygen -f "$SSH_HOME/id_rsa" -t ecdsa -b 521 -N "" +fi + +# ensure our remote path is created. +mkdir -p "$REMOTE_PATH" + +REMOTE_DEFINITION="$REMOTE_PATH/remote.conf" +if [ ! -f "$REMOTE_DEFINITION" ]; then + echo "ERROR: The remote definition could not be found. You may need to re-run 'ss-remote'." + exit 1 +fi + +export REMOTE_DEFINITION="$REMOTE_DEFINITION" +source "$REMOTE_DEFINITION" +export LXD_REMOTE_PASSWORD="$LXD_REMOTE_PASSWORD" +export DEPLOYMENT_STRING="$DEPLOYMENT_STRING" + +# this is our password generation mechanism. Relying on GPG for secure password generation +function new_pass { + gpg --gen-random --armor 1 25 +} + + +function stub_site_definition { + mkdir -p "$SITE_PATH" "$PROJECT_PATH/sites" + + # create a symlink from the PROJECT_PATH/sites/DOMAIN_NAME to the ss-sites/domain name + DOMAIN_SYMLINK_PATH="$PROJECT_PATH/sites/$DOMAIN_NAME" + if [ ! -L "$DOMAIN_SYMLINK_PATH" ]; then + ln -r -s "$SITE_PATH" "$DOMAIN_SYMLINK_PATH" + fi + + if [ ! -f "$SITE_PATH/site.conf" ]; then + # check to see if the enf file exists. exist if not. + SITE_DEFINITION_PATH="$SITE_PATH/site.conf" + if [ ! -f "$SITE_DEFINITION_PATH" ]; then + + # stub out a site.conf with new passwords. + cat >"$SITE_DEFINITION_PATH" <"$PROJECT_DEFINITION_PATH" < /home/ubuntu/.ss-githead" +else + echo "INFO: Skipping www VM." +fi + +export DOMAIN_NAME="$PRIMARY_DOMAIN" +export SITE_PATH="$SITES_PATH/$DOMAIN_NAME" +if [ "$SKIP_BTCPAY" = false ]; then + ./project/btcpayserver/go.sh + + ssh ubuntu@"$BTCPAY_FQDN" "echo $LATEST_GIT_COMMIT > /home/ubuntu/.ss-githead" +else + echo "INFO: Skipping the btcpayserver VM." +fi \ No newline at end of file diff --git a/deployment/deploy_vm.sh b/deployment/deploy_vm.sh new file mode 100755 index 0000000..b9485a3 --- /dev/null +++ b/deployment/deploy_vm.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +set -eu +cd "$(dirname "$0")" + +## This is a weird if clause since we need to LEFT-ALIGN the statement below. +SSH_STRING="Host ${FQDN}" +if ! grep -q "$SSH_STRING" "$SSH_HOME/config"; then + +########## BEGIN +cat >> "$SSH_HOME/config" <<-EOF + +${SSH_STRING} + HostName ${FQDN} + User ubuntu +EOF +### + +fi + +# if the machine doesn't exist, we create it. +if ! lxc list --format csv | grep -q "$LXD_VM_NAME"; then + + # create a base image if needed and instantiate a VM. + if [ -z "$MAC_ADDRESS_TO_PROVISION" ]; then + echo "ERROR: You MUST define a MAC Address for all your machines by setting WWW_SERVER_MAC_ADDRESS, BTCPAYSERVER_MAC_ADDRESS in your site definition." + echo "INFO: IMPORTANT! You MUST have DHCP Reservations for these MAC addresses. You also need records established the DNS." + exit 1 + fi + + # TODO ensure we are only GROWING the volume--never shrinking per zfs volume docs. + VM_ID= + BACKUP_DISK_SIZE_GB= + SSDATA_DISK_SIZE_GB= + DOCKER_DISK_SIZE_GB= + if [ "$VIRTUAL_MACHINE" = www ]; then + VM_ID="w" + BACKUP_DISK_SIZE_GB="$WWW_BACKUP_DISK_SIZE_GB" + SSDATA_DISK_SIZE_GB="$WWW_SSDATA_DISK_SIZE_GB" + DOCKER_DISK_SIZE_GB="$WWW_DOCKER_DISK_SIZE_GB" + fi + + if [ "$VIRTUAL_MACHINE" = btcpayserver ]; then + VM_ID="b" + BACKUP_DISK_SIZE_GB="$BTCPAYSERVER_BACKUP_DISK_SIZE_GB" + SSDATA_DISK_SIZE_GB="$BTCPAYSERVER_SSDATA_DISK_SIZE_GB" + DOCKER_DISK_SIZE_GB="$BTCPAYSERVER_DOCKER_DISK_SIZE_GB" + fi + + DOCKER_VOLUME_NAME="$PRIMARY_DOMAIN_IDENTIFIER-$VM_ID""d" + if ! lxc storage volume list ss-base | grep -q "$DOCKER_VOLUME_NAME"; then + lxc storage volume create ss-base "$DOCKER_VOLUME_NAME" --type=block + fi + + # TODO ensure we are only GROWING the volume--never shrinking + lxc storage volume set ss-base "$DOCKER_VOLUME_NAME" size="${DOCKER_DISK_SIZE_GB}GB" + + SSDATA_VOLUME_NAME="$PRIMARY_DOMAIN_IDENTIFIER-$VM_ID""s" + if ! lxc storage volume list ss-base | grep -q "$SSDATA_VOLUME_NAME"; then + lxc storage volume create ss-base "$SSDATA_VOLUME_NAME" --type=filesystem + fi + + # TODO ensure we are only GROWING the volume--never shrinking per zfs volume docs. + lxc storage volume set ss-base "$SSDATA_VOLUME_NAME" size="${SSDATA_DISK_SIZE_GB}GB" + + + BACKUP_VOLUME_NAME="$PRIMARY_DOMAIN_IDENTIFIER-$VM_ID""b" + if ! lxc storage volume list ss-base | grep -q "$BACKUP_VOLUME_NAME"; then + lxc storage volume create ss-base "$BACKUP_VOLUME_NAME" --type=filesystem + fi + + lxc storage volume set ss-base "$BACKUP_VOLUME_NAME" size="${BACKUP_DISK_SIZE_GB}GB" + + + bash -c "./stub_lxc_profile.sh --vm=$VIRTUAL_MACHINE --lxd-hostname=$LXD_VM_NAME --ss-volume-name=$SSDATA_VOLUME_NAME --backup-volume-name=$BACKUP_VOLUME_NAME" + + # now let's create a new VM to work with. + #lxc init --profile="$LXD_VM_NAME" "$BASE_IMAGE_VM_NAME" "$LXD_VM_NAME" --vm + lxc init "$DOCKER_BASE_IMAGE_NAME" "$LXD_VM_NAME" --vm --profile="$LXD_VM_NAME" + + # let's PIN the HW address for now so we don't exhaust IP + # and so we can set DNS internally. + lxc config set "$LXD_VM_NAME" "volatile.enp5s0.hwaddr=$MAC_ADDRESS_TO_PROVISION" + + # attack the docker block device. + lxc storage volume attach ss-base "$DOCKER_VOLUME_NAME" "$LXD_VM_NAME" + + # if [ "$VIRTUAL_MACHINE" = btcpayserver ]; then + # # attach any volumes + # for CHAIN in testnet mainnet; do + # for DATA in blocks chainstate; do + # MOUNT_PATH="/$CHAIN-$DATA" + # lxc config device add "$LXD_VM_NAME" "$CHAIN-$DATA" disk pool=ss-base source="$CHAIN-$DATA" path="$MOUNT_PATH" + # done + # done + # fi + + lxc start "$LXD_VM_NAME" + sleep 10 + + bash -c "./wait_for_lxc_ip.sh --lxd-name=$LXD_VM_NAME" + + # scan the remote machine and install it's identity in our SSH known_hosts file. + ssh-keyscan -H -t ecdsa "$FQDN" >> "$SSH_HOME/known_hosts" + + ssh "$FQDN" "sudo chown ubuntu:ubuntu $REMOTE_DATA_PATH" + ssh "$FQDN" "sudo chown -R ubuntu:ubuntu $REMOTE_BACKUP_PATH" + + if [ "$VIRTUAL_MACHINE" = btcpayserver ]; then + # send an updated ~/.bashrc so we have quicker access to cli tools + scp ./btcpayserver/bashrc.txt "ubuntu@$FQDN:$REMOTE_HOME/.bashrc" + ssh "$BTCPAY_FQDN" "chown ubuntu:ubuntu $REMOTE_HOME/.bashrc" + ssh "$BTCPAY_FQDN" "chmod 0664 $REMOTE_HOME/.bashrc" + fi +fi \ No newline at end of file diff --git a/deployment/deployment_defaults.sh b/deployment/deployment_defaults.sh new file mode 100644 index 0000000..cb10a98 --- /dev/null +++ b/deployment/deployment_defaults.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -eu + +# file paths +export SSH_HOME="$HOME/.ssh" +export PASS_HOME="$HOME/.password-store" #TODO +export SS_ROOT_PATH="$HOME/ss" +export REMOTES_PATH="$SS_ROOT_PATH/remotes" +export PROJECTS_PATH="$SS_ROOT_PATH/projects" +export SITES_PATH="$SS_ROOT_PATH/sites" +export LXD_CONFIG_PATH="$SS_ROOT_PATH/lxd" +export SS_CACHE_PATH="$SS_ROOT_PATH/cache" + +export BITCOIN_CHAIN=regtest + +# this space is for OS, docker images, etc +# values here are fine for regtest generally. Later scripts adjust +# these values based on testnet/mainnet +export WWW_SSDATA_DISK_SIZE_GB=20 +export WWW_BACKUP_DISK_SIZE_GB=50 +export WWW_DOCKER_DISK_SIZE_GB=30 + +export BTCPAYSERVER_SSDATA_DISK_SIZE_GB=20 +export BTCPAYSERVER_BACKUP_DISK_SIZE_GB=20 +export BTCPAYSERVER_DOCKER_DISK_SIZE_GB=30 + +export WWW_HOSTNAME="www" +export BTCPAY_HOSTNAME="btcpayserver" +export BTCPAY_HOSTNAME_IN_CERT="btcpay" +export NEXTCLOUD_HOSTNAME="nextcloud" +export GITEA_HOSTNAME="git" +export NOSTR_HOSTNAME="relay" +export CLAMS_HOSTNAME="clams" + +export REGISTRY_URL="https://index.docker.io/v1" \ No newline at end of file diff --git a/deployment/down.sh b/deployment/down.sh index 9c04c59..a6a4a83 100755 --- a/deployment/down.sh +++ b/deployment/down.sh @@ -45,7 +45,7 @@ for VIRTUAL_MACHINE in www btcpayserver; do LXD_NAME="$VIRTUAL_MACHINE-${PRIMARY_DOMAIN//./-}" if lxc list | grep -q "$LXD_NAME"; then - bash -c "./project/deploy.sh --stop --skip-$SKIP" + bash -c "./deploy.sh --stop --skip-$SKIP" lxc stop "$LXD_NAME" diff --git a/deployment/remote.sh b/deployment/remote.sh index 01d7d74..fc89ecb 100755 --- a/deployment/remote.sh +++ b/deployment/remote.sh @@ -19,7 +19,7 @@ if [ -z "$REMOTE_NAME" ]; then fi #shellcheck disable=SC1091 -source ../defaults.sh +source ./deployment_defaults.sh export REMOTE_PATH="$REMOTES_PATH/$REMOTE_NAME" REMOTE_DEFINITION="$REMOTE_PATH/remote.conf" diff --git a/deployment/remote_env.sh b/deployment/remote_env.sh index 42e3b4c..14ef8cc 100755 --- a/deployment/remote_env.sh +++ b/deployment/remote_env.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eu +set -exu CURRENT_REMOTE="$(lxc remote get-default)" @@ -25,6 +25,8 @@ if echo "$CURRENT_REMOTE" | grep -q "production"; then fi +. ./deployment_defaults.sh + export REMOTE_PATH="$REMOTES_PATH/$CURRENT_REMOTE" REMOTE_DEFINITION="$REMOTE_PATH/remote.conf" export REMOTE_DEFINITION="$REMOTE_DEFINITION" diff --git a/deployment/stub_lxc_profile.sh b/deployment/stub_lxc_profile.sh new file mode 100755 index 0000000..c4c1be4 --- /dev/null +++ b/deployment/stub_lxc_profile.sh @@ -0,0 +1,286 @@ +#!/bin/bash + +set -eu +cd "$(dirname "$0")" + +VIRTUAL_MACHINE=base +LXD_HOSTNAME= +SSDATA_VOLUME_NAME= +BACKUP_VOLUME_NAME= + +# grab any modifications from the command line. +for i in "$@"; do + case $i in + --lxd-hostname=*) + LXD_HOSTNAME="${i#*=}" + shift + ;; + --vm=*) + VIRTUAL_MACHINE="${i#*=}" + shift + ;; + --ss-volume-name=*) + SSDATA_VOLUME_NAME="${i#*=}" + shift + ;; + --backup-volume-name=*) + BACKUP_VOLUME_NAME="${i#*=}" + shift + ;; + *) + echo "Unexpected option: $1" + exit 1 + ;; + esac +done + +# generate the custom cloud-init file. Cloud init installs and configures sshd +SSH_AUTHORIZED_KEY=$(<"$SSH_PUBKEY_PATH") +eval "$(ssh-agent -s)" +ssh-add "$SSH_HOME/id_rsa" +export SSH_AUTHORIZED_KEY="$SSH_AUTHORIZED_KEY" + +export FILENAME="$LXD_HOSTNAME.yml" +mkdir -p "$PROJECT_PATH/cloud-init" +YAML_PATH="$PROJECT_PATH/cloud-init/$FILENAME" + +# If we are deploying the www, we attach the vm to the underlay via macvlan. +cat > "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" <> "$YAML_PATH" </dev/null 2>&1; then exit 1 fi -. ./defaults.sh - if lxc list --format csv | grep -q "ss-mgmt"; then if lxc list --format csv -q | grep -q "ss-mgmt,RUNNING"; then @@ -61,6 +59,8 @@ if [ "$PURGE_LXD" = true ]; then lxc network delete lxdbr0 fi + # this file contains the BASE_IMAGE_NAME + . ./deployment/base.sh if lxc image list | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then lxc image delete "$UBUNTU_BASE_IMAGE_NAME" fi diff --git a/update.sh b/update.sh new file mode 100644 index 0000000..a9bf588 --- /dev/null +++ b/update.sh @@ -0,0 +1 @@ +#!/bin/bash