Refactoring/cleaup.

defaults.sh

@@ -5,7 +5,7 @@ set -eu
 export WWW_SERVER_MAC_ADDRESS=
 export DEPLOY_WWW_SERVER=false
 export DEPLOY_BTCPAY_SERVER=false
-export DEPLOY_GHOST=true
+export DEPLOY_GHOST=false
 export DEPLOY_NOSTR_RELAY=false
 export DEPLOY_ONION_SITE=false
 export DEPLOY_NEXTCLOUD=false
@@ -19,6 +19,7 @@ export GITEA_HOSTNAME="git"
 export NOSTR_HOSTNAME="relay"
 
 export SITE_LANGUAGE_CODES="en"
+export LANGUAGE_CODE="en"
 export NOSTR_ACCOUNT_PUBKEY=
 
 # this is where the html is sourced from.
@@ -38,8 +39,8 @@ export DUPLICITY_BACKUP_PASSPHRASE=
 export SSH_HOME="$HOME/.ssh"
 export VLAN_INTERFACE=
 export VM_NAME="sovereign-stack-base"
-export DEV_MEMORY_MB="4096"
+export DEV_MEMORY_MB="8096"
-export DEV_CPU_COUNT="4"
+export DEV_CPU_COUNT="6"
 
 export DOCKER_IMAGE_CACHE_FQDN="registry-1.docker.io"
 

deploy.sh

@@ -1,8 +1,11 @@
 #!/bin/bash
 
-set -eu
+set -exu
 cd "$(dirname "$0")"
 
+RESPOSITORY_PATH="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+export RESPOSITORY_PATH="$RESPOSITORY_PATH"
+
 check_dependencies () {
   for cmd in "$@"; do
     if ! command -v "$cmd" >/dev/null 2>&1; then
@@ -25,14 +28,12 @@ DOMAIN_NAME=
 RUN_CERT_RENEWAL=true
 SKIP_WWW=false
 RESTORE_WWW=false
-BACKUP_CERTS=true
+BACKUP_CERTS=false
-BACKUP_APPS=true
+BACKUP_APPS=false
-BACKUP_BTCPAY=true
+BACKUP_BTCPAY=false
 RESTORE_BTCPAY=false
 BTCPAY_RESTORE_ARCHIVE_PATH=
 BTCPAY_LOCAL_BACKUP_PATH=
-MIGRATE_WWW=false
-MIGRATE_BTCPAY=false
 SKIP_BTCPAY=false
 UPDATE_BTCPAY=false
 RECONFIGURE_BTCPAY_SERVER=false
@@ -90,17 +91,6 @@ for i in "$@"; do
             BTCPAY_RESTORE_ARCHIVE_PATH="${i#*=}"
             shift
         ;;
-        --migrate-www)
-            MIGRATE_WWW=true
-            RUN_CERT_RENEWAL=false
-            shift
-        ;;
-        --migrate-btcpay)
-            MIGRATE_BTCPAY=true
-            BACKUP_BTCPAY=true
-            RUN_CERT_RENEWAL=false
-            shift
-        ;;
         --renew-certs)
             RUN_CERT_RENEWAL=true
             shift
@@ -136,8 +126,6 @@ export BACKUP_CERTS="$BACKUP_CERTS"
 export BACKUP_APPS="$BACKUP_APPS"
 export RESTORE_BTCPAY="$RESTORE_BTCPAY"
 export BACKUP_BTCPAY="$BACKUP_BTCPAY"
-export MIGRATE_WWW="$MIGRATE_WWW"
-export MIGRATE_BTCPAY="$MIGRATE_BTCPAY"
 export RUN_CERT_RENEWAL="$RUN_CERT_RENEWAL"
 export CLUSTER_NAME="$CLUSTER_NAME"
 export CLUSTER_PATH="$CLUSTERS_DIR/$CLUSTER_NAME"
@@ -167,50 +155,11 @@ fi
 
 source "$CLUSTER_DEFINITION"
 
-###########################3
-# # This section is done to the management machine. We deploy a registry pull through cache on port 5000
-# if ! docker volume list | grep -q registry_data; then
-#     docker volume create registry_data
-# fi
-
-# if the registry URL isn't defined, then we just use the upstream dockerhub.
-# recommended to run a registry cache on your management machine though.
-# if [ -n "$REGISTRY_URL" ]; then
-
-# cat > "$CLUSTER_PATH/registry.yml" <<EOL
-# version: 0.1
-# http:
-#   addr: 0.0.0.0:5000
-#   host: ${REGISTRY_URL}
-
-# proxy:
-#     remoteurl: ${REGISTRY_URL}
-#     username: ${REGISTRY_USERNAME}
-#     password: ${REGISTRY_PASSWORD}
-# EOL
-
-#     # enable docker swarm mode so we can support docker stacks.
-#     if docker info | grep -q "Swarm: inactive"; then
-#         docker swarm init
-#     fi
-
-#     mkdir -p "${CACHES_DIR}/registry_images"
-
-#     # run a docker registry pull through cache on the management machine.
-#     if [ "$DEPLOY_MGMT_REGISTRY" = true ]; then
-#         if ! docker stack list | grep -q registry; then
-#             docker stack deploy -c management/registry_mirror.yml registry
-#         fi
-#     fi
-# fi
-
-
 # this is our password generation mechanism. Relying on GPG for secure password generation
 function new_pass {
     gpg --gen-random --armor 1 25
 }
 
-
 function instantiate_vms {
 
     export BTC_CHAIN="$BTC_CHAIN"
@@ -227,7 +176,7 @@ function instantiate_vms {
         export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
 
         source "$SITE_PATH/site_definition"
-        source ./domain_env.sh
+        source "$RESPOSITORY_PATH/domain_env.sh"
 
         # VALIDATE THE INPUT from the ENVFILE
         if [ -z "$DOMAIN_NAME" ]; then
@@ -239,7 +188,7 @@ function instantiate_vms {
         # first let's get the DISK_TO_USE and DATA_PLANE_MACVLAN_INTERFACE from the ss-config
         # which is set up during LXD cluster creation ss-cluster.
         LXD_SS_CONFIG_LINE=
-        if lxc network list --format csv | grep lxdbrSS | grep ss-config; then
+        if lxc network list --format csv | grep lxdbrSS | grep -q ss-config; then
             LXD_SS_CONFIG_LINE="$(lxc network list --format csv | grep lxdbrSS | grep ss-config)"
         fi
 
@@ -257,8 +206,6 @@ function instantiate_vms {
 
         ./deployment/create_lxc_base.sh
 
-
-
         export MAC_ADDRESS_TO_PROVISION=
         export VPS_HOSTNAME="$VPS_HOSTNAME"
         export FQDN="$VPS_HOSTNAME.$DOMAIN_NAME"
@@ -270,7 +217,7 @@ function instantiate_vms {
         fi
 
         DDNS_HOST=
-        MIGRATE_VPS=false
+
         if [ "$VIRTUAL_MACHINE" = www ]; then
             if [ "$SKIP_WWW" = true ]; then
                 echo "INFO: Skipping WWW due to command line argument."
@@ -281,9 +228,6 @@ function instantiate_vms {
             MAC_ADDRESS_TO_PROVISION="$WWW_SERVER_MAC_ADDRESS"
             DDNS_HOST="$WWW_HOSTNAME"
             ROOT_DISK_SIZE_GB="$((ROOT_DISK_SIZE_GB + NEXTCLOUD_SPACE_GB))"
-            if [ "$MIGRATE_WWW" = true ]; then
-                MIGRATE_VPS=true
-            fi
         elif [ "$VIRTUAL_MACHINE" = btcpayserver ] || [ "$SKIP_BTCPAY" = true ]; then
             DDNS_HOST="$BTCPAY_HOSTNAME"
             VPS_HOSTNAME="$BTCPAY_HOSTNAME"
@@ -294,10 +238,6 @@ function instantiate_vms {
                 ROOT_DISK_SIZE_GB=70
             fi
 
-            if [ "$MIGRATE_BTCPAY" = true ]; then
-                MIGRATE_VPS=true
-            fi
-
         elif [ "$VIRTUAL_MACHINE" = "sovereign-stack" ]; then
             DDNS_HOST="sovereign-stack-base"
             ROOT_DISK_SIZE_GB=8
@@ -315,45 +255,8 @@ function instantiate_vms {
         export BTCPAY_LOCAL_BACKUP_PATH="$SITE_PATH/backups/btcpayserver/$BACKUP_TIMESTAMP"
         export BTCPAY_LOCAL_BACKUP_ARCHIVE_PATH="$BTCPAY_LOCAL_BACKUP_PATH/$UNIX_BACKUP_TIMESTAMP.tar.gz"
 
-        MACHINE_EXISTS=false
-        if lxc list --format csv | grep -q "$FQDN"; then
-            MACHINE_EXISTS=true
-        fi
-
-        if [ "$MACHINE_EXISTS"  = true ]; then
-            # we delete the machine if the user has directed us to
-            # but before we do, we get a backup of applcation data of the running instance/vm
-            # that backup becomes the basis for restoring to the newer version of the host host (Type-1 VM)
-            if [ "$MIGRATE_VPS" = true ]; then
-
-                # get a backup of the machine. This is what we restore to the new VPS.
-                echo "INFO: Machine exists.  Since we're going to delete it, let's grab a backup. "
-                echo "      We don't need to restore services since we're deleting it."
         ./deployment/deploy_vms.sh
 
-                # delete the remote VPS.
-                lxc delete --force "$LXD_VM_NAME"
-
-                # Then we run the script again to re-instantiate a new VPS, restoring all user data
-                # if restore directory doesn't exist, then we end up with a new site.
-                echo "INFO: Recreating the remote VPS then restoring user data."
-                sleep 2
-
-                ./deployment/deploy_vms.sh
-            else
-                ./deployment/deploy_vms.sh
-            fi
-        else
-            if [ "$MIGRATE_VPS" = true ]; then
-                echo "INFO: User has indicated a migration, but it doesn't exist. Try removing some command line arguments."
-                exit 1
-            fi
-
-            # The machine does not exist. Let's bring it into existence, restoring from latest backup.
-            echo "Machine does not exist. Creating."
-            ./deployment/deploy_vms.sh
-        fi
-
         # if the local docker client isn't logged in, do so;
         # this helps prevent docker pull errors since they throttle.
         if [ ! -f "$HOME/.docker/config.json" ]; then
@@ -418,8 +321,28 @@ EOL
 
 }
 
+CURRENT_PROJECT="$(lxc info | grep "project:" | awk '{print $2}')"
+PROJECT_PATH="$PROJECTS_DIR/$PROJECT_NAME"
+mkdir -p "$PROJECT_PATH" "$CLUSTER_PATH/projects"
+export PROJECT_PATH="$PROJECT_PATH"
+
+# create a symlink from ./clusterpath/projects/project
+if [ ! -d "$CLUSTER_PATH/projects/$PROJECT_NAME" ]; then
+    ln -s "$PROJECT_PATH" "$CLUSTER_PATH/projects/$PROJECT_NAME"
+fi
+
+# check if we need to provision a new lxc project.
+if [ "$PROJECT_NAME" != "$CURRENT_PROJECT" ]; then
+    if ! lxc project list | grep -q "$PROJECT_NAME"; then
+        echo "INFO: The lxd project specified in the cluster_definition did not exist. We'll create one!"
+        lxc project create "$PROJECT_NAME"
+    fi
+
+    echo "INFO: switch to lxd project '$PROJECT_NAME'."
+    lxc project switch "$PROJECT_NAME"
+
+fi
 
-function stub_project_definition {
 
 # check to see if the enf file exists. exist if not.
 PROJECT_DEFINITION_PATH="$PROJECT_PATH/project_definition"
@@ -448,37 +371,10 @@ EOL
 
 # source project defition.
 source "$PROJECT_DEFINITION_PATH"
-}
-
-
-CURRENT_PROJECT="$(lxc info | grep "project:" | awk '{print $2}')"
-PROJECT_PATH="$PROJECTS_DIR/$PROJECT_NAME"
-mkdir -p "$PROJECT_PATH" "$CLUSTER_PATH/projects"
-export PROJECT_PATH="$PROJECT_PATH"
-
-# create a symlink from ./clusterpath/projects/project
-if [ ! -d "$CLUSTER_PATH/projects/$PROJECT_NAME" ]; then
-    ln -s "$PROJECT_PATH" "$CLUSTER_PATH/projects/$PROJECT_NAME"
-fi
-
-# check if we need to provision a new lxc project.
-if [ "$PROJECT_NAME" != "$CURRENT_PROJECT" ]; then
-    if ! lxc project list | grep -q "$PROJECT_NAME"; then
-        echo "INFO: The lxd project specified in the cluster_definition did not exist. We'll create one!"
-        lxc project create "$PROJECT_NAME"
-    fi
-
-    echo "INFO: switch to lxd project '$PROJECT_NAME'."
-    lxc project switch "$PROJECT_NAME"
-
-fi
-
-# stub out the project definition if needed.
-stub_project_definition
 
 # the DOMAIN_LIST is a complete list of all our domains. We often iterate over this list.
 export DOMAIN_LIST="${PRIMARY_DOMAIN},${OTHER_SITES_LIST}"
-export DOMAIN_COUNT=$(("$(echo $DOMAIN_LIST | tr -cd , | wc -c)"+1))
+export DOMAIN_COUNT=$(("$(echo "$DOMAIN_LIST" | tr -cd , | wc -c)"+1))
 
 # let's provision our primary domain first.
 export DOMAIN_NAME="$PRIMARY_DOMAIN"

deployment/btcpayserver/backup_btcpay.sh

@@ -11,6 +11,10 @@ echo "INFO: Starting BTCPAY Backup script for host '$BTCPAY_FQDN'."
 
 ssh "$BTCPAY_FQDN" "mkdir -p $REMOTE_HOME/backups; cd $REMOTE_HOME/; sudo BTCPAY_BASE_DIRECTORY=$REMOTE_HOME bash -c $BTCPAY_SERVER_APPPATH/btcpay-down.sh"
 
+# TODO; not sure if this is necessary, but we want to give the VM additional time to take down all services
+# that way processes can run shutdown procedures and leave files in the correct state.
+sleep 10
+
 # TODO enable encrypted archives
 # TODO switch to btcpay-backup.sh when on LXD fully.
 scp ./remote_scripts/btcpay-backup.sh "$BTCPAY_FQDN:$REMOTE_HOME/btcpay-backup.sh"

deployment/www/generate_certs.sh

@@ -12,9 +12,9 @@ for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
     export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
 
     # source the site path so we know what features it has.
-    source ../../reset_env.sh
+    source "$RESPOSITORY_PATH/reset_env.sh"
     source "$SITE_PATH/site_definition"
-    source ../../domain_env.sh
+    source "$RESPOSITORY_PATH/domain_env.sh"
 
     # with the lxd side, we are trying to expose ALL OUR services from one IP address, which terminates
     # at a cachehing reverse proxy that runs nginx.

deployment/www/go.sh

@@ -14,9 +14,9 @@ for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
     export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
 
     # source the site path so we know what features it has.
-    source ../../reset_env.sh
+    source "$RESPOSITORY_PATH/reset_env.sh"
     source "$SITE_PATH/site_definition"
-    source ../../domain_env.sh
+    source "$RESPOSITORY_PATH/domain_env.sh"
 
 
     ### Let's check to ensure all the requiredsettings are set.

deployment/www/stop_docker_stacks.sh

@@ -9,9 +9,9 @@ for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
     export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
 
     # source the site path so we know what features it has.
-    source ../../reset_env.sh
+    source "$RESPOSITORY_PATH/reset_env.sh"
     source "$SITE_PATH/site_definition"
-    source ../../domain_env.sh
+    source "$RESPOSITORY_PATH/domain_env.sh"
 
     ### Stop all services.
     for APP in ghost nextcloud gitea nostr; do
@@ -48,22 +48,25 @@ for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
     done
 done
 
-
+# remove the nginx stack
 if docker stack list --format "{{.Name}}" | grep -q reverse-proxy; then
     sleep 2
 
     docker stack rm reverse-proxy
 
-    if [ "$STOP_SERVICES" = true ]; then
-        echo "STOPPING as indicated by the --stop flag."
-        exit 1
-    fi
-
     # wait for all docker containers to stop.
     # TODO see if there's a way to check for this.
     sleep 15
 fi
 
+# 
+if [ "$STOP_SERVICES" = true ]; then
+    echo "STOPPING as indicated by the --stop flag."
+    
+    
+    exit 1
+fi
+
 # generate the certs and grab a backup
 if [ "$RUN_CERT_RENEWAL" = true ]; then
     ./generate_certs.sh
@@ -72,12 +75,8 @@ fi
 # Back each domain's certificates under /home/ubuntu/letsencrypt/domain
 for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
     export DOMAIN_NAME="$DOMAIN_NAME"
-    export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
 
-    # source the site path so we know what features it has.
+    source "$RESPOSITORY_PATH/domain_env.sh"
-    source ../../reset_env.sh
-    source "$SITE_PATH/site_definition"
-    source ../../domain_env.sh
 
     # these variable are used by both backup/restore scripts.
     export APP="letsencrypt"

deployment/www/stub/ghost_yml.sh

@@ -8,9 +8,9 @@ for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
     export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
 
     # source the site path so we know what features it has.
-    source ../../../reset_env.sh
+    source "$RESPOSITORY_PATH/reset_env.sh"
     source "$SITE_PATH/site_definition"
-    source ../../../domain_env.sh
+    source "$RESPOSITORY_PATH/domain_env.sh"
 
     # for each language specified in the site_definition, we spawn a separate ghost container
     # at https://www.domain.com/$LANGUAGE_CODE

deployment/www/stub/gitea_yml.sh

@@ -8,9 +8,9 @@ for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
     export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
 
     # source the site path so we know what features it has.
-    source ../../../reset_env.sh
+    source "$RESPOSITORY_PATH/reset_env.sh"
     source "$SITE_PATH/site_definition"
-    source ../../../domain_env.sh
+    source "$RESPOSITORY_PATH/domain_env.sh"
 
     if [ "$DEPLOY_GITEA" = true ]; then
         GITEA_PATH="$REMOTE_GITEA_PATH/$DOMAIN_NAME/${LANGUAGE_CODE}"

deployment/www/stub/nextcloud_yml.sh

@@ -8,9 +8,9 @@ for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
     export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
 
     # source the site path so we know what features it has.
-    source ../../../reset_env.sh
+    source "$RESPOSITORY_PATH/reset_env.sh"
     source "$SITE_PATH/site_definition"
-    source ../../../domain_env.sh
+    source "$RESPOSITORY_PATH/domain_env.sh"
 
     # ensure remote directories exist
     if [ "$DEPLOY_NEXTCLOUD" = true ]; then

deployment/www/stub/nginx_config.sh

@@ -20,9 +20,9 @@ for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
     export CONTAINER_TLS_PATH="/etc/letsencrypt/${DOMAIN_NAME}/live/${DOMAIN_NAME}"
     
     # source the site path so we know what features it has.
-    source ../../../reset_env.sh
+    source "$RESPOSITORY_PATH/reset_env.sh"
     source "$SITE_PATH/site_definition"
-    source ../../../domain_env.sh
+    source "$RESPOSITORY_PATH/domain_env.sh"
 
     if [ $iteration = 0 ]; then
         cat >>"$NGINX_CONF_PATH" <<EOL

deployment/www/stub/nginx_yml.sh

@@ -23,9 +23,9 @@ EOL
         export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
 
         # source the site path so we know what features it has.
-        source ../../../reset_env.sh
+        source "$RESPOSITORY_PATH/reset_env.sh"
         source "$SITE_PATH/site_definition"
-        source ../../../domain_env.sh
+        source "$RESPOSITORY_PATH/domain_env.sh"
 
 
         for LANGUAGE_CODE in ${SITE_LANGUAGE_CODES//,/ }; do
@@ -85,12 +85,7 @@ EOL
 
     for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
         export DOMAIN_NAME="$DOMAIN_NAME"
-        export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
+        source "$RESPOSITORY_PATH/domain_env.sh"
-
-        # source the site path so we know what features it has.
-        source ../../../reset_env.sh
-        source "$SITE_PATH/site_definition"
-        source ../../../domain_env.sh
 
         # for each language specified in the site_definition, we spawn a separate ghost container
         # at https://www.domain.com/$LANGUAGE_CODE

management/registry_mirror.yml

@@ -1,54 +0,0 @@
-version: '3.8'
-services:
-  registry-mirror:
-    image: ${REGISTRY_DOCKER_IMAGE}
-    ports:
-      - 5000:5000
-    volumes:
-      - ${CACHES_DIR}/registry_images:/data
-    environment:
-      REGISTRY_HTTP_ADDR: :5000
-      REGISTRY_HTTP_HOST: http://0.0.0.0:5000
-      REGISTRY_HTTP_SECRET: ${REGISTRY_HTTP_SECRET:-CHANGEME}
-      REGISTRY_PROXY_REMOTEURL: ${REGISTRY_PROXY_REMOTEURL:-https://registry-1.docker.io}
-      REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /data
-    configs:
-      - source: registry-config
-        target: /etc/docker/registry/config.yml
-    deploy:
-      mode: replicated
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-    # secrets:
-    #   - source: regmirror_tls_cert
-    #     target: /var/lib/registry/regmirror:5000.cert
-    #     uid: '0'
-    #     gid: '0'
-    #     mode: 0400
-    #   - source: regmirror_tls_key
-    #     target: /var/lib/registry/regmirror:5000.key
-    #     uid: '0'
-    #     gid: '0'
-    #     mode: 0400
-    #   - source: regmirror_ca_crt
-    #     target: /var/lib/registry/ca.crt
-    #     uid: '0'
-    #     gid: '0'
-    #     mode: 0400
-
-
-# secrets:
-#   regmirror_tls_cert:
-#     file: regmirror.cert
-#   regmirror_tls_key:
-#     file: regmirror.key
-#   regmirror_ca_crt:
-#     file: ca.crt
-
-configs:
-  registry-config:
-    file: ${CLUSTER_PATH}/registry.yml
-
-volumes:
-  registry_data: