Update project.

.gitignore

@@ -0,0 +1 @@
+archive

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "deployment/project"]
+	path = deployment/project
+	url = https://git.sovereign-stack.org/ss/project

.vscode/settings.json

@@ -30,8 +30,8 @@
         "titleBar.inactiveBackground": "#02972799",
         "titleBar.activeForeground": "#e7e7e7",
         "titleBar.inactiveForeground": "#e7e7e799",
-        "statusBar.background": "#ff9900",
-        "statusBarItem.hoverBackground": "#ff9900",
+        "statusBar.background": "#d68100",
+        "statusBarItem.hoverBackground": "#c77700",
         "statusBar.foreground": "#000000"
     }
 }

README.md

@@ -1,3 +1,3 @@
 # Documentation
 
-All documentation for this project can be found at [sovereign-stack.org](https://www.sovereign-stack.org).
+All documentation for this project can be found at [sovereign-stack.org](https://www.sovereign-stack.org). To get started with this code, check out [this post](https://www.sovereign-stack.org/get/).

defaults.sh

@@ -1,104 +0,0 @@
-#!/bin/bash
-
-set -e
-
-export DEPLOY_GHOST=true
-export DEPLOY_CLAMS=true
-
-export DEPLOY_NEXTCLOUD=false
-export DEPLOY_GITEA=false
-
-export WWW_HOSTNAME="www"
-export BTCPAY_HOSTNAME="btcpayserver"
-export BTCPAY_HOSTNAME_IN_CERT="btcpay"
-export NEXTCLOUD_HOSTNAME="nextcloud"
-export GITEA_HOSTNAME="git"
-export NOSTR_HOSTNAME="relay"
-export CLAMS_HOSTNAME="clams"
-
-export SITE_LANGUAGE_CODES="en"
-export LANGUAGE_CODE="en"
-export NOSTR_ACCOUNT_PUBKEY=
-
-# this is where the html is sourced from.
-export SITE_HTML_PATH=
-export BTCPAY_ADDITIONAL_HOSTNAMES=
-
-export GHOST_MYSQL_PASSWORD=
-export GHOST_MYSQL_ROOT_PASSWORD=
-export NEXTCLOUD_MYSQL_PASSWORD=
-export GITEA_MYSQL_PASSWORD=
-export NEXTCLOUD_MYSQL_ROOT_PASSWORD=
-export GITEA_MYSQL_ROOT_PASSWORD=
-export DUPLICITY_BACKUP_PASSPHRASE=
-#opt-add-fireflyiii;opt-add-zammad
-
-
-export SSH_HOME="$HOME/.ssh"
-export PASS_HOME="$HOME/.password-store"
-
-
-export BTCPAY_SERVER_CPU_COUNT="4"
-export BTCPAY_SERVER_MEMORY_MB="4096"
-export WWW_SERVER_CPU_COUNT="4"
-export WWW_SERVER_MEMORY_MB="4096"
-
-export DOCKER_IMAGE_CACHE_FQDN="registry-1.docker.io"
-
-export NEXTCLOUD_SPACE_GB=10
-
-DEFAULT_DB_IMAGE="mariadb:10.9.3-jammy"
-
-
-# run the docker stack.
-export GHOST_IMAGE="ghost:5.38.0"
-
-# TODO switch to mysql. May require intricate export work for existing sites. 
-# THIS MUST BE COMPLETED BEFORE v1 RELEASE
-#https://forum.ghost.org/t/how-to-migrate-from-mariadb-10-to-mysql-8/29575
-export GHOST_DB_IMAGE="$DEFAULT_DB_IMAGE"
-
-
-export NGINX_IMAGE="nginx:1.23.3"
-
-# version of backup is 24.0.3
-export NEXTCLOUD_IMAGE="nextcloud:25.0.4"
-export NEXTCLOUD_DB_IMAGE="$DEFAULT_DB_IMAGE"
-
-# TODO PIN the gitea version number.
-export GITEA_IMAGE="gitea/gitea:latest"
-export GITEA_DB_IMAGE="$DEFAULT_DB_IMAGE"
-
-export NOSTR_RELAY_IMAGE="scsibug/nostr-rs-relay"
-
-export WWW_SERVER_MAC_ADDRESS=
-export BTCPAYSERVER_MAC_ADDRESS=
-
-export REMOTES_DIR="$HOME/ss-remotes"
-export PROJECTS_DIR="$HOME/ss-projects"
-export SITES_PATH="$HOME/ss-sites"
-
-# The base VM image.
-export LXD_UBUNTU_BASE_VERSION="jammy"
-export BASE_IMAGE_VM_NAME="ss-base-${LXD_UBUNTU_BASE_VERSION//./-}"
-export BASE_LXC_IMAGE="ubuntu/$LXD_UBUNTU_BASE_VERSION/cloud"
-export UBUNTU_BASE_IMAGE_NAME="ss-ubuntu-${LXD_UBUNTU_BASE_VERSION//./-}"
-export DOCKER_BASE_IMAGE_NAME="ss-docker-${LXD_UBUNTU_BASE_VERSION//./-}"
-
-export OTHER_SITES_LIST=
-export BTCPAY_ALT_NAMES=
-export BITCOIN_CHAIN=regtest
-export REMOTE_HOME="/home/ubuntu"
-
-export BTCPAY_SERVER_APPPATH="$REMOTE_HOME/btcpayserver-docker"
-export REMOTE_CERT_BASE_DIR="$REMOTE_HOME/.certs"
-
-# this space is for OS, docker images, etc. DOES NOT INCLUDE USER DATA.
-export ROOT_DISK_SIZE_GB=20
-export REGISTRY_URL="https://index.docker.io/v1/"
-export PRIMARY_DOMAIN=
-
-# this is the git commit of the project/ sub git repo.
-# used in the migration script to switch into past for backup
-# then back to present (TARGET_PROJECT_GIT_COMMIT) for restore.
-export TARGET_PROJECT_GIT_COMMIT=29acc1796dca29f56f80005f869d3bdc1faf6c58

deployment/.gitignore

@@ -1 +0,0 @@
-project

deployment/base.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# The base VM image.
+export INCUS_UBUNTU_BASE_VERSION="jammy"
+export BASE_IMAGE_VM_NAME="ss-base-${INCUS_UBUNTU_BASE_VERSION//./-}"
+export BASE_INCUS_IMAGE="ubuntu/$INCUS_UBUNTU_BASE_VERSION/cloud"
+WEEK_NUMBER=$(date +%U)
+export UBUNTU_BASE_IMAGE_NAME="ss-ubuntu-${INCUS_UBUNTU_BASE_VERSION//./-}"
+export DOCKER_BASE_IMAGE_NAME="ss-docker-${INCUS_UBUNTU_BASE_VERSION//./-}-$WEEK_NUMBER"

deployment/create_base.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+set -exu
+cd "$(dirname "$0")"
+
+. ./base.sh
+
+bash -c "./stub_profile.sh --incus-hostname=$BASE_IMAGE_VM_NAME"
+
+if incus list -q --project default | grep -q "$BASE_IMAGE_VM_NAME" ; then
+    incus delete -f "$BASE_IMAGE_VM_NAME" --project default
+fi
+
+# let's download our base image.
+if ! incus image list --format csv --columns l --project default | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then
+    # copy the image down from canonical.
+    incus image copy "images:$BASE_INCUS_IMAGE" "$REMOTE_NAME": --alias "$UBUNTU_BASE_IMAGE_NAME" --public --vm --auto-update --target-project default
+fi
+
+# If the VM does exist, then we will delete it (so we can start fresh) 
+if incus list --format csv -q --project default | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then
+    # if there's no snapshot, we dispense with the old image and try again.
+    if ! incus info "$BASE_IMAGE_VM_NAME"  --project default | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then
+        incus delete "$BASE_IMAGE_VM_NAME" --force --project default
+        ssh-keygen -f "$SSH_HOME/known_hosts" -R "$BASE_IMAGE_VM_NAME"
+    fi
+else
+
+    if ! incus list --project default | grep -q "$BASE_IMAGE_VM_NAME"; then
+        # the base image is ubuntu:22.04.
+        script -q -c "incus init -q --profile=$BASE_IMAGE_VM_NAME $UBUNTU_BASE_IMAGE_NAME $BASE_IMAGE_VM_NAME --vm --project default" /dev/null
+    fi
+
+    if incus info "$BASE_IMAGE_VM_NAME" --project default | grep -q "Status: STOPPED"; then
+        # TODO move this sovereign-stack-base construction VM to separate dedicated IP
+        incus config set "$BASE_IMAGE_VM_NAME" --project default
+        incus start "$BASE_IMAGE_VM_NAME" --project default
+        sleep 15
+    fi
+
+    # for CHAIN in mainnet testnet; do
+    #     for DATA in blocks chainstate; do
+    #         incus storage volume attach ss-base "$CHAIN-$DATA" "$BASE_IMAGE_VM_NAME" "/home/ubuntu/bitcoin/$DATA"
+    #     done
+    # done
+
+    if incus info "$BASE_IMAGE_VM_NAME" --project default | grep -q "Status: RUNNING"; then
+
+        while incus exec "$BASE_IMAGE_VM_NAME" --project default -- [ ! -f /var/lib/cloud/instance/boot-finished ]; do
+            sleep 1
+        done
+
+        # ensure the ssh service is listening at localhost
+        incus exec "$BASE_IMAGE_VM_NAME" --project default -- wait-for-it -t 100 127.0.0.1:22
+
+        # # If we have any chaninstate or blocks in our SSME, let's push them to the
+        # # remote host as a zfs volume that way deployments can share a common history
+        # # of chainstate/blocks.
+        # for CHAIN in testnet mainnet; do
+        #     for DATA in blocks chainstate; do
+        #         # if the storage snapshot doesn't yet exist, create it.
+        #         if ! incus storage volume list ss-base -q --format csv -c n | grep -q "$CHAIN-$DATA/snap0"; then
+        #             DATA_PATH="/home/ubuntu/.ss/cache/bitcoin/$CHAIN/$DATA"
+        #             if [ -d "$DATA_PATH" ]; then
+        #                 COMPLETE_FILE_PATH="$DATA_PATH/complete"
+        #                 if incus exec "$BASE_IMAGE_VM_NAME" -- [ ! -f "$COMPLETE_FILE_PATH" ]; then
+        #                     incus file push --recursive --project default "$DATA_PATH/" "$BASE_IMAGE_VM_NAME""$DATA_PATH/"
+        #                     incus exec "$BASE_IMAGE_VM_NAME" -- su ubuntu - bash -c "echo $(date) > $COMPLETE_FILE_PATH"
+        #                     incus exec "$BASE_IMAGE_VM_NAME" -- chown -R 999:999 "$DATA_PATH/$DATA"
+        #                 else
+        #                     echo "INFO: it appears as though $CHAIN/$DATA has already been initialized. Continuing."
+        #                 fi
+        #             fi
+        #         fi
+        #     done
+        # done
+
+        # stop the VM and get a snapshot.
+        incus stop "$BASE_IMAGE_VM_NAME" --project default
+    fi
+    
+    incus snapshot create "$BASE_IMAGE_VM_NAME" "$UBUNTU_BASE_IMAGE_NAME" --project default
+
+fi
+
+echo "INFO: Publishing '$BASE_IMAGE_VM_NAME' as image '$DOCKER_BASE_IMAGE_NAME'. Please wait."
+incus publish -q --public "$BASE_IMAGE_VM_NAME/$UBUNTU_BASE_IMAGE_NAME" \
+                 --project default --alias="$DOCKER_BASE_IMAGE_NAME" \
+                 --compression none
+
+echo "INFO: Success creating the base image. Deleting artifacts from the build process."
+incus delete -f "$BASE_IMAGE_VM_NAME" --project default
+
+# # now let's get a snapshot of each of the blocks/chainstate directories.
+# for CHAIN in testnet mainnet; do
+#     for DATA in blocks chainstate; do
+#         if ! incus storage volume list ss-base -q --format csv -c n | grep -q "$CHAIN-$DATA/snap0"; then
+#             echo "INFO: Creating a snapshot 'ss-base/$CHAIN-$DATA/snap0'."
+#             incus storage volume snapshot ss-base --project default "$CHAIN-$DATA"
+#         fi
+#     done
+# done

deployment/deploy_vm.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+set -exu
+cd "$(dirname "$0")"
+
+. ./base.sh
+
+## This is a weird if clause since we need to LEFT-ALIGN the statement below.
+SSH_STRING="Host ${FQDN}"
+if ! grep -q "$SSH_STRING" "$SSH_HOME/config"; then
+
+########## BEGIN
+    cat >> "$SSH_HOME/config" <<-EOF
+
+${SSH_STRING}
+    HostName ${FQDN}
+    User ubuntu
+EOF
+###
+
+fi
+
+# if the machine doesn't exist, we create it.
+if ! incus list --format csv | grep -q "$INCUS_VM_NAME"; then
+
+    # create a base image if needed and instantiate a VM.
+    if [ -z "$MAC_ADDRESS_TO_PROVISION" ]; then
+        echo "ERROR: You MUST define a MAC Address for all your machines in your project definition."
+        echo "INFO: IMPORTANT! You MUST have DHCP Reservations for these MAC addresses. You also need records established the DNS."
+        exit 1
+    fi
+
+    # TODO ensure we are only GROWING the volume--never shrinking per zfs volume docs.
+    BACKUP_DISK_SIZE_GB=
+    SSDATA_DISK_SIZE_GB=
+    DOCKER_DISK_SIZE_GB=
+    if [ "$VIRTUAL_MACHINE" = www ]; then
+        if [ "$SKIP_WWW_SERVER" = true ]; then
+            exit 0
+        fi
+
+        BACKUP_DISK_SIZE_GB="$WWW_BACKUP_DISK_SIZE_GB"
+        SSDATA_DISK_SIZE_GB="$WWW_SSDATA_DISK_SIZE_GB"
+        DOCKER_DISK_SIZE_GB="$WWW_DOCKER_DISK_SIZE_GB"
+    fi
+
+    if [ "$VIRTUAL_MACHINE" = btcpayserver ]; then
+        if [ "$SKIP_BTCPAY_SERVER" = true ]; then
+            exit 0
+        fi
+
+        BACKUP_DISK_SIZE_GB="$BTCPAYSERVER_BACKUP_DISK_SIZE_GB"
+        SSDATA_DISK_SIZE_GB="$BTCPAYSERVER_SSDATA_DISK_SIZE_GB"
+        DOCKER_DISK_SIZE_GB="$BTCPAYSERVER_DOCKER_DISK_SIZE_GB"
+    fi
+
+    SSDATA_VOLUME_NAME=
+    BACKUP_VOLUME_NAME=
+    if [ "$VIRTUAL_MACHINE" != lnplayserver ]; then
+        DOCKER_VOLUME_NAME="$VIRTUAL_MACHINE-docker"
+        if ! incus storage volume list ss-base | grep -q "$DOCKER_VOLUME_NAME"; then
+            incus storage volume create ss-base "$DOCKER_VOLUME_NAME" --type=block
+            incus storage volume set ss-base "$DOCKER_VOLUME_NAME" size="${DOCKER_DISK_SIZE_GB}GB"
+        fi
+
+        SSDATA_VOLUME_NAME="$VIRTUAL_MACHINE-ss-data"
+        if ! incus storage volume list ss-base | grep -q "$SSDATA_VOLUME_NAME"; then
+            incus storage volume create ss-base "$SSDATA_VOLUME_NAME" --type=filesystem
+            incus storage volume set ss-base "$SSDATA_VOLUME_NAME" size="${SSDATA_DISK_SIZE_GB}GB"
+        fi
+
+        BACKUP_VOLUME_NAME="$VIRTUAL_MACHINE-backup"
+        if ! incus storage volume list ss-base | grep -q "$BACKUP_VOLUME_NAME"; then
+            incus storage volume create ss-base "$BACKUP_VOLUME_NAME" --type=filesystem
+            incus storage volume set ss-base "$BACKUP_VOLUME_NAME" size="${BACKUP_DISK_SIZE_GB}GB"
+        fi
+
+    fi
+
+
+    bash -c "./stub_profile.sh --vm=$VIRTUAL_MACHINE --incus-hostname=$INCUS_VM_NAME --ss-volume-name=$SSDATA_VOLUME_NAME --backup-volume-name=$BACKUP_VOLUME_NAME"
+
+  INCUS_LNPLAYSERVER_IMAGE_NAME="lnplayserver-$DOMAIN_NAME"
+  if ! incus image list -q --format csv | grep -q "$INCUS_LNPLAYSERVER_IMAGE_NAME"; then
+        script -q -c "incus init -q $DOCKER_BASE_IMAGE_NAME $INCUS_VM_NAME --vm --profile=$INCUS_VM_NAME" /dev/null
+    elif [ "$VIRTUAL_MACHINE" = lnplayserver ]; then
+        script -q -c "incus init -q $INCUS_LNPLAYSERVER_IMAGE_NAME $INCUS_VM_NAME --vm --profile=$INCUS_VM_NAME" /dev/null
+    fi
+
+    # let's PIN the HW address for now so we don't exhaust IP
+    # and so we can set DNS internally.
+    incus config set "$INCUS_VM_NAME" "volatile.enp5s0.hwaddr=$MAC_ADDRESS_TO_PROVISION"
+
+    if [ "$VIRTUAL_MACHINE" != lnplayserver ]; then
+        # attack the docker block device.
+        incus storage volume attach ss-base "$DOCKER_VOLUME_NAME" "$INCUS_VM_NAME"
+    fi
+
+    # if [ "$VIRTUAL_MACHINE" = btcpayserver ]; then
+    #     # attach any volumes
+    #     for CHAIN in testnet mainnet; do
+    #         for DATA in blocks chainstate; do
+    #             MOUNT_PATH="/$CHAIN-$DATA"
+    #             incus config device add "$INCUS_VM_NAME" "$CHAIN-$DATA" disk pool=ss-base source="$CHAIN-$DATA" path="$MOUNT_PATH"
+    #         done
+    #     done
+    # fi
+
+    incus start "$INCUS_VM_NAME"
+    sleep 15
+
+    bash -c "./wait_for_ip.sh --incus-name=$INCUS_VM_NAME"
+
+    # scan the remote machine and install it's identity in our SSH known_hosts file.
+    ssh-keyscan -H "$FQDN" >> "$SSH_HOME/known_hosts"
+
+    if [ "$VIRTUAL_MACHINE" != lnplayserver ]; then
+        ssh "$FQDN" "sudo chown ubuntu:ubuntu $REMOTE_DATA_PATH"
+        ssh "$FQDN" "sudo chown -R ubuntu:ubuntu $REMOTE_BACKUP_PATH"
+    fi
+
+fi

deployment/deployment_defaults.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -eu
+
+# file paths
+export SSH_HOME="$HOME/.ssh"
+export PASS_HOME="$HOME/.password-store" #TODO
+export SS_ROOT_PATH="$HOME/ss"
+export REMOTES_PATH="$SS_ROOT_PATH/remotes"
+export PROJECTS_PATH="$SS_ROOT_PATH/projects"
+export SITES_PATH="$SS_ROOT_PATH/sites"
+export INCUS_CONFIG_PATH="$SS_ROOT_PATH/incus"
+export SS_CACHE_PATH="$SS_ROOT_PATH/cache"
+
+export REMOTE_HOME="/home/ubuntu"
+export REMOTE_DATA_PATH="$REMOTE_HOME/ss-data"
+export REMOTE_DATA_PATH_LETSENCRYPT="$REMOTE_DATA_PATH/letsencrypt"
+export REMOTE_BACKUP_PATH="$REMOTE_HOME/backups"
+export BTCPAY_SERVER_APPPATH="$REMOTE_DATA_PATH/btcpayserver-docker"
+
+export BITCOIN_CHAIN=regtest
+
+# this space is for OS, docker images, etc
+# values here are fine for regtest generally. Later scripts adjust
+# these values based on testnet/mainnet
+export WWW_SSDATA_DISK_SIZE_GB=20
+export WWW_BACKUP_DISK_SIZE_GB=50
+export WWW_DOCKER_DISK_SIZE_GB=30
+
+export BTCPAYSERVER_SSDATA_DISK_SIZE_GB=20
+export BTCPAYSERVER_BACKUP_DISK_SIZE_GB=20
+export BTCPAYSERVER_DOCKER_DISK_SIZE_GB=30
+
+export WWW_HOSTNAME="www"
+export BTCPAY_SERVER_HOSTNAME="btcpayserver"
+export LNPLAY_SERVER_HOSTNAME="lnplayserver"
+export BTCPAY_HOSTNAME_IN_CERT="btcpay"
+export NEXTCLOUD_HOSTNAME="nextcloud"
+export GITEA_HOSTNAME="git"
+export NOSTR_HOSTNAME="relay"
+
+export REGISTRY_URL="https://index.docker.io/v1"
+
+export BTCPAY_SERVER_CPU_COUNT="4"
+export BTCPAY_SERVER_MEMORY_MB="4096"
+export WWW_SERVER_CPU_COUNT="4"
+export WWW_SERVER_MEMORY_MB="4096"
+export LNPLAY_SERVER_CPU_COUNT="4"
+export LNPLAY_SERVER_MEMORY_MB="4096"
+export DOCKER_IMAGE_CACHE_FQDN="registry-1.docker.io"

deployment/destroy.sh

@@ -1,108 +0,0 @@
-#!/bin/bash
-
-set -e
-cd "$(dirname "$0")"
-
-# this script destroys all resources in the current project.
-
-if lxc remote get-default | grep -q "local"; then
-    echo "ERROR: you are on the local lxc remote. Nothing to destroy"
-    exit 1
-fi
-
-echo "WARNING: This will DESTROY any existing VMs!"
-
-RESPONSE=
-read -r -p "Are you sure you want to continue (y/n):  ": RESPONSE
-if [ "$RESPONSE" != "y" ]; then
-    echo "STOPPING."
-    exit 0
-fi
-
-USER_TARGET_PROJECT=
-
-# grab any modifications from the command line.
-for i in "$@"; do
-    case $i in
-        --project=*)
-            USER_TARGET_PROJECT="${i#*=}"
-            shift
-        ;;
-
-        *)
-        echo "Unexpected option: $1"
-        exit 1
-        ;;
-    esac
-done
-
-. ../defaults.sh
-
-. ./remote_env.sh
-
-for PROJECT_CHAIN in ${DEPLOYMENT_STRING//,/ }; do
-    NO_PARENS="${PROJECT_CHAIN:1:${#PROJECT_CHAIN}-2}"
-    PROJECT_PREFIX=$(echo "$NO_PARENS" | cut -d'|' -f1)
-    BITCOIN_CHAIN=$(echo "$NO_PARENS" | cut -d'|' -f2)
-
-    PROJECT_NAME="$PROJECT_PREFIX-$BITCOIN_CHAIN"
-    PROJECT_PATH="$PROJECTS_DIR/$PROJECT_NAME"
-
-    # if the user sets USER_TARGET_PROJECT, let's ensure the project exists.
-    if [ -n "$USER_TARGET_PROJECT" ]; then
-        if ! lxc project list | grep -q "$USER_TARGET_PROJECT"; then
-            echo "ERROR: the project does not exist! Nothing to destroy."
-            exit 1
-        fi
-
-        if [ "$PROJECT_NAME" != "$USER_TARGET_PROJECT" ]; then
-            echo "INFO: Skipping project '$PROJECT_NAME' since the system owner has used the --project switch."
-            exit
-        fi
-    fi
-
-    export PROJECT_NAME="$PROJECT_NAME"
-    export PROJECT_PATH="$PROJECT_PATH"
-
-    . ./project_env.sh
-
-    if ! lxc info | grep "project:" | grep -q "$PROJECT_NAME"; then
-        if lxc project list | grep -q "$PROJECT_NAME"; then
-            lxc project switch "$PROJECT_NAME"
-        fi
-    fi
-
-    for VM in www btcpayserver; do
-        LXD_NAME="$VM-${DOMAIN_NAME//./-}"
-
-        if lxc list | grep -q "$LXD_NAME"; then
-            lxc delete -f "$LXD_NAME"
-
-            # remove the ssh known endpoint else we get warnings.
-            ssh-keygen -f "$SSH_HOME/known_hosts" -R "$LXD_NAME"
-        fi
-
-        if lxc profile list | grep -q "$LXD_NAME"; then
-            lxc profile delete "$LXD_NAME"
-        fi
-    done
-
-    if lxc network list -q | grep -q ss-ovn; then
-        lxc network delete ss-ovn
-    fi
-
-    if ! lxc info | grep "project:" | grep -q default; then
-        lxc project switch default
-    fi
-
-    if lxc project list | grep -q "$PROJECT_NAME"; then
-        lxc project delete "$PROJECT_NAME"
-    fi
-
-    # delete the base image so it can be created.
-    if lxc list | grep -q "$BASE_IMAGE_VM_NAME"; then
-        lxc delete -f "$BASE_IMAGE_VM_NAME"
-        # remove the ssh known endpoint else we get warnings.
-        ssh-keygen -f "$SSH_HOME/known_hosts" -R "$LXD_NAME"
-    fi
-done

deployment/domain_list.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -exu
+
+# the DOMAIN_LIST is a complete list of all our domains. We often iterate over this list.
+DOMAIN_LIST="${PRIMARY_DOMAIN}"
+if [ -n "$OTHER_SITES_LIST" ]; then
+    DOMAIN_LIST="${DOMAIN_LIST},${OTHER_SITES_LIST}"
+fi
+
+export DOMAIN_LIST="$DOMAIN_LIST"
+export DOMAIN_COUNT=$(("$(echo "$DOMAIN_LIST" | tr -cd , | wc -c)"+1))
+export OTHER_SITES_LIST="$OTHER_SITES_LIST"
+
+export PRIMARY_WWW_FQDN="$WWW_HOSTNAME.$PRIMARY_DOMAIN"
+export BTCPAY_SERVER_FQDN="$BTCPAY_SERVER_HOSTNAME.$PRIMARY_DOMAIN"
+export LNPLAY_SERVER_FQDN="$LNPLAY_SERVER_HOSTNAME.$PRIMARY_DOMAIN"

deployment/down.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+# https://www.sovereign-stack.org/ss-down/
+
+set -exu
+cd "$(dirname "$0")"
+
+if incus remote get-default -q | grep -q "local"; then
+    echo "ERROR: you are on the local incus remote. Nothing to take down"
+    exit 1
+fi
+
+KEEP_ZFS_STORAGE_VOLUMES=true
+OTHER_SITES_LIST=
+SKIP_BTCPAY_SERVER=false
+SKIP_WWW_SERVER=false
+SKIP_LNPLAY_SERVER=false
+BACKUP_WWW_APPS=true
+
+WWW_SERVER_MAC_ADDRESS=
+BTCPAY_SERVER_MAC_ADDRESS=
+LNPLAY_SERVER_MAC_ADDRESS=
+
+# grab any modifications from the command line.
+for i in "$@"; do
+    case $i in
+        --purge)
+            KEEP_ZFS_STORAGE_VOLUMES=false
+            shift
+        ;;
+        --skip-btcpayserver)
+            SKIP_BTCPAY_SERVER=true
+            shift
+        ;;
+        --skip-wwwserver)
+            SKIP_WWW_SERVER=true
+            shift
+        ;;
+        --skip-lnplayserver)
+            SKIP_LNPLAY_SERVER=true
+            shift
+        ;;
+        *)
+        echo "Unexpected option: $1"
+        exit 1
+        ;;
+    esac
+done
+
+. ./deployment_defaults.sh
+
+. ./remote_env.sh
+
+. ./project_env.sh
+
+# let's bring down services on the remote deployment if necessary.
+export DOMAIN_NAME="$PRIMARY_DOMAIN"
+export SITE_PATH="$SITES_PATH/$PRIMARY_DOMAIN"
+
+source "$SITE_PATH/site.conf"
+source ./project/domain_env.sh
+
+source ./domain_list.sh
+
+
+SERVERS=
+if [ "$SKIP_WWW_SERVER" = false ] && [ -n "$WWW_SERVER_MAC_ADDRESS" ]; then
+    SERVERS="www $SERVERS"
+fi
+
+if [ "$SKIP_BTCPAY_SERVER" = false ] && [ -n "$BTCPAY_SERVER_MAC_ADDRESS" ]; then
+    SERVERS="$SERVERS btcpayserver"
+fi
+
+if [ "$SKIP_LNPLAY_SERVER" = false ] && [ -n "$LNPLAY_SERVER_MAC_ADDRESS" ]; then
+    SERVERS="$SERVERS lnplayserver"
+fi
+
+for VIRTUAL_MACHINE in $SERVERS; do
+
+    INCUS_VM_NAME="$VIRTUAL_MACHINE-${PRIMARY_DOMAIN//./-}"
+
+    if incus list | grep -q "$INCUS_VM_NAME"; then
+        bash -c "./stop.sh --server=$VIRTUAL_MACHINE"
+
+        incus stop "$INCUS_VM_NAME"
+
+        incus delete "$INCUS_VM_NAME"
+    fi
+
+    # remove the ssh known endpoint else we get warnings.
+    ssh-keygen -f "$SSH_HOME/known_hosts" -R "$VIRTUAL_MACHINE.$PRIMARY_DOMAIN" | exit
+
+    if incus profile list | grep -q "$INCUS_VM_NAME"; then
+        incus profile delete "$INCUS_VM_NAME"
+    fi
+
+    if [ "$KEEP_ZFS_STORAGE_VOLUMES" = false ]; then
+        # d for docker; b for backup; s for ss-data
+        for DATA in docker backup ss-data; do
+            VOLUME_NAME="$VIRTUAL_MACHINE-$DATA"
+            if incus storage volume list ss-base -q | grep -q "$VOLUME_NAME"; then
+                RESPONSE=
+                read -r -p "Are you sure you want to delete the '$VOLUME_NAME' volume intended for '$INCUS_VM_NAME'?": RESPONSE
+            
+                if [ "$RESPONSE" = "y" ]; then
+                    incus storage volume delete ss-base "$VOLUME_NAME"
+                fi
+            fi
+        done
+    fi
+done
+
+
+BACKUP_WWW_APPS=true
+echo "BACKUP_WWW_APPS: $BACKUP_WWW_APPS"
+
+
+echo "SERVERS: $SERVERS"
+echo "BACKUP_WWW_APPS: $BACKUP_WWW_APPS"
+
+
+# let's grab a snapshot of the 
+if [ "$BACKUP_WWW_APPS" = true ]; then
+    SNAPSHOT_ID=$(cat /dev/urandom | tr -dc 'a-aA-Z' | fold -w 6 | head -n 1)
+    incus storage volume snapshot create ss-base www-ss-data "$SNAPSHOT_ID"
+    BACKUP_LOCATION="$HOME/ss/backups"
+    mkdir -p "$BACKUP_LOCATION"
+    #incus storage volume export ss-base "www-ss-data" "$BACKUP_LOCATION/project-$(incus project list --format csv | grep "(current)" | awk '{print $1}')_www-ss-data_""$(date +%s)"".tar.gz"
+    #incus storage volume snapshot delete ss-base "www-ss-data" "$SNAPSHOT_ID"
+fi
+
+if [[ "$SERVERS" == *"www"* && "$SERVERS" == *"btcpay"* ]]; then
+    if incus network list -q | grep -q ss-ovn; then
+        incus network delete ss-ovn
+    fi
+fi

deployment/help.txt

@@ -2,11 +2,14 @@
 You are in the Sovereign Stack Management Environment (SSME). From here, you can issue several commands:
 
     ss-remote  - Take a remote SSH endpoint under management of Sovereign Stack.
-    ss-deploy  - Creates a deployment to your active LXD remote.
-    ss-destroy - Destroys the active deployment (WARNING: destructive).
-    ss-update  - brings an existing deployment up to the newest version of Sovereign Stack.
-    ss-show    - show the lxd resources associated with the current remote.
+    ss-reset   - The opposite of ss-remote; de-provisions an existing remote.
+    ss-up      - Instantiate a deployment to your active project according to your
+                 various project.conf and site.conf files.
+    ss-down    - Reverses ss-up. Takes the active project down. Non-destructive of user data,
+                 unless you provide the --purge flag.
+    ss-update  - This is just ss-down then ss-up.
+    ss-show    - show the incus resources associated with the current remote.
 
 For more infomation about all these topics, consult the Sovereign Stack website starting with:
 
-  - https://www.sovereign-stack.org/tag/instance-management/
+  - https://www.sovereign-stack.org/tag/deployment-management/

deployment/project

@@ -0,0 +1 @@
+Subproject commit e8470d789a3811e2fe3f6818fd9a6fea859ba71c

deployment/project_env.sh

@@ -1,8 +1,16 @@
 #!/bin/bash
 
 set -eu
-cd "$(dirname "$0")"
 
+PROJECT_NAME="$(incus info | grep "project:" | awk '{print $2}')"
+export PROJECT_NAME="$PROJECT_NAME"
+
+if [ "$PROJECT_NAME" = default ]; then
+    echo "ERROR: You are on the default project. Use 'incus project list' and 'incus project switch <project>'."
+    exit 1
+fi
+
+export PROJECT_PATH="$PROJECTS_PATH/$PROJECT_NAME"
 
 PROJECT_DEFINITION_PATH="$PROJECT_PATH/project.conf"
 
@@ -13,22 +21,45 @@ fi
 
 source "$PROJECT_DEFINITION_PATH"
 
-export PRIMARY_SITE_DEFINITION_PATH="$SITES_PATH/$PRIMARY_DOMAIN/site.conf"
 
-if [ ! -f "$PRIMARY_SITE_DEFINITION_PATH" ]; then
-    echo "ERROR: the site definition does not exist."
-    exit 1
-fi
+export PRIMARY_SITE_DEFINITION_PATH="$SITES_PATH/$PRIMARY_DOMAIN/site.conf"
 
 if [ -z "$PRIMARY_DOMAIN" ]; then
     echo "ERROR: The PRIMARY_DOMAIN is not specified. Check your remote definition at '$PRIMARY_SITE_DEFINITION_PATH'."
     exit 1
 fi
 
-source "$PRIMARY_SITE_DEFINITION_PATH"
+SHASUM_OF_PRIMARY_DOMAIN="$(echo -n "$PRIMARY_DOMAIN" | sha256sum | awk '{print $1;}' )"
+export PRIMARY_DOMAIN_IDENTIFIER="${SHASUM_OF_PRIMARY_DOMAIN: -6}"
 
-if ! lxc info | grep "project:" | grep -q "$PROJECT_NAME"; then
-    if lxc project list | grep -q "$PROJECT_NAME"; then
-        lxc project switch "$PROJECT_NAME"
-    fi
-fi
+
+# default values are already at regtest mode.
+if [ "$BITCOIN_CHAIN" = testnet ]; then
+
+    WWW_SSDATA_DISK_SIZE_GB=30
+    WWW_BACKUP_DISK_SIZE_GB=30
+    WWW_DOCKER_DISK_SIZE_GB=50
+
+    BTCPAYSERVER_SSDATA_DISK_SIZE_GB=30
+    BTCPAYSERVER_BACKUP_DISK_SIZE_GB=30
+    BTCPAYSERVER_DOCKER_DISK_SIZE_GB=100
+
+elif [ "$BITCOIN_CHAIN" = mainnet ]; then
+    
+    WWW_SSDATA_DISK_SIZE_GB=40
+    WWW_BACKUP_DISK_SIZE_GB=40
+    WWW_DOCKER_DISK_SIZE_GB=50
+
+    BTCPAYSERVER_SSDATA_DISK_SIZE_GB=30
+    BTCPAYSERVER_BACKUP_DISK_SIZE_GB=30
+    BTCPAYSERVER_DOCKER_DISK_SIZE_GB=300
+
+fi
+
+export WWW_SSDATA_DISK_SIZE_GB="$WWW_SSDATA_DISK_SIZE_GB"
+export WWW_BACKUP_DISK_SIZE_GB="$WWW_BACKUP_DISK_SIZE_GB"
+export WWW_DOCKER_DISK_SIZE_GB="$WWW_DOCKER_DISK_SIZE_GB"
+
+export BTCPAYSERVER_SSDATA_DISK_SIZE_GB="$BTCPAYSERVER_SSDATA_DISK_SIZE_GB"
+export BTCPAYSERVER_BACKUP_DISK_SIZE_GB="$BTCPAYSERVER_BACKUP_DISK_SIZE_GB"
+export BTCPAYSERVER_DOCKER_DISK_SIZE_GB="$BTCPAYSERVER_DOCKER_DISK_SIZE_GB"

deployment/remote.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-set -e
+set -exu
 cd "$(dirname "$0")"
 
 # This script is meant to be executed on the management machine.
 # it reaches out to an SSH endpoint and provisions that machine
-# to use LXD.
+# to use incus.
 
 DATA_PLANE_MACVLAN_INTERFACE=
 DISK_TO_USE=
@@ -13,15 +13,16 @@ DISK_TO_USE=
 # override the remote name.
 REMOTE_NAME="${1:-}"
 if [ -z "$REMOTE_NAME" ]; then
-    echo "ERROR: The remote name was not provided. Syntax is: 'ss-remote REMOTE_NAME SSH_HOST_FQDN'"
-    echo "  for example: 'ss-remote dev host01.domain.tld"
+    echo "ERROR: The remote name was not provided. Syntax is: 'ss-remote <remote_name> <remote01.domain.tld>'"
+    echo "  for example: 'ss-remote development clusterhost00.domain.tld"
     exit 1
 fi
 
-#shellcheck disable=SC1091
-source ../defaults.sh
+. ./deployment_defaults.sh
 
-export REMOTE_PATH="$REMOTES_DIR/$REMOTE_NAME"
+. ./base.sh
+
+export REMOTE_PATH="$REMOTES_PATH/$REMOTE_NAME"
 REMOTE_DEFINITION="$REMOTE_PATH/remote.conf"
 export REMOTE_DEFINITION="$REMOTE_DEFINITION"
 
@@ -31,8 +32,6 @@ if [ ! -f "$REMOTE_DEFINITION" ]; then
     cat >"$REMOTE_DEFINITION" <<EOL
 # https://www.sovereign-stack.org/ss-remote
 
-LXD_REMOTE_PASSWORD="$(gpg --gen-random --armor 1 14)"
-DEPLOYMENT_STRING="(dev|regtest),(staging|testnet)"
 # REGISTRY_URL=http://registry.domain.tld:5000
 
 EOL
@@ -46,7 +45,7 @@ fi
 
 source "$REMOTE_DEFINITION"
 
-if ! lxc remote list | grep -q "$REMOTE_NAME"; then
+if ! incus remote list | grep -q "$REMOTE_NAME"; then
     FQDN="${2:-}"
 
     if [ -z "$FQDN" ]; then
@@ -87,30 +86,20 @@ if ! lxc remote list | grep -q "$REMOTE_NAME"; then
     # first let's copy our ssh pubkey to the remote server so we don't have to login constantly.
     ssh-copy-id -i "$HOME/.ssh/id_rsa.pub" "ubuntu@$FQDN"
 
-    if [ -z "$DATA_PLANE_MACVLAN_INTERFACE" ]; then
-        echo "INFO: It looks like you didn't provide input on the command line for the data plane macvlan interface."
-        echo "      We need to know which interface that is! Enter it here now."
-        echo ""
-
-        ssh "ubuntu@$FQDN" ip link
-
-        echo "Please enter the network interface that's dedicated to the Sovereign Stack data plane: "
-        read -r DATA_PLANE_MACVLAN_INTERFACE
-
-    fi
-
     if [ -z "$DISK_TO_USE" ]; then
-        echo "INFO: It looks like the DISK_TO_USE has not been set. Enter it now."
-        echo ""
+        if ! ssh "ubuntu@$FQDN" incus storage list -q | grep -q ss-base; then
+            echo "INFO: It looks like the DISK_TO_USE has not been set. Enter it now."
+            echo ""
 
-        ssh "ubuntu@$FQDN" lsblk --paths
+            ssh "ubuntu@$FQDN" lsblk --paths
 
-        echo "Please enter the disk or partition that Sovereign Stack will use to store data (default: loop):  "
-        read -r DISK_TO_USE
+            echo "Please enter the disk or partition that Sovereign Stack will use to store data:  "
+            read -r DISK_TO_USE
+        fi
     fi
 
 else
-    echo "ERROR: the remote already exists! You need to go delete your lxd remote if you want to re-create your remote."
+    echo "ERROR: the remote already exists! You need to go delete your incus remote if you want to re-create your remote."
     echo "       It's may also be helpful to reset/rename your remote path."
     exit 1
 fi
@@ -128,68 +117,58 @@ if [ "$DISK_TO_USE" != loop ]; then
     fi
 fi
 
-# The MGMT Plane IP is the IP address that the LXD API binds to, which happens
-# to be the same as whichever SSH connection you're coming in on.
-MGMT_PLANE_IP="$(ssh ubuntu@"$FQDN" env | grep SSH_CONNECTION | cut -d " " -f 3)"
-IP_OF_MGMT_MACHINE="$(ssh ubuntu@"$FQDN" env | grep SSH_CLIENT | cut -d " " -f 1 )"
-IP_OF_MGMT_MACHINE="${IP_OF_MGMT_MACHINE#*=}"
-IP_OF_MGMT_MACHINE="$(echo "$IP_OF_MGMT_MACHINE" | cut -d: -f1)"
-
-# error out if the remote password is unset.
-if [ -z "$LXD_REMOTE_PASSWORD" ]; then
-    echo "ERROR: LXD_REMOTE_PASSWORD must be set in your remote.conf file."
-    exit 1
-fi
-
-if ! command -v lxc >/dev/null 2>&1; then
-    if lxc profile list --format csv | grep -q "$BASE_IMAGE_VM_NAME"; then
-        lxc profile delete "$BASE_IMAGE_VM_NAME"
+if ! command -v incus >/dev/null 2>&1; then
+    if incus profile list --format csv | grep -q "$BASE_IMAGE_VM_NAME"; then
+        incus profile delete "$BASE_IMAGE_VM_NAME"
         sleep 1
     fi
 
-    if lxc network list --format csv | grep -q lxdbr0; then
-        lxc network delete lxdbr0
+    if incus network list --format csv -q --project default | grep -q incusbr0; then
+        incus network delete incusbr0 --project default
         sleep 1
     fi
 
 
-    if lxc network list --format csv | grep -q lxdbr1; then
-        lxc network delete lxdbr1
+    if incus network list --format csv -q project default | grep -q incusbr1; then
+        incus network delete incusbr1 --project default
         sleep 1
     fi
 
 fi
 
 # install dependencies.
-ssh -t "ubuntu@$FQDN" 'sudo apt update && sudo apt upgrade -y && sudo apt install htop dnsutils nano -y'
-if ! ssh "ubuntu@$FQDN" snap list | grep -q lxd; then
-    ssh -t "ubuntu@$FQDN" 'sudo snap install lxd --channel=5.11/stable'
-    sleep 5
-fi
+ssh -t "ubuntu@$FQDN" 'sudo apt update && sudo apt upgrade -y && sudo apt install htop dnsutils nano zfsutils-linux -y'
+
+REMOTE_SCRIPT_PATH="$REMOTE_HOME/install_incus.sh"
+scp ../install_incus.sh "ubuntu@$FQDN:$REMOTE_SCRIPT_PATH"
+ssh -t "ubuntu@$FQDN" "chmod +x $REMOTE_SCRIPT_PATH"
+ssh -t "ubuntu@$FQDN" "sudo bash -c $REMOTE_SCRIPT_PATH"
+ssh -t "ubuntu@$FQDN" "sudo adduser ubuntu incus-admin"
 
 # install OVN for the project-specific bridge networks
-ssh -t "ubuntu@$FQDN" "sudo apt-get install -y ovn-host ovn-central"
+ssh -t "ubuntu@$FQDN" "sudo apt-get install -y ovn-host ovn-central && sudo ovs-vsctl set open_vswitch . external_ids:ovn-remote=unix:/var/run/ovn/ovnsb_db.sock external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=127.0.0.1"
 
-ssh -t "ubuntu@$FQDN" "sudo ovs-vsctl set open_vswitch . external_ids:ovn-remote=unix:/var/run/ovn/ovnsb_db.sock external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=127.0.0.1"
-
-# if the DATA_PLANE_MACVLAN_INTERFACE is not specified, then we 'll
-# just attach VMs to the network interface used for for the default route.
+# if the user did not specify the interface, we just use whatever is used for the default route.
 if [ -z "$DATA_PLANE_MACVLAN_INTERFACE" ]; then
-    DATA_PLANE_MACVLAN_INTERFACE="$(ssh -t ubuntu@"$FQDN" ip route | grep default | cut -d " " -f 5)"
+    DATA_PLANE_MACVLAN_INTERFACE="$(ssh ubuntu@"$FQDN" ip route | grep "default via" | awk '{print $5}')"
 fi
 
 export DATA_PLANE_MACVLAN_INTERFACE="$DATA_PLANE_MACVLAN_INTERFACE"
 
-# run lxd init on the remote server.
-cat <<EOF | ssh ubuntu@"$FQDN" lxd init --preseed
+MGMT_PLANE_IP="$(ssh ubuntu@"$FQDN" env | grep SSH_CONNECTION | cut -d " " -f 3)"
+IP_OF_MGMT_MACHINE="$(ssh ubuntu@"$FQDN" env | grep SSH_CLIENT | cut -d " " -f 1 )"
+IP_OF_MGMT_MACHINE="${IP_OF_MGMT_MACHINE#*=}"
+IP_OF_MGMT_MACHINE="$(echo "$IP_OF_MGMT_MACHINE" | cut -d: -f1)"
+
+# run incus admin init on the remote server.
+cat <<EOF | ssh ubuntu@"$FQDN" incus admin init --preseed
 config:
   core.https_address: ${MGMT_PLANE_IP}:8443
-  core.trust_password: ${LXD_REMOTE_PASSWORD}
   core.dns_address: ${MGMT_PLANE_IP}
   images.auto_update_interval: 15
   
 networks:
-- name: lxdbr0
+- name: incusbr0
   description: "ss-config,${DATA_PLANE_MACVLAN_INTERFACE:-error}"
   type: bridge
   config:
@@ -198,8 +177,8 @@ networks:
     ipv4.nat: true
     ipv6.address: none
     dns.mode: managed
-- name: lxdbr1
-  description: "Non-natting bridge for ovn networks to connect to."
+- name: incusbr1
+  description: "Non-natting bridge needed for ovn networks."
   type: bridge
   config:
     ipv4.address: 10.10.10.1/24
@@ -216,41 +195,48 @@ profiles:
       pool: ss-base
       type: disk
   name: default
-cluster:
-  server_name: ${REMOTE_NAME}
-  enabled: true
-  member_config: []
-  cluster_address: ""
-  cluster_certificate: ""
-  server_address: ""
-  cluster_password: ""
-  cluster_certificate_path: ""
-  cluster_token: ""
 EOF
 
-# ensure the lxd service is available over the network, then add a lxc remote, then switch the active remote to it.
-if wait-for-it -t 20 "$FQDN:8443"; then
-    # now create a remote on your local LXC client and switch to it.
-    # the software will now target the new remote.
-    lxc remote add "$REMOTE_NAME" "$FQDN" --password="$LXD_REMOTE_PASSWORD" --protocol=lxd --auth-type=tls --accept-certificate
-    lxc remote switch "$REMOTE_NAME"
+ssh ubuntu@"$FQDN" incus project list -q >> /dev/null
 
-    echo "INFO: You have create a new remote named '$REMOTE_NAME'. Great! We switched your lxd remote to it."
+# ensure the incus service is available over the network, then add a incus remote, then switch the active remote to it.
+if wait-for-it -t 20 "$FQDN:8443"; then
+    # before we add the remote, we need a trust token from the incus server
+    INCUS_CERT_TRUST_TOKEN=$(ssh ubuntu@"$FQDN" incus config trust add ss-mgmt | tail -n 1)
+
+    # now create a remote on your local incus client and switch to it.
+    # the software will now target the new remote.
+    incus remote add "$REMOTE_NAME" "$FQDN" --auth-type=tls --accept-certificate --token="$INCUS_CERT_TRUST_TOKEN"
+    incus remote switch "$REMOTE_NAME"
+
+    echo "INFO: A new remote named '$REMOTE_NAME' has been created. Your incus client has been switched to it."
 else
-    echo "ERROR: Could not detect the LXD endpoint. Something went wrong."
+    echo "ERROR: Could not detect the incus endpoint. Something went wrong."
     exit 1
 fi
 
 # create the default storage pool if necessary
-if ! lxc storage list --format csv | grep -q ss-base; then
+if ! incus storage list --format csv | grep -q ss-base; then
 
     if [ "$DISK_TO_USE" != loop ]; then
-        # we omit putting a size here so, so LXD will consume the entire disk if '/dev/sdb' or partition if '/dev/sdb1'.
+        # we omit putting a size here so, so incus will consume the entire disk if '/dev/sdb' or partition if '/dev/sdb1'.
         # TODO do some sanity/resource checking on DISK_TO_USE. Impelment full-disk encryption?
-        lxc storage create ss-base zfs source="$DISK_TO_USE"
+        incus storage create ss-base zfs source="$DISK_TO_USE"
     else
         # if a disk is the default 'loop', then we create a zfs storage pool 
-        # on top of the existing filesystem using a loop device, per LXD docs
-        lxc storage create ss-base zfs
+        # on top of the existing filesystem using a loop device, per incus docs
+        incus storage create ss-base zfs
     fi
+
+    # # create the testnet/mainnet blocks/chainstate subvolumes.
+    # for CHAIN in mainnet testnet; do
+    #     for DATA in blocks chainstate; do
+    #         if ! incus storage volume list ss-base | grep -q "$CHAIN-$DATA"; then
+    #             incus storage volume create ss-base "$CHAIN-$DATA" --type=filesystem
+    #         fi
+    #     done
+    # done
+
 fi
+
+echo "INFO: completed remote.sh."

deployment/remote_env.sh

@@ -1,11 +1,20 @@
 #!/bin/bash
 
 set -eu
-cd "$(dirname "$0")"
 
-CURRENT_REMOTE="$(lxc remote get-default)"
+CURRENT_REMOTE="$(incus remote get-default)"
+DEPLOYMENT_STRING=
 
-if echo "$CURRENT_REMOTE" | grep -q "production"; then
+SS_ROOT_PATH="$HOME/ss"
+REMOTES_PATH="$SS_ROOT_PATH/remotes"
+PROJECTS_PATH="$SS_ROOT_PATH/projects"
+SITES_PATH="$SS_ROOT_PATH/sites"
+INCUS_CONFIG_PATH="$SS_ROOT_PATH/incus"
+SS_CACHE_PATH="$SS_ROOT_PATH/cache"
+
+
+
+if echo "$CURRENT_REMOTE" | grep -q "prod"; then
     echo "WARNING: You are running a migration procedure on a production system."
     echo ""
 
@@ -26,7 +35,9 @@ if echo "$CURRENT_REMOTE" | grep -q "production"; then
 
 fi
 
-export REMOTE_PATH="$REMOTES_DIR/$CURRENT_REMOTE"
+. ./deployment_defaults.sh
+
+export REMOTE_PATH="$REMOTES_PATH/$CURRENT_REMOTE"
 REMOTE_DEFINITION="$REMOTE_PATH/remote.conf"
 export REMOTE_DEFINITION="$REMOTE_DEFINITION"
 
@@ -38,3 +49,4 @@ if [ ! -f "$REMOTE_DEFINITION" ]; then
 fi
 
 source "$REMOTE_DEFINITION"
+

deployment/reset.sh

@@ -1,55 +1,88 @@
 #!/bin/bash
 
 
-set -e
+set -exu
 cd "$(dirname "$0")"
 
-source ../defaults.sh
+echo "WARNING: THIS SCRIPT NEEDS WORK"
+exit 1
 
-./destroy.sh
+PURGE_INCUS=false
+
+# grab any modifications from the command line.
+for i in "$@"; do
+    case $i in
+        --purge)
+            PURGE_INCUS=true
+            shift
+        ;;
+        *)
+        echo "Unexpected option: $1"
+        exit 1
+        ;;
+    esac
+done
+
+
+./down.sh
 
 # these only get initialzed upon creation, so we MUST delete here so they get recreated.
-if lxc profile list | grep -q "$BASE_IMAGE_VM_NAME"; then
-    lxc profile delete "$BASE_IMAGE_VM_NAME"
+if incus profile list | grep -q "$BASE_IMAGE_VM_NAME"; then
+    incus profile delete "$BASE_IMAGE_VM_NAME"
 fi
 
-if lxc image list | grep -q "$BASE_IMAGE_VM_NAME"; then
-    lxc image rm "$BASE_IMAGE_VM_NAME"
+if incus image list | grep -q "$BASE_IMAGE_VM_NAME"; then
+    incus image rm "$BASE_IMAGE_VM_NAME"
 fi
 
-if lxc image list | grep -q "$DOCKER_BASE_IMAGE_NAME"; then
-    lxc image rm "$DOCKER_BASE_IMAGE_NAME"
+if incus image list | grep -q "$DOCKER_BASE_IMAGE_NAME"; then
+    incus image rm "$DOCKER_BASE_IMAGE_NAME"
 fi
 
-CURRENT_PROJECT="$(lxc info | grep "project:" | awk '{print $2}')"
-if ! lxc info | grep -q "project: default"; then
-    lxc project switch default
-    lxc project delete "$CURRENT_PROJECT"
-fi
-
-if lxc profile show default | grep -q "root:"; then
-    lxc profile device remove default root
-fi
-
-if lxc profile show default| grep -q "eth0:"; then
-    lxc profile device remove default eth0
-fi
-
-if lxc network list --format csv | grep -q lxdbr0; then
-    lxc network delete lxdbr0
-fi
-
-if lxc network list --format csv | grep -q lxdbr1; then
-    lxc network delete lxdbr1
+CURRENT_PROJECT="$(incus info | grep "project:" | awk '{print $2}')"
+if ! incus info | grep -q "project: default"; then
+    incus project switch default
+    incus project delete "$CURRENT_PROJECT"
 fi
 
 
-if lxc storage list --format csv | grep -q ss-base; then
-    lxc storage delete ss-base
-fi
+if [ "$PURGE_INCUS" = true ]; then
 
-CURRENT_REMOTE="$(lxc remote get-default)"
-if ! lxc remote get-default | grep -q "local"; then
-    lxc remote switch local
-    lxc remote remove "$CURRENT_REMOTE"
+    if incus profile show default | grep -q "root:"; then
+        incus profile device remove default root
+    fi
+
+    if incus profile show default| grep -q "eth0:"; then
+        incus profile device remove default eth0
+    fi
+
+    if incus network list --format csv -q --project default | grep -q incusbr0; then
+        incus network delete incusbr0 --project default
+    fi
+
+    if incus network list --format csv -q --project default | grep -q incusbr1; then
+        incus network delete incusbr1 --project default
+    fi
+
+    # # create the testnet/mainnet blocks/chainstate subvolumes.
+    # for CHAIN in mainnet testnet; do
+    #     for DATA in blocks chainstate; do
+    #         if incus storage volume list ss-base | grep -q "$CHAIN-$DATA"; then
+    #             incus storage volume delete ss-base "$CHAIN-$DATA"
+    #         fi
+    #     done
+    # done
+
+    echo "WARNING: ss-basae NOT DELETED. NEED TO TEST THIS SCRIPT"
+    # if incus storage list --format csv | grep -q ss-base; then
+    #     incus storage delete ss-base
+    # fi
+
+    CURRENT_REMOTE="$(incus remote get-default)"
+    if ! incus remote get-default | grep -q "local"; then
+        incus remote switch local
+        incus remote remove "$CURRENT_REMOTE"
+
+        echo "INFO: The remote '$CURRENT_REMOTE' has been removed! You are now controlling your local instance."
+    fi
 fi

deployment/restore.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -exu
+

deployment/show.sh

@@ -1,12 +1,42 @@
 #!/bin/bash
 
-echo "LXD REMOTE: $(lxc remote get-default)"
+set -e
+cd "$(dirname "$0")"
 
-lxc project list
+. ./deployment_defaults.sh
 
-lxc storage list
-lxc image list
-lxc project list
-lxc network list
-lxc profile list
-lxc list
+. ./remote_env.sh
+
+echo "Global Settings:"
+
+incus image list
+incus storage list
+
+
+echo
+echo
+
+PROJECT_NAME="$(incus info | grep "project:" | awk '{print $2}')"
+export export="$PROJECT_NAME"
+export PROJECT_PATH="$PROJECTS_PATH/$PROJECT_NAME"
+
+echo
+echo
+echo "Active project: $PROJECT_NAME"
+echo "----------------------------------------------------------"
+
+echo "  Networks:"
+incus network list
+
+echo
+echo "  Storage Volumes:"
+incus storage volume list ss-base
+
+echo
+echo "  Profiles:"
+incus profile list
+
+
+echo
+echo "  Instances (VMs):"
+incus list

deployment/stop.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# https://www.sovereign-stack.org/ss-down/
+
+set -eu
+cd "$(dirname "$0")"
+
+if incus remote get-default -q | grep -q "local"; then
+    echo "ERROR: you are on the local incus remote. Nothing to take down"
+    exit 1
+fi
+
+SERVER_TO_STOP=
+OTHER_SITES_LIST=
+
+# grab any modifications from the command line.
+for i in "$@"; do
+    case $i in
+        --server=*)
+            SERVER_TO_STOP="${i#*=}"
+            shift
+        ;;
+        *)
+        echo "Unexpected option: $1"
+        exit 1
+        ;;
+    esac
+done
+
+if [ -z "$SERVER_TO_STOP" ]; then
+    echo "ERROR: you MUST specify a server to stop with '--server=www' for example."
+    exit 1
+fi
+
+
+. ./deployment_defaults.sh
+
+. ./remote_env.sh
+
+. ./project_env.sh
+
+# let's bring down services on the remote deployment if necessary.
+export DOMAIN_NAME="$PRIMARY_DOMAIN"
+export SITE_PATH="$SITES_PATH/$PRIMARY_DOMAIN"
+
+source "$SITE_PATH/site.conf"
+source ./project/domain_env.sh
+
+source ./domain_list.sh
+
+if [ "$SERVER_TO_STOP" = www ]; then
+    DOCKER_HOST="ssh://ubuntu@$PRIMARY_WWW_FQDN" ./project/www/stop_docker_stacks.sh
+fi
+
+if [ "$SERVER_TO_STOP" = btcpayserver ]; then
+    if wait-for-it -t 5 "$BTCPAY_SERVER_FQDN":22; then
+        ssh "$BTCPAY_SERVER_FQDN" "bash -c $BTCPAY_SERVER_APPPATH/btcpay-down.sh"
+    else
+        echo "ERROR: the remote BTCPAY Server is not available on ssh."
+        exit 1
+    fi
+fi
+
+if [ "$SERVER_TO_STOP" = lnplayserver ]; then
+    DOCKER_HOST="ssh://ubuntu@$LNPLAY_SERVER_FQDN" ./project/lnplay/down.sh
+fi

deployment/stub_profile.sh

@@ -0,0 +1,338 @@
+#!/bin/bash
+
+set -exu
+cd "$(dirname "$0")"
+
+VIRTUAL_MACHINE=base
+INCUS_HOSTNAME=
+SSDATA_VOLUME_NAME=
+BACKUP_VOLUME_NAME=
+
+# grab any modifications from the command line.
+for i in "$@"; do
+    case $i in
+        --incus-hostname=*)
+            INCUS_HOSTNAME="${i#*=}"
+            shift
+        ;;
+        --vm=*)
+            VIRTUAL_MACHINE="${i#*=}"
+            shift
+        ;;
+        --ss-volume-name=*)
+            SSDATA_VOLUME_NAME="${i#*=}"
+            shift
+        ;;
+        --backup-volume-name=*)
+            BACKUP_VOLUME_NAME="${i#*=}"
+            shift
+        ;;
+        *)
+        echo "Unexpected option: $1"
+        exit 1
+        ;;
+    esac
+done
+
+# generate the custom cloud-init file. Cloud init installs and configures sshd
+SSH_AUTHORIZED_KEY=$(<"$SSH_PUBKEY_PATH")
+eval "$(ssh-agent -s)" > /dev/null
+ssh-add "$SSH_HOME/id_rsa" > /dev/null
+export SSH_AUTHORIZED_KEY="$SSH_AUTHORIZED_KEY"
+
+export FILENAME="$INCUS_HOSTNAME.yml"
+mkdir -p "$PROJECT_PATH/cloud-init"
+YAML_PATH="$PROJECT_PATH/cloud-init/$FILENAME"
+
+# If we are deploying the www, we attach the vm to the underlay via macvlan.
+cat > "$YAML_PATH" <<EOF
+config:
+EOF
+
+
+if [ "$VIRTUAL_MACHINE" = base ]; then
+    cat >> "$YAML_PATH" <<EOF
+  limits.cpu: 4
+  limits.memory: 4096MB
+
+EOF
+fi
+
+if [ "$VIRTUAL_MACHINE" = www ]; then
+    cat >> "$YAML_PATH" <<EOF
+  limits.cpu: "${WWW_SERVER_CPU_COUNT}"
+  limits.memory: "${WWW_SERVER_MEMORY_MB}MB"
+
+EOF
+fi
+
+if [ "$VIRTUAL_MACHINE" = btcpayserver ]; then
+    cat >> "$YAML_PATH" <<EOF
+  limits.cpu: "${BTCPAY_SERVER_CPU_COUNT}"
+  limits.memory: "${BTCPAY_SERVER_MEMORY_MB}MB"
+
+EOF
+
+elif [ "$VIRTUAL_MACHINE" = lnplayserver ]; then
+    cat >> "$YAML_PATH" <<EOF
+  limits.cpu: "${LNPLAY_SERVER_CPU_COUNT}"
+  limits.memory: "${LNPLAY_SERVER_MEMORY_MB}MB"
+
+EOF
+
+fi
+
+# if VIRTUAL_MACHINE=base, then we doing the base image.
+if [ "$VIRTUAL_MACHINE" = base ]; then
+    # this is for the base image only...
+    cat >> "$YAML_PATH" <<EOF
+  user.vendor-data: |
+    #cloud-config
+    package_update: true
+    package_upgrade: false
+    package_reboot_if_required: false
+
+    preserve_hostname: false
+    fqdn: ${BASE_IMAGE_VM_NAME}
+
+    packages:
+      - curl
+      - ssh-askpass
+      - apt-transport-https
+      - ca-certificates
+      - gnupg-agent
+      - software-properties-common
+      - lsb-release
+      - net-tools
+      - htop
+      - rsync
+      - duplicity
+      - sshfs
+      - fswatch
+      - jq
+      - git
+      - nano
+      - wait-for-it
+      - dnsutils
+      - wget
+
+    groups:
+      - docker
+
+    users:
+      - name: ubuntu
+        groups: docker
+        shell: /bin/bash
+        lock_passwd: false
+        sudo: ALL=(ALL) NOPASSWD:ALL
+        ssh_authorized_keys:
+          - ${SSH_AUTHORIZED_KEY}
+
+EOF
+
+    if [ "$REGISTRY_URL" != "https://index.docker.io/v1" ]; then
+        cat >> "$YAML_PATH" <<EOF
+    write_files:
+      - path: /etc/docker/daemon.json
+        permissions: 0644
+        owner: root
+        content: |
+            {
+                "registry-mirrors": [
+                  "${REGISTRY_URL}"
+                ]
+            }
+
+
+EOF
+
+    fi
+
+fi
+
+if [ "$VIRTUAL_MACHINE" = base ]; then
+    cat >> "$YAML_PATH" <<EOF
+    runcmd:
+      - sudo mkdir -m 0755 -p /etc/apt/keyrings
+      - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+      - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable" | sudo tee /etc/apt/sources.list.d/docker.list
+      - sudo apt-get update
+      - sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+      - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server
+
+EOF
+
+fi
+
+if [ "$VIRTUAL_MACHINE" != base ]; then
+    # all other machines that are not the base image
+    cat >> "$YAML_PATH" <<EOF
+  user.vendor-data: |
+    #cloud-config
+    apt_mirror: http://us.archive.ubuntu.com/ubuntu/
+    package_update: false
+    package_upgrade: false
+    package_reboot_if_required: false
+
+    preserve_hostname: true
+    fqdn: ${FQDN}
+
+EOF
+fi
+
+if [ "$VIRTUAL_MACHINE" = www ] || [ "$VIRTUAL_MACHINE" = btcpayserver ]; then
+    # all other machines that are not the base image
+    cat >> "$YAML_PATH" <<EOF
+    resize_rootfs: false
+    disk_setup:
+      /dev/sdb:
+        table_type: 'gpt'
+        layout: true
+        overwrite: false
+
+    fs_setup:
+      - label: docker-data
+        filesystem: 'ext4'
+        device: '/dev/sdb1'
+        overwrite: false
+
+    mounts:
+      - [ sdb, /var/lib/docker ]
+
+    mount_default_fields: [ None, None, "auto", "defaults,nofail", "0", "2" ]
+
+EOF
+fi
+
+if [ "$VIRTUAL_MACHINE" != base ]; then
+    cat >> "$YAML_PATH" <<EOF
+  user.network-config: |
+    version: 2
+    ethernets:
+      enp5s0:
+        dhcp4: true
+        dhcp4-overrides:
+          route-metric: 50
+        match:
+          macaddress: ${MAC_ADDRESS_TO_PROVISION}
+        set-name: enp5s0
+EOF
+fi
+
+# TODO try to get DHCP working reliably.
+if [ "$VIRTUAL_MACHINE" = btcpayserver ]; then
+    cat >> "$YAML_PATH" <<EOF
+      enp6s0:
+        addresses:
+          - 10.10.10.66/24
+
+EOF
+fi
+
+if [ "$VIRTUAL_MACHINE" = www ]; then
+    cat >> "$YAML_PATH" <<EOF
+      enp6s0:
+        addresses:
+          - 10.10.10.65/24
+
+EOF
+fi
+
+# All profiles get a root disk and cloud-init config.
+cat >> "$YAML_PATH" <<EOF
+description: Default incus profile for ${FILENAME}
+devices:
+
+EOF
+
+if [ "$VIRTUAL_MACHINE" = lnplayserver ]; then
+    # All profiles get a root disk and cloud-init config.
+    cat >> "$YAML_PATH" <<EOF
+  root:
+    path: /
+    pool: ss-base
+    type: disk
+    size: 20GiB
+EOF
+else
+    # All profiles get a root disk and cloud-init config.
+    cat >> "$YAML_PATH" <<EOF
+  root:
+    path: /
+    pool: ss-base
+    type: disk
+EOF
+
+fi
+
+cat >> "$YAML_PATH" <<EOF
+  config:
+    source: cloud-init:config
+    type: disk
+EOF
+
+if [ "$VIRTUAL_MACHINE" = www ] || [ "$VIRTUAL_MACHINE" = btcpayserver ]; then
+    cat >> "$YAML_PATH" <<EOF
+  ss-data:
+    path: ${REMOTE_DATA_PATH}
+    pool: ss-base
+    source: ${SSDATA_VOLUME_NAME}
+    type: disk
+  ss-backup:
+    path: ${REMOTE_BACKUP_PATH}
+    pool: ss-base
+    source: ${BACKUP_VOLUME_NAME}
+    type: disk
+EOF
+fi
+
+# Stub out the network piece for the base image.
+if [ "$VIRTUAL_MACHINE" = base ]; then
+    cat >> "$YAML_PATH" <<EOF
+  enp6s0:
+    name: enp6s0
+    network: incusbr0
+    type: nic
+name: ${FILENAME}
+EOF
+
+else
+
+    # all other vms attach to the network underlay
+    cat >> "$YAML_PATH" <<EOF
+  enp5s0:
+    nictype: macvlan
+    parent: ${DATA_PLANE_MACVLAN_INTERFACE}
+    type: nic
+EOF
+
+    if [ "$VIRTUAL_MACHINE" = www ] || [ "$VIRTUAL_MACHINE" = btcpayserver ]; then
+        cat >> "$YAML_PATH" <<EOF
+  enp6s0:
+    name: enp6s0
+    network: ss-ovn
+    type: nic
+EOF
+    fi
+
+    cat >> "$YAML_PATH" <<EOF
+name: ${PRIMARY_DOMAIN}
+EOF
+
+fi
+
+if [ "$VIRTUAL_MACHINE" = base ]; then
+    if ! incus profile list --format csv --project default | grep -q "$INCUS_HOSTNAME"; then
+        incus profile create "$INCUS_HOSTNAME" --project default
+    fi
+
+    # configure the profile with our generated cloud-init.yml file.
+    incus profile edit "$INCUS_HOSTNAME" --project default < "$YAML_PATH"
+else
+    if ! incus profile list --format csv | grep -q "$INCUS_HOSTNAME"; then
+        incus profile create "$INCUS_HOSTNAME"
+    fi
+
+    # configure the profile with our generated cloud-init.yml file.
+    incus profile edit "$INCUS_HOSTNAME" < "$YAML_PATH"
+fi

deployment/up.sh

@@ -0,0 +1,478 @@
+#!/bin/bash
+
+set -exu
+cd "$(dirname "$0")"
+
+# check to ensure dependencies are met.
+for cmd in wait-for-it dig rsync sshfs incus; do
+    if ! command -v "$cmd" >/dev/null 2>&1; then
+        echo "This script requires \"${cmd}\" to be installed. Please run 'install.sh'."
+        exit 1
+    fi
+done
+
+# do a spot check; if we are on production warn.
+if incus remote get-default | grep -q "production"; then
+    echo "WARNING: You are running command against a production system!"
+    echo ""
+
+    # check if there are any uncommited changes. It's dangerous to 
+    # alter production systems when you have commits to make or changes to stash.
+    if git update-index --refresh | grep -q "needs update"; then
+        echo "ERROR: You have uncommited changes! You MUST commit or stash all changes to continue."
+        exit 1
+    fi
+
+    RESPONSE=
+    read -r -p "         Are you sure you want to continue (y)  ": RESPONSE
+    if [ "$RESPONSE" != "y" ]; then
+        echo "STOPPING."
+        exit 1
+    fi
+
+fi
+
+OTHER_SITES_LIST=
+PRIMARY_DOMAIN=
+RUN_CERT_RENEWAL=true
+SKIP_BASE_IMAGE_CREATION=false
+RESTORE_WWW=false
+RESTORE_CERTS=false
+BACKUP_CERTS=true
+BACKUP_BTCPAY=true
+SKIP_BTCPAY_SERVER=false
+SKIP_WWW_SERVER=false
+SKIP_LNPLAY_SERVER=false
+BACKUP_BTCPAY_ARCHIVE_PATH= 
+RESTORE_BTCPAY=false
+UPDATE_BTCPAY=false
+REMOTE_NAME="$(incus remote get-default)"
+USER_SAYS_YES=false
+
+WWW_SERVER_MAC_ADDRESS=
+BTCPAY_SERVER_MAC_ADDRESS=
+LNPLAY_SERVER_MAC_ADDRESS=
+LNPLAY_ENV_PATH=
+LNPLAY_VM_EXPIRATION_DATE=
+LNPLAY_ORDER_ID=
+
+# grab any modifications from the command line.
+for i in "$@"; do
+    case $i in
+        --restore-certs)
+            RESTORE_CERTS=true
+            shift
+        ;;
+        --restore-wwwserver)
+            RESTORE_WWW=true
+            shift
+        ;;
+        --restore-btcpay)
+            RESTORE_BTCPAY=true
+            shift
+        ;;
+        --skip-btcpayserver)
+            SKIP_BTCPAY_SERVER=true
+            shift
+        ;;
+        --skip-wwwserver)
+            SKIP_WWW_SERVER=true
+            shift
+        ;;
+        --skip-lnplayserver)
+            SKIP_LNPLAY_SERVER=true
+            shift
+        ;;
+        --backup-btcpayserver)
+            BACKUP_BTCPAY=true
+            shift
+        ;;
+        --backup-archive-path=*)
+            BACKUP_BTCPAY_ARCHIVE_PATH="${i#*=}"
+            shift
+        ;;
+        --update-btcpay)
+            UPDATE_BTCPAY=true
+            shift
+        ;;
+        --skip-base-image)
+            SKIP_BASE_IMAGE_CREATION=true
+            shift
+        ;;
+        --no-cert-renew)
+            RUN_CERT_RENEWAL=false
+            shift
+        ;;
+        --lnplay-env-path=*)
+            LNPLAY_ENV_PATH="${i#*=}"
+            shift
+        ;;
+        --vm-expiration-date=*)
+            LNPLAY_VM_EXPIRATION_DATE="${i#*=}"
+            shift
+        ;;
+        --order-id=*)
+            LNPLAY_ORDER_ID="${i#*=}"
+            shift
+        ;;
+        -y)
+            USER_SAYS_YES=true
+            shift
+        ;;
+        *)
+        echo "Unexpected option: $1"
+        exit 1
+        ;;
+    esac
+done
+
+
+if [ "$RESTORE_BTCPAY" = true ] && [ -z "$BACKUP_BTCPAY_ARCHIVE_PATH" ]; then
+    echo "ERROR: Use the '--backup-archive-path=/path/to/btcpay/archive.tar.gz' option when restoring btcpay server."
+    exit 1
+fi
+
+if [ "$RESTORE_BTCPAY" = true ] && [ ! -f "$BACKUP_BTCPAY_ARCHIVE_PATH" ]; then
+    echo "ERROR: The backup archive path you specified DOES NOT exist!"
+    exit 1
+fi
+
+. ./remote_env.sh
+
+export REGISTRY_DOCKER_IMAGE="registry:2"
+export BACKUP_CERTS="$BACKUP_CERTS"
+export RESTORE_BTCPAY="$RESTORE_BTCPAY"
+export RESTORE_WWW="$RESTORE_WWW"
+export BACKUP_BTCPAY="$BACKUP_BTCPAY"
+export RUN_CERT_RENEWAL="$RUN_CERT_RENEWAL"
+export REMOTE_NAME="$REMOTE_NAME"
+export REMOTE_PATH="$REMOTES_PATH/$REMOTE_NAME"
+export USER_SAYS_YES="$USER_SAYS_YES"
+export BACKUP_BTCPAY_ARCHIVE_PATH="$BACKUP_BTCPAY_ARCHIVE_PATH"
+export RESTORE_CERTS="$RESTORE_CERTS"
+
+# todo convert this to Trezor-T
+SSH_PUBKEY_PATH="$SSH_HOME/id_rsa.pub"
+export SSH_PUBKEY_PATH="$SSH_PUBKEY_PATH"
+
+# ensure our remote path is created.
+mkdir -p "$REMOTE_PATH"
+
+REMOTE_DEFINITION="$REMOTE_PATH/remote.conf"
+if [ ! -f "$REMOTE_DEFINITION" ]; then
+    echo "ERROR: The remote definition could not be found. You may need to re-run 'ss-remote'."
+    exit 1
+fi
+
+export REMOTE_DEFINITION="$REMOTE_DEFINITION"
+source "$REMOTE_DEFINITION"
+
+
+# this is our password generation mechanism. Relying on GPG for secure password generation
+# TODO see if this is a secure way to do it.
+function new_pass {
+    gpg --gen-random --armor 1 25
+}
+
+function stub_site_definition {
+    mkdir -p "$SITE_PATH" "$PROJECT_PATH/sites"
+
+    # create a symlink from the PROJECT_PATH/sites/DOMAIN_NAME to the ss-sites/domain name
+    DOMAIN_SYMLINK_PATH="$PROJECT_PATH/sites/$DOMAIN_NAME"
+    if [ ! -L "$DOMAIN_SYMLINK_PATH" ]; then
+        ln -r -s "$SITE_PATH" "$DOMAIN_SYMLINK_PATH"
+    fi
+
+    if [ ! -f "$SITE_PATH/site.conf" ]; then
+        # check to see if the enf file exists. exist if not.
+        SITE_DEFINITION_PATH="$SITE_PATH/site.conf"
+        if [ ! -f "$SITE_DEFINITION_PATH" ]; then
+
+            # stub out a site.conf with new passwords.
+            cat >"$SITE_DEFINITION_PATH" <<EOL
+# https://www.sovereign-stack.org/ss-up/#siteconf
+
+DOMAIN_NAME="${DOMAIN_NAME}"
+# BTCPAY_ALT_NAMES="tip,store,pay,send"
+SITE_LANGUAGE_CODES="en"
+DUPLICITY_BACKUP_PASSPHRASE="$(new_pass)"
+DEPLOY_GHOST=true
+
+DEPLOY_NEXTCLOUD=false
+DEPLOY_NOSTR=false
+NOSTR_ACCOUNT_PUBKEY=
+DEPLOY_GITEA=false
+GHOST_MYSQL_PASSWORD="$(new_pass)"
+GHOST_MYSQL_ROOT_PASSWORD="$(new_pass)"
+NEXTCLOUD_MYSQL_PASSWORD="$(new_pass)"
+NEXTCLOUD_MYSQL_ROOT_PASSWORD="$(new_pass)"
+GITEA_MYSQL_PASSWORD="$(new_pass)"
+GITEA_MYSQL_ROOT_PASSWORD="$(new_pass)"
+
+
+#GHOST_DEPLOY_SMTP=true
+#MAILGUN_FROM_ADDRESS=false
+#MAILGUN_SMTP_USERNAME=
+#MAILGUN_SMTP_PASSWORD=
+
+EOL
+
+            chmod 0744 "$SITE_DEFINITION_PATH"
+            echo "INFO: we stubbed a new site.conf for you at '$SITE_DEFINITION_PATH'. Go update it!"
+            exit 1
+
+        fi
+    fi
+
+}
+
+PROJECT_NAME="$(incus info | grep "project:" | awk '{print $2}')"
+export PROJECT_NAME="$PROJECT_NAME"
+export PROJECT_PATH="$PROJECTS_PATH/$PROJECT_NAME"
+export SKIP_BTCPAY_SERVER="$SKIP_BTCPAY_SERVER"
+export SKIP_WWW_SERVER="$SKIP_WWW_SERVER"
+export SKIP_LNPLAY_SERVER="$SKIP_LNPLAY_SERVER"
+
+
+mkdir -p "$PROJECT_PATH" "$REMOTE_PATH/projects"
+
+# create a symlink from ./remotepath/projects/project
+PROJECT_SYMLINK="$REMOTE_PATH/projects/$PROJECT_NAME"
+if [ ! -L "$PROJECT_SYMLINK" ]; then
+    ln -r -s "$PROJECT_PATH" "$PROJECT_SYMLINK"
+fi
+
+# check to see if the enf file exists. exist if not.
+PROJECT_DEFINITION_PATH="$PROJECT_PATH/project.conf"
+if [ ! -f "$PROJECT_DEFINITION_PATH" ]; then
+
+        # stub out a project.conf
+    cat >"$PROJECT_DEFINITION_PATH" <<EOL
+# see https://www.sovereign-stack.org/ss-up/#projectconf for more info.
+
+PRIMARY_DOMAIN="domain0.tld"
+# OTHER_SITES_LIST="domain1.tld,domain2.tld,domain3.tld"
+
+WWW_SERVER_MAC_ADDRESS=
+# WWW_SSDATA_DISK_SIZE_GB=100
+# WWW_SERVER_CPU_COUNT="6"
+# WWW_SERVER_MEMORY_MB="4096"
+
+BTCPAY_SERVER_MAC_ADDRESS=
+# BTCPAY_SERVER_CPU_COUNT="4"
+# BTCPAY_SERVER_MEMORY_MB="4096"
+
+LNPLAY_SERVER_MAC_ADDRESS=
+# LNPLAY_SERVER_CPU_COUNT="4"
+# LNPLAY_SERVER_MEMORY_MB="4096"
+
+# BITCOIN_CHAIN=mainnet
+
+EOL
+
+    chmod 0744 "$PROJECT_DEFINITION_PATH"
+    echo "INFO: we stubbed a new project.conf for you at '$PROJECT_DEFINITION_PATH'. Go update it!"
+    echo "INFO: Learn more at https://www.sovereign-stack.org/ss-up/"
+
+    exit 1
+fi
+
+. ./project_env.sh
+
+if [ -z "$PRIMARY_DOMAIN" ]; then
+    echo "ERROR: The PRIMARY_DOMAIN is not specified. Check your project.conf."
+    exit 1
+fi
+
+source ./domain_list.sh
+
+# let's provision our primary domain first.
+export DOMAIN_NAME="$PRIMARY_DOMAIN"
+export PRIMARY_DOMAIN="$PRIMARY_DOMAIN"
+export BITCOIN_CHAIN="$BITCOIN_CHAIN"
+export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
+export PRIMARY_SITE_PATH="$SITES_PATH/$PRIMARY_DOMAIN"
+
+stub_site_definition
+
+# bring the VMs up under the primary domain name.
+
+export UPDATE_BTCPAY="$UPDATE_BTCPAY"
+
+# iterate over all our server endpoints and provision them if needed.
+# www
+VPS_HOSTNAME=
+
+. ./base.sh
+if ! incus image list --format csv | grep -q "$DOCKER_BASE_IMAGE_NAME"; then
+    # create the incus base image.
+    if [ "$SKIP_BASE_IMAGE_CREATION" = false ]; then
+        ./create_base.sh
+    fi
+fi
+
+
+VMS_TO_PROVISION=""
+if [ -n "$WWW_SERVER_MAC_ADDRESS" ] && [ "$SKIP_WWW_SERVER" = false ]; then
+    VMS_TO_PROVISION="www"
+fi
+
+if [ -n "$BTCPAY_SERVER_MAC_ADDRESS" ] && [ "$SKIP_BTCPAY_SERVER" = false ]; then
+    VMS_TO_PROVISION="$VMS_TO_PROVISION btcpayserver"
+fi
+
+if [ -n "$LNPLAY_SERVER_MAC_ADDRESS" ] || [ "$SKIP_LNPLAY_SERVER" = false ]; then
+    VMS_TO_PROVISION="$VMS_TO_PROVISION lnplayserver"
+fi
+
+for VIRTUAL_MACHINE in $VMS_TO_PROVISION; do
+
+    export VIRTUAL_MACHINE="$VIRTUAL_MACHINE"
+    FQDN=
+
+    export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
+
+    source "$SITE_PATH/site.conf"
+    source ./project/domain_env.sh
+
+    # VALIDATE THE INPUT from the ENVFILE
+    if [ -z "$DOMAIN_NAME" ]; then
+        echo "ERROR: DOMAIN_NAME not specified in your site.conf."
+        exit 1
+    fi
+
+    # Goal is to get the macvlan interface.
+    INCUS_SS_CONFIG_LINE=
+    if incus network list --format csv --project default | grep incusbr0 | grep -q "ss-config"; then
+        INCUS_SS_CONFIG_LINE="$(incus network list --format csv --project default | grep incusbr0 | grep ss-config)"
+    fi
+
+    if [ -z "$INCUS_SS_CONFIG_LINE" ]; then
+        echo "ERROR: the MACVLAN interface has not been specified. You may need to run 'ss-remote' again."
+        exit 1
+    fi
+
+    CONFIG_ITEMS="$(echo "$INCUS_SS_CONFIG_LINE" | awk -F'"' '{print $2}')"
+    DATA_PLANE_MACVLAN_INTERFACE="$(echo "$CONFIG_ITEMS" | cut -d ',' -f2)"
+    export DATA_PLANE_MACVLAN_INTERFACE="$DATA_PLANE_MACVLAN_INTERFACE"
+
+
+    # Now let's switch to the new project to ensure new resources are created under the project scope.
+    if ! incus info | grep "project:" | grep -q "$PROJECT_NAME"; then
+        incus project switch "$PROJECT_NAME"
+    fi
+
+    # check if the OVN network exists in this project.
+    if ! incus network list | grep -q "ss-ovn"; then
+        incus network create ss-ovn --type=ovn network=incusbr1 ipv6.address=none
+    fi
+
+    export MAC_ADDRESS_TO_PROVISION=
+    export VPS_HOSTNAME="$VPS_HOSTNAME"
+    export FQDN="$VPS_HOSTNAME.$DOMAIN_NAME"
+
+    if [ "$VIRTUAL_MACHINE" = www ] && [ -n "$WWW_SERVER_MAC_ADDRESS" ]; then
+        FQDN="$WWW_HOSTNAME.$DOMAIN_NAME"
+        VPS_HOSTNAME="$WWW_HOSTNAME"
+        MAC_ADDRESS_TO_PROVISION="$WWW_SERVER_MAC_ADDRESS"
+
+    elif [ "$VIRTUAL_MACHINE" = btcpayserver ] && [ -n "$BTCPAY_SERVER_MAC_ADDRESS" ]; then
+        FQDN="$BTCPAY_SERVER_HOSTNAME.$DOMAIN_NAME"
+        VPS_HOSTNAME="$BTCPAY_SERVER_HOSTNAME"
+        MAC_ADDRESS_TO_PROVISION="$BTCPAY_SERVER_MAC_ADDRESS"
+    
+    elif [ "$VIRTUAL_MACHINE" = lnplayserver ] && [ -n "$LNPLAY_SERVER_MAC_ADDRESS" ]; then
+        FQDN="$LNPLAY_SERVER_HOSTNAME.$DOMAIN_NAME"
+        VPS_HOSTNAME="$LNPLAY_SERVER_HOSTNAME"
+        MAC_ADDRESS_TO_PROVISION="$LNPLAY_SERVER_MAC_ADDRESS"
+
+    elif [ "$VIRTUAL_MACHINE" = "$BASE_IMAGE_VM_NAME" ]; then
+        FQDN="$BASE_IMAGE_VM_NAME"
+    fi
+
+    export FQDN="$FQDN"
+    export INCUS_VM_NAME="${FQDN//./-}"
+    export MAC_ADDRESS_TO_PROVISION="$MAC_ADDRESS_TO_PROVISION"
+    export PROJECT_PATH="$PROJECT_PATH"
+
+    ./deploy_vm.sh
+
+done
+
+# let's stub out the rest of our site definitions, if any.
+for DOMAIN_NAME in ${OTHER_SITES_LIST//,/ }; do
+    export DOMAIN_NAME="$DOMAIN_NAME"
+    export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
+
+    # stub out the site_defition if it's doesn't exist.
+    stub_site_definition
+done
+
+if [ "$SKIP_BTCPAY_SERVER" = false ]; then
+    if [ -n "$BTCPAY_SERVER_MAC_ADDRESS" ]; then
+        export DOCKER_HOST="ssh://ubuntu@$BTCPAY_SERVER_FQDN"
+        ./project/btcpayserver/go.sh
+    fi
+fi
+
+if [ "$SKIP_WWW_SERVER" = false ]; then
+    # now let's run the www and btcpay-specific provisioning scripts.
+    if [ -n "$WWW_SERVER_MAC_ADDRESS" ]; then
+        export DOCKER_HOST="ssh://ubuntu@$WWW_FQDN"
+
+        # enable docker swarm mode so we can support docker stacks.
+        if docker info | grep -q "Swarm: inactive"; then
+            docker swarm init --advertise-addr enp6s0
+        fi
+
+        ./project/www/go.sh
+    fi
+fi
+
+# don't run lnplay stuff if user specifies --skip-lnplay
+if [ "$SKIP_LNPLAY_SERVER" = false ]; then
+    # now let's run the www and btcpay-specific provisioning scripts.
+    if [ -n "$LNPLAY_SERVER_MAC_ADDRESS" ]; then
+        export DOCKER_HOST="ssh://ubuntu@$LNPLAY_SERVER_FQDN"
+
+        LNPLAY_ENV_FILE="$PRIMARY_SITE_PATH/$LNPLAY_SERVER_FQDN/lnplay.conf"
+        if [ ! -f "$LNPLAY_ENV_FILE" ]; then
+            # and we have to set our environment file as well.
+            cat > "$LNPLAY_ENV_FILE" <<EOL
+DOCKER_HOST=ssh://ubuntu@${LNPLAY_SERVER_FQDN}
+BACKEND_FQDN=lnplay.${PRIMARY_DOMAIN}
+FRONTEND_FQDN=remote.${PRIMARY_DOMAIN}
+ENABLE_TLS=true
+BTC_CHAIN=${BITCOIN_CHAIN}
+CHANNEL_SETUP=none
+LNPLAY_SERVER_PATH=${SITES_PATH}/${PRIMARY_DOMAIN}/lnplayserver
+DEPLOY_PRISM_PLUGIN=true
+EOL
+
+        fi
+
+        INCUS_LNPLAYSERVER_IMAGE_NAME="lnplayserver-$DOMAIN_NAME"
+        if ! incus image list -q --format csv | grep -q "$INCUS_LNPLAYSERVER_IMAGE_NAME"; then
+
+            # do all the docker image creation steps, but don't run services.
+            bash -c "./project/lnplay/up.sh -y --no-services  --lnplay-conf-path=$LNPLAY_ENV_FILE"
+
+            # stop the instance so we can get an image yo
+            INCUS_VM_NAME="${LNPLAY_SERVER_FQDN//./-}"
+
+            incus stop "$INCUS_VM_NAME"
+
+            # create the incus image.
+            incus publish -q --public "$INCUS_VM_NAME" --alias="$INCUS_LNPLAYSERVER_IMAGE_NAME" --compression none
+
+            incus start "$INCUS_VM_NAME"
+            sleep 10
+
+            bash -c "./wait_for_ip.sh --incus-name=$INCUS_VM_NAME"
+            sleep 3
+        fi
+
+        # bring up lnplay services.
+        bash -c "./project/lnplay/up.sh -y --lnplay-conf-path=$LNPLAY_ENV_FILE"
+    fi
+fi

deployment/update.sh

@@ -1,129 +0,0 @@
-#!/bin/bash
-
-set -eu
-cd "$(dirname "$0")"
-
-
-# check if there are any uncommited changes. It's dangerous to 
-# alter production systems when you have commits to make or changes to stash.
-if git update-index --refresh | grep -q "needs update"; then
-    echo "ERROR: You have uncommited changes! You MUST commit or stash all changes to continue."
-    exit 1
-fi
-
-
-echo "WARNING: this script backs up your existing remote and saves all data locally in the SSME."
-echo "         Then, all your VMs are destroyed on the remote resulting is destruction of user data."
-echo "         But then we re-create everything using the new codebase, then restore user data to the"
-echo "         newly provisioned VMs."
-
-RESPONSE=
-read -r -p "Are you sure you want to continue (y/n):  ": RESPONSE
-if [ "$RESPONSE" != "y" ]; then
-    echo "STOPPING."
-    exit 0
-fi
-
-
-USER_TARGET_PROJECT=
-
-# grab any modifications from the command line.
-for i in "$@"; do
-    case $i in
-        --project=*)
-            USER_TARGET_PROJECT="${i#*=}"
-            shift
-        ;;
-
-        *)
-        echo "Unexpected option: $1"
-        exit 1
-        ;;
-    esac
-done
-
-
-. ../defaults.sh
-
-. ./remote_env.sh
-
-for PROJECT_CHAIN in ${DEPLOYMENT_STRING//,/ }; do
-    NO_PARENS="${PROJECT_CHAIN:1:${#PROJECT_CHAIN}-2}"
-    PROJECT_PREFIX=$(echo "$NO_PARENS" | cut -d'|' -f1)
-    BITCOIN_CHAIN=$(echo "$NO_PARENS" | cut -d'|' -f2)
-    export PROJECT_PREFIX="$PROJECT_PREFIX"
-    export BITCOIN_CHAIN="$BITCOIN_CHAIN"
-
-    PROJECT_NAME="$PROJECT_PREFIX-$BITCOIN_CHAIN"
-    PROJECT_PATH="$PROJECTS_DIR/$PROJECT_NAME"
-
-    # if the user sets USER_TARGET_PROJECT, let's ensure the project exists.
-    if [ -n "$USER_TARGET_PROJECT" ]; then
-        if ! lxc project list | grep -q "$USER_TARGET_PROJECT"; then
-            echo "ERROR: the project does not exist! Nothing to update."
-            exit 1
-        fi
-
-        if [ "$PROJECT_NAME" != "$USER_TARGET_PROJECT" ]; then
-            echo "INFO: Skipping project '$PROJECT_NAME' since the system owner has used the --project switch."
-            exit
-        fi
-    fi
-
-    export PROJECT_NAME="$PROJECT_NAME"
-    export PROJECT_PATH="$PROJECT_PATH"
-
-    . ./project_env.sh
-
-    # Check to see if any of the VMs actually don't exist. 
-    # (we only migrate instantiated vms)
-    for VM in www btcpayserver; do
-        LXD_NAME="$VM-${DOMAIN_NAME//./-}"
-
-        # if the VM doesn't exist, the we emit an error message and hard quit.
-        if ! lxc list --format csv | grep -q "$LXD_NAME"; then
-            echo "ERROR: there is no VM named '$LXD_NAME'. You probably need to run ss-deploy again."
-            exit 1
-        fi
-    done
-
-    BTCPAY_RESTORE_ARCHIVE_PATH="$SITES_PATH/$PRIMARY_DOMAIN/backups/btcpayserver/$(date +%s).tar.gz"
-    echo "INFO: The BTCPAY_RESTORE_ARCHIVE_PATH for this migration will be: $BTCPAY_RESTORE_ARCHIVE_PATH" 
-
-    # first we run ss-deploy --stop
-    # this grabs a backup of all data (backups are on by default) and saves them to the management machine
-    # the --stop flag ensures that services do NOT come back online.
-    # by default, we grab a backup. 
-
-    # first, let's grab the GIT commit from the remote machine.
-    export DOMAIN_NAME="$PRIMARY_DOMAIN"
-    export SITE_PATH="$SITES_PATH/$PRIMARY_DOMAIN"
-
-    # source the site path so we know what features it has.
-    source ../defaults.sh
-    source "$SITE_PATH/site.conf"
-    source ./project/domain_env.sh
-
-    # now we want to switch the git HEAD of the project subdirectory to the 
-    # version of code that was last used
-    GIT_COMMIT_ON_REMOTE_HOST="$(ssh ubuntu@$BTCPAY_FQDN cat /home/ubuntu/.ss-githead)"
-    cd project/
-    git checkout "$GIT_COMMIT_ON_REMOTE_HOST"
-    cd -
-    sleep 5
-
-    # run deploy which backups up everything, but doesnt restart any services.
-    bash -c "./project/deploy.sh --stop --no-cert-renew --backup-archive-path=$BTCPAY_RESTORE_ARCHIVE_PATH"
-
-    # call the destroy script. If user proceed, then user data is DESTROYED!
-    ./destroy.sh
-
-    cd project/
-    git checkout "$TARGET_PROJECT_GIT_COMMIT"
-    cd -
-
-    sleep 5
-    # Then we can run a restore operation and specify the backup archive at the CLI.
-    bash -c "./project/deploy.sh -y --restore-www --restore-btcpay --backup-archive-path=$BTCPAY_RESTORE_ARCHIVE_PATH"
-
-done

deployment/wait_for_ip.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -e
+
+INCUS_INSTANCE_NAME=
+
+# grab any modifications from the command line.
+for i in "$@"; do
+    case $i in
+        --incus-name=*)
+            INCUS_INSTANCE_NAME="${i#*=}"
+            shift
+        ;;
+        *)
+        echo "Unexpected option: $1"
+        exit 1
+        ;;
+    esac
+done
+
+# if the invoker did not set the instance name, throw an error.
+if [ -z "$INCUS_INSTANCE_NAME" ]; then
+    echo "ERROR: The instance name was not specified. Use '--incus-name' when calling wait_for_ip.sh."
+    exit 1
+fi
+
+if ! incus list --format csv | grep -q "$INCUS_INSTANCE_NAME"; then
+    echo "ERROR: the instance '$INCUS_INSTANCE_NAME' does not exist."
+    exit 1
+fi
+
+IP_V4_ADDRESS=
+while true; do
+    IP_V4_ADDRESS="$(incus list "$INCUS_INSTANCE_NAME" --format csv --columns=4 | grep enp5s0 | grep -Eo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')" || true
+    export IP_V4_ADDRESS="$IP_V4_ADDRESS"
+    if [ -n "$IP_V4_ADDRESS" ]; then
+        # give the machine extra time to spin up.
+        wait-for-it -t 300 "$IP_V4_ADDRESS:22"
+        break
+    else
+        sleep 1
+        printf '.'
+    fi
+done
+
+# wait for cloud-init to complet before returning.
+while incus exec "$INCUS_INSTANCE_NAME" -- [ ! -f /var/lib/cloud/instance/boot-finished ]; do
+    sleep 1
+done
+
+sleep 1

install.sh

@@ -1,58 +1,50 @@
 #!/bin/bash
 
-set -eu
+set -exu
 cd "$(dirname "$0")"
 
-# see https://www.sovereign-stack.org/management/
+# https://www.sovereign-stack.org/install/
 
-. ./defaults.sh
-
-# the DISK variable here tells us which disk (partition) the admin wants to use for 
-# lxd resources. By default, we provision the disk under / as a loop device. Admin
-# can override with CLI modifications.
-DISK="rpool/lxd"
-
-#DISK="/dev/sda1"
-
-export DISK="$DISK"
-
-# let's check to ensure the management machine is on the Baseline ubuntu 21.04
-if ! lsb_release -d | grep -q "Ubuntu 22.04"; then
-    echo "ERROR: Your machine is not running the Ubuntu 22.04 LTS baseline OS on your management machine."
+# this script is not meant to be executed from the SSME; Let's let's check and abort if so.
+if [ "$(hostname)" = ss-mgmt ]; then
+    echo "ERROR: This command is meant to be executed from the bare metal management machine -- not the SSME."
     exit 1
 fi
 
-# install snap
-if ! snap list | grep -q lxd; then
-    sudo snap install lxd --channel=5.11/stable
-    sleep 5
+# ensure the iptables forward policy is set to ACCEPT so your host can act as a router
+# Note this is necessary if docker is running (or has been previuosly installed) on the
+# same host running incus.
+sudo iptables -F FORWARD
+sudo iptables -P FORWARD ACCEPT
 
-    # run lxd init on the remote server./dev/nvme1n1
-    # 
-    cat <<EOF | lxd init --preseed
+# run the incus install script.
+sudo bash -c ./install_incus.sh
+
+# run incus init
+cat <<EOF | sudo incus admin init --preseed
 config: {}
 networks:
 - config:
     ipv4.address: auto
     ipv4.dhcp: true
-    ipv4.nat: true
     ipv6.address: none
   description: "Default network bridge for ss-mgmt outbound network access."
-  name: lxdbr0
+  name: incusbr0
   type: bridge
+  project: default
 storage_pools:
 - config:
-    source: ${DISK}
+    size: 30GiB
   description: ""
   name: sovereign-stack
   driver: zfs
 profiles:
 - config: {}
-  description: ""
+  description: "Default profile for ss-mgmt."
   devices:
     enp5s0:
       name: enp5s0
-      network: lxdbr0
+      network: incusbr0
       type: nic
     root:
       path: /
@@ -60,91 +52,163 @@ profiles:
       type: disk
   name: default
 projects: []
+cluster: null
 
 EOF
 
+
+. ./deployment/deployment_defaults.sh
+
+. ./deployment/base.sh
+
+
+# we need to get the base image. IMport it if it's cached, else download it then cache it.
+if ! incus image list | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then
+        # if the image if cached locally, import it from disk, otherwise download it from ubuntu
+    IMAGE_PATH="$HOME/ss/cache/ss-ubuntu-jammy"
+    IMAGE_IDENTIFIER=$(find "$IMAGE_PATH" | grep ".qcow2" | head -n1 | cut -d "." -f1)
+    METADATA_FILE="$IMAGE_PATH/meta-$IMAGE_IDENTIFIER.tar.xz"
+    IMAGE_FILE="$IMAGE_PATH/$IMAGE_IDENTIFIER.qcow2"
+    if [ -d "$IMAGE_PATH" ] && [ -f "$METADATA_FILE" ] && [ -f "$IMAGE_FILE" ]; then
+        incus image import "$METADATA_FILE" "$IMAGE_FILE" --alias "$UBUNTU_BASE_IMAGE_NAME"
+    else
+        incus image copy "images:$BASE_INCUS_IMAGE" local: --alias "$UBUNTU_BASE_IMAGE_NAME" --vm --auto-update
+        mkdir -p "$IMAGE_PATH"
+        incus image export "$UBUNTU_BASE_IMAGE_NAME" "$IMAGE_PATH" --vm
+    fi
 fi
 
-# pull the vm image down if it's not there.
-if ! lxc image list | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then
-    lxc image copy "images:$BASE_LXC_IMAGE" local: --alias "$UBUNTU_BASE_IMAGE_NAME" --vm --auto-update
+# if the ss-mgmt doesn't exist, create it.
+SSH_PATH="$HOME/.ssh"
+SSH_PRIVKEY_PATH="$SSH_PATH/id_rsa"
+SSH_PUBKEY_PATH="$SSH_PRIVKEY_PATH.pub"
+
+if [ ! -f "$SSH_PRIVKEY_PATH" ]; then
+    ssh-keygen -f "$SSH_PRIVKEY_PATH" -t rsa -b 4096
 fi
 
-if ! lxc list --format csv | grep -q ss-mgmt; then
-    lxc init "images:$BASE_LXC_IMAGE" ss-mgmt --vm -c limits.cpu=4 -c limits.memory=4GiB --profile=default
+# add SSH_PUBKEY_PATH to authorized_keys
+grep -qxF "$(cat "$SSH_PUBKEY_PATH")" "$SSH_PATH/authorized_keys" || cat "$SSH_PUBKEY_PATH" >> "$SSH_PATH/authorized_keys"
+
+FROM_BUILT_IMAGE=false
+if ! incus list --format csv | grep -q ss-mgmt; then
+
+    # TODO check to see if there's an existing ss-mgmt image to spawn from, otherwise do this.
+    if incus image list | grep -q ss-mgmt; then
+        FROM_BUILT_IMAGE=true
+        incus init ss-mgmt ss-mgmt --vm -c limits.cpu=4 -c limits.memory=4GiB --profile=default
+    else
+        incus init "images:$BASE_INCUS_IMAGE" ss-mgmt --vm -c limits.cpu=4 -c limits.memory=4GiB --profile=default
+    fi
 
-    # mount the pre-verified sovereign stack git repo into the new vm
-    lxc config device add ss-mgmt sscode disk source="$(pwd)" path=/home/ubuntu/sovereign-stack
 fi
 
-if lxc list --format csv | grep -q "ss-mgmt,STOPPED"; then
-    lxc start ss-mgmt
-    sleep 20
+# mount the pre-verified sovereign stack git repo into the new vm
+if ! incus config device show ss-mgmt | grep -q ss-code; then
+    incus config device add ss-mgmt ss-code disk source="$(pwd)" path=/home/ubuntu/sovereign-stack
 fi
 
-. ./management/wait_for_lxc_ip.sh
+# create the ~/ss path and mount it into the vm.
+source ./deployment/deployment_defaults.sh
+source ./deployment/base.sh
 
+mkdir -p "$SS_ROOT_PATH"
 
-# TODO wait for cloud-init to finish (but in the VM)
-# while [ ! -f /var/lib/cloud/instance/boot-finished ]; do
-#     sleep 1
-# done
-
-SSH_PUBKEY_PATH="$HOME/.ssh/id_rsa.pub"
-if [ ! -f "$SSH_PUBKEY_PATH" ]; then
-    ssh-keygen -f "$SSH_HOME/id_rsa" -t ecdsa -b 521 -N ""
+if ! incus config device show ss-mgmt | grep -q ss-root; then
+    incus config device add ss-mgmt ss-root disk source="$SS_ROOT_PATH" path=/home/ubuntu/ss
 fi
 
-# now run the mgmt provisioning script.
+# if a ~/.bitcoin/testnet3/blocks direrectory exists, mount it in.
+BITCOIN_DIR="$HOME/.bitcoin"
+REMOTE_BITCOIN_CACHE_PATH="/home/ubuntu/ss/cache/bitcoin"
+BITCOIN_TESTNET_BLOCKS_PATH="$BITCOIN_DIR/testnet3/blocks"
+if [ -d "$BITCOIN_TESTNET_BLOCKS_PATH" ]; then
+    if ! incus config device show ss-mgmt | grep -q ss-testnet-blocks; then
+        incus config device add ss-mgmt ss-testnet-blocks disk source="$BITCOIN_TESTNET_BLOCKS_PATH" path=$REMOTE_BITCOIN_CACHE_PATH/testnet/blocks
+    fi
+fi
 
+# if a ~/.bitcoin/testnet3/blocks direrectory exists, mount it in.
+BITCOIN_TESTNET_CHAINSTATE_PATH="$BITCOIN_DIR/testnet3/chainstate"
+if [ -d "$BITCOIN_TESTNET_CHAINSTATE_PATH" ]; then
+    if ! incus config device show ss-mgmt | grep -q ss-testnet-chainstate; then
+        incus config device add ss-mgmt ss-testnet-chainstate disk source="$BITCOIN_TESTNET_CHAINSTATE_PATH" path="$REMOTE_BITCOIN_CACHE_PATH/testnet/chainstate"
+    fi
+fi
+
+# if a ~/.bitcoin/blocks dir exists, mount it in.
+BITCOIN_MAINNET_BLOCKS_PATH="$BITCOIN_DIR/blocks"
+if [ -d "$BITCOIN_MAINNET_BLOCKS_PATH" ]; then
+    if ! incus config device show ss-mgmt | grep -q ss-mainnet-blocks; then
+        incus config device add ss-mgmt ss-mainnet-blocks disk source="$BITCOIN_MAINNET_BLOCKS_PATH" path="$REMOTE_BITCOIN_CACHE_PATH/mainnet/blocks"
+    fi
+fi
+
+    # if a ~/.bitcoin/testnet3/blocks direrectory exists, mount it in.
+BITCOIN_MAINNET_CHAINSTATE_PATH="$BITCOIN_DIR/chainstate"
+if [ -d "$BITCOIN_MAINNET_CHAINSTATE_PATH" ]; then
+    if ! incus config device show ss-mgmt | grep -q ss-mainnet-blocks; then
+        incus config device add ss-mgmt ss-mainnet-chainstate disk source="$BITCOIN_MAINNET_CHAINSTATE_PATH" path="$REMOTE_BITCOIN_CACHE_PATH/mainnet/chainstate"
+    fi
+fi
+
+# mount the ssh directory in there.
 if [ -f "$SSH_PUBKEY_PATH" ]; then
-    lxc file push "$SSH_PUBKEY_PATH" ss-mgmt/home/ubuntu/.ssh/authorized_keys
+    if ! incus config device show ss-mgmt | grep -q ss-ssh; then
+        incus config device add ss-mgmt ss-ssh disk source="$HOME/.ssh" path=/home/ubuntu/.ssh
+    fi
 fi
 
-lxc file push ./management/bash_profile ss-mgmt/home/ubuntu/.bash_profile
-lxc file push ./management/bashrc ss-mgmt/home/ubuntu/.bashrc
-lxc file push ./management/motd ss-mgmt/etc/update-motd.d/sovereign-stack
+# start the vm if it's not already running
+if incus list --format csv | grep -q "ss-mgmt,STOPPED"; then
+    incus start ss-mgmt
+    sleep 10
+fi
 
-lxc exec ss-mgmt apt-get update
-lxc exec ss-mgmt -- apt-get install -y openssh-server
-lxc file push ./management/sshd_config ss-mgmt/etc/ssh/sshd_config
-lxc exec ss-mgmt -- sudo systemctl restart sshd
+# wait for the vm to have an IP address
+. ./management/wait_for_ip.sh
 
-# make the Sovereign Stack commands available to the user via ~/.bashrc
-# we use ~/.bashrc
+# do some other preparations for user experience
+incus file push ./management/bash_aliases ss-mgmt/home/ubuntu/.bash_aliases
+incus file push ./management/bash_profile ss-mgmt/home/ubuntu/.bash_profile
+incus file push ./management/bashrc ss-mgmt/home/ubuntu/.bashrc
+incus file push ./management/motd ss-mgmt/etc/update-motd.d/sovereign-stack
+
+# install SSH
+incus exec ss-mgmt apt-get update
+incus exec ss-mgmt -- apt-get install -y openssh-server
+incus file push ./management/sshd_config ss-mgmt/etc/ssh/sshd_config
+incus exec ss-mgmt -- sudo systemctl restart sshd
+
+# add 'ss-manage' to the bare metal ~/.bashrc
 ADDED_COMMAND=false
 if ! < "$HOME/.bashrc" grep -q "ss-manage"; then
     echo "alias ss-manage='$(pwd)/manage.sh \$@'" >> "$HOME/.bashrc"
     ADDED_COMMAND=true
 fi
 
-wait-for-it -t 300 "$IP_V4_ADDRESS:22" > /dev/null 2>&1
-
 # Let's remove any entry in our known_hosts, then add it back.
 # we are using IP address here so we don't have to rely on external DNS 
 # configuration for the base image preparataion.
 ssh-keygen -R "$IP_V4_ADDRESS"
 
-ssh-keyscan -H -t ecdsa "$IP_V4_ADDRESS" >> "$SSH_HOME/known_hosts"
+ssh-keyscan -H "$IP_V4_ADDRESS" >> "$SSH_HOME/known_hosts"
 
 ssh "ubuntu@$IP_V4_ADDRESS" sudo chown -R ubuntu:ubuntu /home/ubuntu
 
-ssh "ubuntu@$IP_V4_ADDRESS" /home/ubuntu/sovereign-stack/management/provision.sh
+if [ "$FROM_BUILT_IMAGE" = false ]; then
+    ssh "ubuntu@$IP_V4_ADDRESS" /home/ubuntu/sovereign-stack/management/provision.sh
 
-lxc restart ss-mgmt
+    incus stop ss-mgmt
+
+    if ! incus image list | grep -q "ss-mgmt"; then
+        echo "Publishing image. Please wait, this may take a while..."
+        incus publish ss-mgmt --alias=ss-mgmt
+    fi
+
+    incus start ss-mgmt
+fi
 
 if [ "$ADDED_COMMAND" = true ]; then
     echo "NOTICE! You need to run 'source ~/.bashrc' before continuing. After that, type 'ss-manage' to enter your management environment."
 fi
-
-
-# As part of the install script, we pull down any other sovereign-stack git repos
-PROJECTS_SCRIPTS_REPO_URL="https://git.sovereign-stack.org/ss/project"
-PROJECTS_SCRIPTS_PATH="$(pwd)/deployment/project"
-if [ ! -d "$PROJECTS_SCRIPTS_PATH" ]; then
-    git clone "$PROJECTS_SCRIPTS_REPO_URL" "$PROJECTS_SCRIPTS_PATH"
-else
-    cd "$PROJECTS_SCRIPTS_PATH"
-    git pull origin main
-    cd -
-fi

install_incus.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -exu
+cd "$(dirname "$0")"
+
+if [ $UID -ne 0 ]; then
+    echo "ERROR: run with sudo."
+    exit 1
+fi
+
+# put the zabbly key in there.
+mkdir -p /etc/apt/keyrings/
+cat <<EOF > /etc/apt/keyrings/zabbly.asc
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQGNBGTlYcIBDACYQoVXVyQ6Y3Of14GwEaiv/RstQ8jWnH441OtvDbD/VVT8yF0P
+pUfypWjQS8aq0g32Qgb9H9+b8UAAKojA2W0szjJFlmmSq19YDMMmNC4AnfeZlKYM
+61Zonna7fPaXmlsTlSiUeo/PGvmAXrkFURC9S8FbhZdWEcUpf9vcKAoEzV8qGA4J
+xbKlj8EOjSkdq3OQ1hHjP8gynbbzMhZQwjbnWqoiPj35ed9EMn+0QcX+GmynGq6T
+hBXdRdeQjZC6rmXzNF2opCyxqx3BJ0C7hUtpHegmeoH34wnJHCqGYkEKFAjlRLoW
+tOzHY9J7OFvB6U7ENtnquj7lg2VQK+hti3uiHW+oide06QgjVw2irucCblQzphgo
+iX5QJs7tgFFDsA9Ee0DZP6cu83hNFdDcXEZBc9MT5Iu0Ijvj7Oeym3DJpkCuIWgk
+SeP56sp7333zrg73Ua7YZsZHRayAe/4YdNUua+90P4GD12TpTtJa4iRWRd7bis6m
+tSkKRj7kxyTsxpEAEQEAAbQmWmFiYmx5IEtlcm5lbCBCdWlsZHMgPGluZm9AemFi
+Ymx5LmNvbT6JAdQEEwEKAD4WIQRO/FkGlssVuHxzo62CzIeXyDjc/QUCZOVhwgIb
+AwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCCzIeXyDjc/W05C/4n
+lGRTlyOETF2K8oWbjtan9wlttQ+pwymJCnP8T+JJDycGL8dPsGdG1ldHdorVZpFi
+1P+Bem9bbiW73TpbX+WuCfP1g3WN7AVa2mYRfSVhsLNeBAMRgWgNW9JYsmg99lmY
+aPsRYZdGu/PB+ffMIyWhjL3CKCbYS6lV5N5Mi4Lobyz/I1Euxpk2vJhhUqh786nJ
+pQpDnvEl1CRANS6JD9bIvEdfatlAhFlrz1TTf6R7SlppyYI7tme4I/G3dnnHWYSG
+cGRaLwpwobTq0UNSO71g7+at9eY8dh5nn2lZUvvxZvlbXoOoPxKUoeGVXqoq5F7S
+QcMVAogYtyNlnLnsUfSPw6YFRaQ5o00h30bR3hk+YmJ47AJCRY9GIc/IEdSnd/Z5
+Ea7CrP2Bo4zxPgcl8fe311FQRTRoWr19l5PXZgGjzy6siXTrYQi6GjLtqVB5SjJf
+rrIIy1vZRyDL96WPu6fS+XQMpjsSygj+DBFk8OAvHhQhMCXHgT4BMyg4D5GE0665
+AY0EZOVhwgEMAMIztf6WlRsweysb0tzktYE5E/GxIK1lwcD10Jzq3ovJJPa2Tg2t
+J6ZBmMQfwU4OYO8lJxlgm7t6MYh41ZZaRhySCtbJiAXqK08LP9Gc1iWLRvKuMzli
+NFSiFDFGT1D6kwucVfL/THxvZlQ559kK+LB4iXEKXz37r+MCX1K9uiv0wn63Vm0K
+gD3HDgfXWYJcNyXXfJBe3/T5AhuSBOQcpa7Ow5n8zJ+OYg3FFKWHDBTSSZHpbJFr
+ArMIGARz5/f+EVj9XGY4W/+ZJlxNh8FzrTLeRArmCWqKLPRG/KF36dTY7MDpOzlw
+vu7frv+cgiXHZ2NfPrkH8oOl4L+ufze5KBGcN0QwFDcuwCkv/7Ft9Ta7gVaIBsK7
+12oHInUJ6EkBovxpuaLlHlP8IfmZLZbbHzR2gR0e6IhLtrzd7urB+gXUtp6+wCL+
+kWD14TTJhSQ+SFU8ajvUah7/1m2bxdjZNp9pzOPGkr/jEjCM0CpZiCY62SeIJqVc
+4/ID9NYLAGmSIwARAQABiQG8BBgBCgAmFiEETvxZBpbLFbh8c6OtgsyHl8g43P0F
+AmTlYcICGwwFCQPCZwAACgkQgsyHl8g43P0wEgv+LuknyXHpYpiUcJOl9Q5yLokd
+o7tJwJ+9Fu7EDAfM7mPgyBj7Ad/v9RRP+JKWHqIYEjyrRnz9lmzciU+LT/CeoQu/
+MgpU8wRI4gVtLkX2238amrTKKlVjQUUNHf7cITivUs/8e5W21JfwvcSzu5z4Mxyw
+L6vMlBUAixtzZSXD6O7MO9uggHUZMt5gDSPXG2RcIgWm0Bd1yTHL7jZt67xBgZ4d
+hUoelMN2XIDLv4SY78jbHAqVN6CLLtWrz0f5YdaeYj8OT6Ohr/iJQdlfVaiY4ikp
+DzagLi0LvG9/GuB9eO6yLuojg45JEH8DC7NW5VbdUITxQe9NQ/j5kaRKTEq0fyZ+
+qsrryTyvXghxK8oMUcI10l8d41qXDDPCA40kruuspCZSAle3zdqpYqiu6bglrgWr
+Zr2Nm9ecm/kkqMIcyJ8e2mlkuufq5kVem0Oez+GIDegvwnK3HAqWQ9lzdWKvnLiE
+gNkvg3bqIwZ/WoHBnSwOwwAzwarJl/gn8OG6CIeP
+=8Uc6
+-----END PGP PUBLIC KEY BLOCK-----
+
+EOF
+
+sh -c 'cat <<EOF > /etc/apt/sources.list.d/zabbly-incus-stable.sources
+Enabled: yes
+Types: deb
+URIs: https://pkgs.zabbly.com/incus/stable
+Suites: $(. /etc/os-release && echo ${VERSION_CODENAME})
+Components: main
+Architectures: $(dpkg --print-architecture)
+Signed-By: /etc/apt/keyrings/zabbly.asc
+
+EOF'
+
+apt-get update
+
+# we || true this here because installing incus fails.
+# TODO see if this can be fixed by installing JUST the incus client.
+# none of the systemd/daemon stuff is needed necessarily.
+apt-get install incus -y
+

manage.sh

@@ -1,28 +1,39 @@
 #!/bin/bash
 
+# https://www.sovereign-stack.org/ss-manage/
+
 set -eu
 cd "$(dirname "$0")"
 
-if ! lxc remote get-default | grep -q "local"; then
-    lxc remote switch "local"
+# check to ensure dependencies are met.
+if ! command -v incus  >/dev/null 2>&1; then
+    echo "This script requires incus to be installed. Please run 'install.sh'."
+    exit 1
+fi
+
+if ! incus remote get-default | grep -q "local"; then
+    incus remote switch "local"
+fi
+
+if ! incus list -q --format csv | grep -q ss-mgmt; then
+    echo "ERROR: the 'ss-mgmt' VM does not exist. You may need to run install.sh"
+    exit 1
 fi
 
 # if the mgmt machine doesn't exist, then warn the user to perform ./install.sh
-if ! lxc list --format csv | grep -q "ss-mgmt"; then
+if ! incus list --format csv | grep -q "ss-mgmt"; then
     echo "ERROR: the management machine VM does not exist. You probably need to run './install.sh'."
     echo "INFO: check out https://www.sovereign-stack.org/tag/code-lifecycle-management/ for more information."
 fi
 
 # if the machine does exist, let's make sure it's RUNNING.
-if lxc list --format csv | grep -q "ss-mgmt,STOPPED"; then
-    echo "INFO: The management machine was in a STOPPED state. Starting the environment. Please wait."
-    lxc start ss-mgmt
+if incus list --format csv | grep -q "ss-mgmt,STOPPED"; then
+    echo "INFO: The SSME was in a STOPPED state. Starting the environment. Please wait."
+    incus start ss-mgmt
     sleep 30
 fi
 
-. ./management/wait_for_lxc_ip.sh
-
-wait-for-it -t 300 "$IP_V4_ADDRESS:22" > /dev/null 2>&1
+. ./management/wait_for_ip.sh
 
 # let's ensure ~/.ssh/ssh_config is using the correct IP address for ss-mgmt.
 ssh ubuntu@"$IP_V4_ADDRESS"

management/bash_aliases

@@ -0,0 +1,12 @@
+#!/bin/bash
+alias ss-help='cat /home/ubuntu/sovereign-stack/deployment/help.txt'
+alias ss-show='/home/ubuntu/sovereign-stack/deployment/show.sh $@'
+alias ss-remote='/home/ubuntu/sovereign-stack/deployment/remote.sh $@'
+alias ss-up='/home/ubuntu/sovereign-stack/deployment/up.sh $@'
+alias ss-down='/home/ubuntu/sovereign-stack/deployment/down.sh $@'
+alias ss-reset='/home/ubuntu/sovereign-stack/deployment/reset.sh $@'
+alias ss-stop='/home/ubuntu/sovereign-stack/deployment/stop.sh $@'
+alias ss-start='/home/ubuntu/sovereign-stack/deployment/start.sh $@'
+alias ss-restore='/home/ubuntu/sovereign-stack/deployment/restore.sh $@'
+
+alias ll='ls -lah'

management/bash_profile

@@ -1,11 +1,6 @@
 #!/bin/bash
 
-alias ss-deploy='/home/ubuntu/sovereign-stack/deployment/project/deploy.sh $@'
-alias ss-remote='/home/ubuntu/sovereign-stack/deployment/remote.sh $@'
-alias ss-show='/home/ubuntu/sovereign-stack/deployment/show.sh $@'
-alias ss-reset='/home/ubuntu/sovereign-stack/deployment/reset.sh $@'
-alias ss-update='/home/ubuntu/sovereign-stack/deployment/update.sh $@'
-alias ss-destroy='/home/ubuntu/sovereign-stack/deployment/destroy.sh $@'
-alias ss-help='cat /home/ubuntu/sovereign-stack/deployment/help.txt'
-
-alias ll='ls -lah'
+# this wires up the aliases for remote ssh sessions.
+if [ -f ~/.bashrc ]; then
+  . ~/.bashrc
+fi

management/provision.sh

@@ -5,9 +5,9 @@ cd "$(dirname "$0")"
 
 # NOTE! This script MUST be executed as root.
 sudo apt-get update
-sudo apt-get install -y gnupg ca-certificates curl lsb-release
+sudo apt-get install -y ca-certificates curl gnupg lsb-release jq bc
 
-mkdir -p /etc/apt/keyrings
+sudo mkdir -m 0755 -p /etc/apt/keyrings
 
 # add the docker gpg key to keyring for docker-ce-cli
 if [ ! -f /etc/apt/keyrings/docker.gpg ]; then
@@ -17,46 +17,26 @@ fi
 
 # TODO REVIEW mgmt software requirements
 sudo apt-get update
-sudo apt-get install -y wait-for-it dnsutils rsync sshfs apt-transport-https docker-ce-cli \
-    libcanberra-gtk-module snapd nano git
+sudo apt-get install -y wait-for-it dnsutils rsync sshfs apt-transport-https docker-ce-cli libcanberra-gtk-module nano git gridsite-clients
 
-sleep 1
+sudo bash -c "$HOME/sovereign-stack/install_incus.sh"
 
-#apt install python3-pip python3-dev libusb-1.0-0-dev libudev-dev pinentry-curses  for trezor stuff
-# for trezor installation
-#pip3 install setuptools wheel
-#pip3 install trezor_agent
+sudo incus admin init --minimal
 
-# ensure the trezor-t udev rules are in place.
-# if [ ! -f /etc/udev/rules.d/51-trezor.rules ]; then
-#     sudo cp ./51-trezor.rules /etc/udev/rules.d/51-trezor.rules
-# fi
-
-# install snap
-if ! snap list | grep -q lxd; then
-    sudo snap install lxd --channel=5.11/stable
-    sleep 6
-
-    # We just do an auto initialization. All we are using is the LXD client inside the management environment.
-    sudo lxd init --auto
+# add docker group
+if ! grep -q "^docker:" /etc/group; then
+    sudo groupadd docker
 fi
 
-# run a lxd command so we don't we a warning upon first invocation
-lxc list > /dev/null 2>&1
+# add incus-admin group
+if ! grep -q "^incus-admin:" /etc/group; then
+    sudo groupadd incus-admin
+fi
 
-
-# add groups for docker and lxd
-if ! groups ubuntu | grep -q docker; then
-    sudo addgroup docker
+if ! groups ubuntu | grep -q "\bdocker\b"; then
     sudo usermod -aG docker ubuntu
-    sudo usermod -aG lxd ubuntu
 fi
 
-
-# if an SSH pubkey does not exist, we create one.
-if [ ! -f /home/ubuntu/.ssh/id_rsa.pub ]; then
-    # generate a new SSH key for the base vm image.
-    ssh-keygen -f /home/ubuntu/.ssh/id_rsa -t ecdsa -b 521 -N ""
+if ! groups ubuntu | grep -q "\bincus-admin\b"; then
+    sudo usermod -aG incus-admin ubuntu
 fi
-
-echo "Your management machine has been provisioned!"

management/wait_for_ip.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+IP_V4_ADDRESS=
+while true; do
+    # wait for 
+    if incus list ss-mgmt | grep -q enp5s0; then
+        break;
+    else
+        sleep 1
+    fi
+done
+
+while true; do
+    IP_V4_ADDRESS=$(incus list ss-mgmt --format csv --columns=4 | grep enp5s0 | grep -Eo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')
+    if [ -n "$IP_V4_ADDRESS" ]; then
+        # give the machine extra time to spin up.
+        break;
+    else
+        sleep 1
+        printf '.'
+    fi
+done
+
+
+export IP_V4_ADDRESS="$IP_V4_ADDRESS"
+
+# wait for the VM to complete its default cloud-init.
+while incus exec ss-mgmt -- [ ! -f /var/lib/cloud/instance/boot-finished ]; do
+    sleep 1
+done

management/wait_for_lxc_ip.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-set -e
-
-IP_V4_ADDRESS=
-while true; do
-    # wait for 
-    if lxc list ss-mgmt | grep -q enp5s0; then
-        break;
-    else
-        sleep 1
-    fi
-done
-
-while true; do
-    IP_V4_ADDRESS=$(lxc list ss-mgmt --format csv --columns=4 | grep enp5s0 | grep -Eo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')
-    if [ -n "$IP_V4_ADDRESS" ]; then
-        # give the machine extra time to spin up.
-        break;
-    else
-        sleep 1
-        printf '.'
-    fi
-done
-
-
-export IP_V4_ADDRESS="$IP_V4_ADDRESS"

publish.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -eu
-cd "$(dirname "$0")"
-
-# this script will tag the repo then push it to origin
-TAG_NAME="$(head -n 1 ./version.txt)"
-TAG_MESSAGE="Creating tag $TAG_NAME on $(date)."
-
-# create the git tag.
-if ! git tag | grep -q "$TAG_NAME"; then
-    git tag -a "$TAG_NAME" -m "$TAG_MESSAGE" -s
-fi
-
-# push commits and tags to origin
-git push --set-upstream origin --all
-git push --set-upstream origin --tags
-
-## note this will only work if you have permissions to update HEAD on https://git.sovereign-stack.org/ss/sovereign-stack.git
-RESPONSE=
-read -r -p "         Would you like to push this to the main Sovereign Stack repo? (y)  ": RESPONSE
-if [ "$RESPONSE" = "y" ]; then
-    # optional; push to remote
-    git push --set-upstream ss-upstream --all
-    git push --set-upstream ss-upstream --tags
-fi

uninstall.sh

@@ -1,54 +1,93 @@
 #!/bin/bash
 
-set -eu
+set -exu
+
+# this script uninstalls incus from the MANAGEMENT MACHINE
+# if you want to remove incus from remote cluster hosts, run ss-reset.
+
+PURGE_INCUS=false
+
+# grab any modifications from the command line.
+for i in "$@"; do
+    case $i in
+        --purge)
+            PURGE_INCUS=true
+            shift
+        ;;
+        *)
+        echo "Unexpected option: $1"
+        exit 1
+        ;;
+    esac
+done
 
 # this script undoes install.sh
-if ! command -v lxc >/dev/null 2>&1; then
-    echo "This script requires 'lxc' to be installed. Please run 'install.sh'."
+if ! command -v incus >/dev/null 2>&1; then
+    echo "This script requires incus to be installed. Please run 'install.sh'."
     exit 1
 fi
 
-. ./defaults.sh
+if ! incus remote get-default | grep -q "local"; then
+    echo "ERROR: You MUST be on the local remote when uninstalling the SSME."
+    echo "INFO: You can use 'incus remote switch local' to do this."
+    exit 1
+fi
 
-if lxc list --format csv | grep -q ss-mgmt; then
 
-    if ! lxc list --format csv | grep ss-mgmt | grep -q "RUNNING"; then
-        lxc stop ss-mgmt
+if ! incus project list | grep -q "default (current)"; then
+    echo "ERROR: You MUST be on the default project when uninstalling the SSME."
+    echo "INFO: You can use 'incus project switch default' to do this."
+    exit 1
+fi
+
+
+if incus list --format csv | grep -q "ss-mgmt"; then
+
+    if incus list --format csv -q | grep -q "ss-mgmt,RUNNING"; then
+        incus stop ss-mgmt
     fi
 
-    lxc config device remove ss-mgmt sscode
-    lxc delete ss-mgmt -f
+    if incus config device list ss-mgmt -q | grep -q "ss-code"; then
+        incus config device remove ss-mgmt ss-code
+    fi
+
+    if incus config device list ss-mgmt -q | grep -q "ss-root"; then
+        incus config device remove ss-mgmt ss-root
+    fi
+
+    if incus config device list ss-mgmt -q | grep -q "ss-ssh"; then
+        incus config device remove ss-mgmt ss-ssh
+    fi
+
+    incus delete ss-mgmt
 fi
 
-if lxc profile device list default | grep -q root; then
-    lxc profile device remove default root
-fi
+if [ "$PURGE_INCUS" = true ]; then
 
-if lxc profile device list default | grep -q enp5s0; then
-    lxc profile device remove default enp5s0
-fi
+    if incus profile device list default | grep -q root; then
+        incus profile device remove default root
+    fi
 
-if lxc network list | grep -q lxdbr0; then
-    lxc network delete lxdbr0
-fi
+    if incus profile device list default | grep -q enp5s0; then
+        incus profile device remove default enp5s0
+    fi
 
-if lxc image list | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then
-    lxc image delete "$UBUNTU_BASE_IMAGE_NAME"
-fi
+    if incus network list --project default | grep -q incusbr0; then
+        incus network delete incusbr0
+    fi
 
-if lxc storage list --format csv | grep -q sovereign-stack; then
-    lxc storage delete sovereign-stack
-fi
+    # this file contains the BASE_IMAGE_NAME
+    . ./deployment/base.sh
+    if incus image list | grep -q "$UBUNTU_BASE_IMAGE_NAME"; then
+        incus image delete "$UBUNTU_BASE_IMAGE_NAME"
+    fi
 
-if snap list | grep -q lxd; then
-    sudo snap remove lxd
-    sleep 2
-fi
+    if incus storage list --format csv | grep -q sovereign-stack; then
+        incus storage delete sovereign-stack
+    fi
 
-if zfs list | grep -q sovereign-stack; then
-    sudo zfs destroy -r sovereign-stack
-fi
+    if dpkg -l | grep -q incus; then
+        sudo apt purge incus -y
+    fi
 
-if zfs list | grep -q "sovereign-stack"; then
-    sudo zfs destroy -r "rpool/lxd"
-fi
+fi

version.txt

@@ -1 +0,0 @@
-v0.0.24