1
1

Removed all AWS functionality. Big day.

This commit is contained in:
Derek Smith 2022-10-30 22:24:25 -04:00
parent e5164ebc7e
commit 08b158da09
Signed by: farscapian
GPG Key ID: 8F1CD799CCA516CC
11 changed files with 138 additions and 419 deletions

View File

@ -21,42 +21,10 @@ export NOSTR_HOSTNAME="relay"
export SITE_LANGUAGE_CODES="en"
export NOSTR_ACCOUNT_PUBKEY=
# used by 'aws' deployments only; planned deprecation
export DDNS_PASSWORD=
# this is where the html is sourced from.
export SITE_HTML_PATH=
export BTCPAY_ADDITIONAL_HOSTNAMES=
# enter your AWS Access Key and Secret Access Key here.
export AWS_ACCESS_KEY=
export AWS_SECRET_ACCESS_KEY=
# if overridden, the app will be deployed to proxy $BTCPAY_HOSTNAME.$DOMAIN_NAME requests to the URL specified.
# this is useful when you want to oursource your BTCPAY fullnode/lightning node.
#export BTCPAY_HANDLER_URL=
export SMTP_SERVER="smtp.mailgun.org"
export SMTP_PORT="587"
# default AWS region and AMI (free-tier AMI ubuntu 20.10)
export AWS_REGION="us-east-1"
# AMI NAME:
# ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20220420
export AWS_AMI_ID="ami-09d56f8956ab235b3"
WWW_INSTANCE_TYPE="t2.small"
BTCPAY_INSTANCE_TYPE="t2.medium"
# goal will be to keep any particular instance to run AT OR BELOW t2.medium.
# other options are t2.small, micro, nano; micro is the free-tier eligible.
# [1=vCPUs, 1=Mem(GiB)]
# nano [1,0.5], micro [1,1] (free-tier eligible), small [1,2], medium [2,4], large [2,8], xlarge [4,16], 2xlarge [8,32]
export WWW_INSTANCE_TYPE="$WWW_INSTANCE_TYPE"
export BTCPAY_INSTANCE_TYPE="$BTCPAY_INSTANCE_TYPE"
export GHOST_MYSQL_PASSWORD=
export GHOST_MYSQL_ROOT_PASSWORD=
export NEXTCLOUD_MYSQL_PASSWORD=
@ -90,15 +58,6 @@ export NEXTCLOUD_SPACE_GB=10
# exit 1
# fi
# TODO
# 1 add check for ~/.aws/credentials and stub one out
# 2 ensure install.sh has been run by checking for tor, docker-machine, lxd, wait-for-it, etc.
# 3 pretty much just run the install script if anything is awry
# 4 maybe check to ensure all the CNAME and A+ records are there first so we can quit before machine creation.
BTC_CHAIN=regtest
export BTC_CHAIN="$BTC_CHAIN"

View File

@ -13,9 +13,7 @@ check_dependencies () {
}
# Check system's dependencies
check_dependencies wait-for-it dig rsync sshfs lxc docker-machine
# TODO remove dependency on Docker-machine. That's what we use to provision VM on 3rd party vendors. Looking for LXD endpoint.
check_dependencies wait-for-it dig rsync sshfs lxc
# let's check to ensure the management machine is on the Baseline ubuntu 21.04
if ! lsb_release -d | grep -q "Ubuntu 22.04"; then
@ -24,7 +22,6 @@ if ! lsb_release -d | grep -q "Ubuntu 22.04"; then
fi
DOMAIN_NAME=
VPS_HOSTING_TARGET=lxd
RUN_CERT_RENEWAL=true
SKIP_WWW=false
RESTORE_WWW=false
@ -45,10 +42,6 @@ STOP_SERVICES=false
# grab any modifications from the command line.
for i in "$@"; do
case $i in
--aws)
VPS_HOSTING_TARGET=aws
shift
;;
--restore-www)
RESTORE_WWW=true
BACKUP_APPS=false
@ -146,17 +139,6 @@ export BACKUP_BTCPAY="$BACKUP_BTCPAY"
export MIGRATE_WWW="$MIGRATE_WWW"
export MIGRATE_BTCPAY="$MIGRATE_BTCPAY"
export RUN_CERT_RENEWAL="$RUN_CERT_RENEWAL"
if [ "$VPS_HOSTING_TARGET" = aws ]; then
if [ -z "$DOMAIN_NAME" ]; then
echo "ERROR: Please specify a domain name with --domain= when using --aws."
exit 1
fi
CLUSTER_NAME="docker-machine"
fi
export CLUSTER_NAME="$CLUSTER_NAME"
export CLUSTER_PATH="$CLUSTERS_DIR/$CLUSTER_NAME"
@ -173,7 +155,7 @@ if [ ! -f "$CLUSTER_PATH/authorized_keys" ]; then
exit 1
fi
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
CLUSTER_DEFINITION="$CLUSTER_PATH/cluster_definition"
export CLUSTER_DEFINITION="$CLUSTER_DEFINITION"
@ -221,7 +203,7 @@ EOL
fi
fi
fi
fi
# this is our password generation mechanism. Relying on GPG for secure password generation
function new_pass {
@ -231,7 +213,6 @@ function new_pass {
function instantiate_vms {
export VPS_HOSTING_TARGET="$VPS_HOSTING_TARGET"
export BTC_CHAIN="$BTC_CHAIN"
export UPDATE_BTCPAY="$UPDATE_BTCPAY"
export RECONFIGURE_BTCPAY_SERVER="$RECONFIGURE_BTCPAY_SERVER"
@ -254,7 +235,7 @@ function instantiate_vms {
exit 1
fi
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
# first let's get the DISK_TO_USE and DATA_PLANE_MACVLAN_INTERFACE from the ss-config
# which is set up during LXD cluster creation ss-cluster.
LXD_SS_CONFIG_LINE=
@ -276,7 +257,7 @@ function instantiate_vms {
./deployment/create_lxc_base.sh
fi
export MAC_ADDRESS_TO_PROVISION=
export VPS_HOSTNAME="$VPS_HOSTNAME"
@ -334,21 +315,8 @@ function instantiate_vms {
export BTCPAY_LOCAL_BACKUP_PATH="$SITE_PATH/backups/btcpayserver/$BACKUP_TIMESTAMP"
export BTCPAY_LOCAL_BACKUP_ARCHIVE_PATH="$BTCPAY_LOCAL_BACKUP_PATH/$UNIX_BACKUP_TIMESTAMP.tar.gz"
# This next section of if statements is our sanity checking area.
if [ "$VPS_HOSTING_TARGET" = aws ]; then
# we require DDNS on AWS to set the public DNS to the right host.
if [ -z "$DDNS_PASSWORD" ]; then
echo "ERROR: Ensure DDNS_PASSWORD is configured in your site_definition."
exit 1
fi
fi
MACHINE_EXISTS=false
if [ "$VPS_HOSTING_TARGET" = aws ] && docker-machine ls -q | grep -q "$FQDN"; then
MACHINE_EXISTS=true
fi
if [ "$VPS_HOSTING_TARGET" = lxd ] && lxc list --format csv | grep -q "$FQDN"; then
if lxc list --format csv | grep -q "$FQDN"; then
MACHINE_EXISTS=true
fi
@ -364,18 +332,7 @@ function instantiate_vms {
./deployment/deploy_vms.sh
# delete the remote VPS.
if [ "$VPS_HOSTING_TARGET" = aws ]; then
RESPONSE=
read -r -p "Do you want to continue with deleting '$FQDN' (y/n)": RESPONSE
if [ "$RESPONSE" = y ]; then
docker-machine rm -f "$FQDN"
else
echo "STOPPING the migration. User entered something other than 'y'."
exit 1
fi
elif [ "$VPS_HOSTING_TARGET" = lxd ]; then
lxc delete --force "$LXD_VM_NAME"
fi
# Then we run the script again to re-instantiate a new VPS, restoring all user data
# if restore directory doesn't exist, then we end up with a new site.
@ -493,8 +450,6 @@ EOL
source "$PROJECT_DEFINITION_PATH"
}
# let's iterate over the user-supplied domain list and provision each domain.
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
CURRENT_PROJECT="$(lxc info | grep "project:" | awk '{print $2}')"
PROJECT_PATH="$PROJECTS_DIR/$PROJECT_NAME"
@ -557,10 +512,3 @@ if [ "$VPS_HOSTING_TARGET" = lxd ]; then
if [ "$SKIP_BTCPAY" = false ]; then
bash -c "./deployment/btcpayserver/go.sh"
fi
elif [ "$VPS_HOSTING_TARGET" = aws ]; then
stub_site_definition
# if we're on AWS, we can just provision each system separately.
run_domain
fi

View File

@ -59,14 +59,7 @@ if [ "$RUN_SERVICES" = true ]; then
fi
if [ "$OPEN_URL" = true ]; then
if [ "$VPS_HOSTING_TARGET" = lxd ]; then
if wait-for-it -t 5 "$PRIMARY_WWW_FQDN:80"; then
xdg-open "http://$PRIMARY_WWW_FQDN" > /dev/null 2>&1
fi
else
if wait-for-it -t 5 "$FQDN:443"; then
xdg-open "https://$FQDN" > /dev/null 2>&1
fi
fi
fi

View File

@ -43,16 +43,6 @@ function prepare_host {
}
# when set to true, this flag indicates that a new VPS was created during THIS script run.
if [ "$VPS_HOSTING_TARGET" = aws ]; then
# let's create the remote VPS if needed.
if ! docker-machine ls -q --filter name="$FQDN" | grep -q "$FQDN"; then
./provision_vps.sh
prepare_host
fi
elif [ "$VPS_HOSTING_TARGET" = lxd ]; then
ssh-keygen -f "$SSH_HOME/known_hosts" -R "$FQDN"
# if the machine doesn't exist, we create it.
@ -69,5 +59,3 @@ elif [ "$VPS_HOSTING_TARGET" = lxd ]; then
fi
prepare_host
fi

View File

@ -5,39 +5,6 @@ cd "$(dirname "$0")"
./stub_lxc_profile.sh "$LXD_VM_NAME"
function run_ddns {
# now that the VM has an IP, we can update the DNS record. TODO add additional DNS providers here; namecheap only atm.
DDNS_STRING="$VPS_HOSTNAME"
if [ "$VPS_HOSTNAME" = www ]; then
# next update our DDNS record. TODO enable local/remote name provider.
DDNS_STRING="@"
fi
# if the DNS record is incorrect, we run DDNS to get it corrected yo.
if "$(getent hosts "$FQDN" | awk '{ print $1 }')" != "$IP_V4_ADDRESS"; then
curl "https://dynamicdns.park-your-domain.com/update?host=$DDNS_STRING&domain=$DOMAIN_NAME&password=$DDNS_PASSWORD&ip=$IP_V4_ADDRESS"
DDNS_SLEEP_SECONDS=60
while true; do
# we test the www CNAME here so we can be assured the underlying has corrected.
if [[ "$(getent hosts "$FQDN" | awk '{ print $1 }')" == "$IP_V4_ADDRESS" ]]; then
echo ""
echo "SUCCESS: The DNS appears to be configured correctly."
# we're waiting here to allow dns records to stale out.
# this is necessary for certificate renewal; letsencrypt might have stale records
echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow cached DNS records to expire."
sleep "$DDNS_SLEEP_SECONDS";
break;
fi
printf "." && sleep 2;
done
fi
}
# now let's create a new VM to work with.
lxc init --profile="$LXD_VM_NAME" "$VM_NAME" "$LXD_VM_NAME" --vm
@ -49,11 +16,3 @@ lxc config device override "$LXD_VM_NAME" root size="${ROOT_DISK_SIZE_GB}GB"
lxc start "$LXD_VM_NAME"
./wait_for_lxc_ip.sh "$LXD_VM_NAME"
if [ "$VPS_HOSTING_TARGET" = aws ]; then
run_ddns
# remove any existing SSH identities for the host, then add it back.
ssh-keygen -R "$IP_V4_ADDRESS"
fi

View File

@ -1,84 +0,0 @@
#!/bin/bash
set -eu
cd "$(dirname "$0")"
if [ ! -f "$HOME/.aws/credentials" ]; then
# TODO write a credential file baseline
echo "ERROR: Please update your '$HOME/.aws/credentials' file before continuing."
mkdir -p "$HOME/.aws"
touch "$HOME/.aws/credentials"
# stub out a site_definition with new passwords.
cat >"$HOME/.aws/credentials" <<EOL
#!/bin/bash
# enter your AWS Access Key and Secret Access Key here.
export AWS_ACCESS_KEY=
export AWS_SECRET_ACCESS_KEY=
EOL
exit 1
fi
source "$HOME/.aws/credentials"
if [ -z "$AWS_ACCESS_KEY" ]; then
echo "ERROR: AWS_ACCESS_KEY is not set."
exit 1
fi
if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
echo "ERROR: AWS_SECRET_ACCESS_KEY is not set."
exit 1
fi
# Note, we assume the script has already made sure the machine doesn't exist.
if [ "$VIRTUAL_MACHINE" = www ] || [ "$VIRTUAL_MACHINE" = certonly ]; then
# creates a public VM in AWS and provisions the bcm website.
docker-machine create --driver amazonec2 \
--amazonec2-open-port 80 \
--amazonec2-open-port 443 \
--amazonec2-access-key "$AWS_ACCESS_KEY" \
--amazonec2-secret-key "$AWS_SECRET_ACCESS_KEY" \
--amazonec2-region "$AWS_REGION" \
--amazonec2-ami "$AWS_AMI_ID" \
--amazonec2-root-size "$ROOT_DISK_SIZE_GB" \
--amazonec2-instance-type "$WWW_INSTANCE_TYPE" \
"$FQDN"
# --engine-label commit="$LATEST_GIT_COMMIT" \
elif [ "$VIRTUAL_MACHINE" = btcpayserver ]; then
# creates a public VM in AWS and provisions the bcm website.
docker-machine create --driver amazonec2 \
--amazonec2-open-port 80 \
--amazonec2-open-port 443 \
--amazonec2-open-port 9735 \
--amazonec2-access-key "$AWS_ACCESS_KEY" \
--amazonec2-secret-key "$AWS_SECRET_ACCESS_KEY" \
--amazonec2-region "$AWS_REGION" \
--amazonec2-ami "$AWS_AMI_ID" \
--amazonec2-root-size "$ROOT_DISK_SIZE_GB" \
--amazonec2-instance-type "$BTCPAY_INSTANCE_TYPE" \
"$FQDN"
# --engine-label commit="$LATEST_GIT_COMMIT" \
fi
docker-machine scp "$CLUSTER_PATH/authorized_keys" "$FQDN:$REMOTE_HOME/authorized_keys"
docker-machine ssh "$FQDN" "cat $REMOTE_HOME/authorized_keys >> $REMOTE_HOME/.ssh/authorized_keys"
# we have to ensure ubuntu is able to do sudo less docker commands.
docker-machine ssh "$FQDN" sudo usermod -aG docker ubuntu
# we restart so dockerd starts with fresh group membership.
docker-machine ssh "$FQDN" sudo systemctl restart docker
# TODO INSTALL DOCKER COMPOSE
# let's wire up the DNS so subsequent ssh commands resolve to the VPS.
./run_ddns.sh
# remove the SSH hostname from known_hosts as we'll
# todo why do we need this again?
ssh-keygen -f "$SSH_HOME/known_hosts" -R "$FQDN"

View File

@ -13,15 +13,6 @@ fi
# wait for DNS to get setup. Pass in the IP address of the actual VPS.
MACHINE_IP="$(docker-machine ip "$FQDN")"
if [ "$VPS_HOSTING_TARGET" = aws ]; then
# wire DNS entries using namecheap DDNS API (via HTTPS rather than ddclient)
curl "https://dynamicdns.park-your-domain.com/update?host=$DDNS_STRING&domain=$DOMAIN_NAME&password=$DDNS_PASSWORD&ip=$MACHINE_IP"
#install dependencies.
docker-machine ssh "$FQDN" sudo apt-get -qq install -y wait-for-it git rsync duplicity sshfs
fi
DDNS_SLEEP_SECONDS=60
while true; do
# we test the www CNAME here so we can be assured the underlying has corrected.

View File

@ -6,31 +6,7 @@ set -e
# let's do a refresh of the certificates. Let's Encrypt will not run if it's not time.
docker pull certbot/certbot:latest
# when deploying to AWS, www exists on a separate IP address from btcpay, etc.
# thus, we structure the certificate accordingly.
if [ "$VPS_HOSTING_TARGET" = aws ]; then
docker run -it --rm \
--name certbot \
-p 80:80 \
-p 443:443 \
-v "$REMOTE_HOME/letsencrypt":/etc/letsencrypt \
-v /var/lib/letsencrypt:/var/lib/letsencrypt \
-v "$REMOTE_HOME/letsencrypt_logs":/var/log/letsencrypt \
certbot/certbot certonly -v --noninteractive --agree-tos --key-type ecdsa --standalone --expand -d "$DOMAIN_NAME" -d "$FQDN" -d "$NEXTCLOUD_FQDN" -d "$GITEA_FQDN" --email "$CERTIFICATE_EMAIL_ADDRESS"
elif [ "$VPS_HOSTING_TARGET" = lxd ]; then
# with the lxd side, we are trying to expose ALL OUR services from one IP address, which terminates
# at a cachehing reverse proxy that runs nginx.
# docker run -it --rm \
# --name certbot \
# -p 80:80 \
# -p 443:443 \
# -v "$REMOTE_HOME/letsencrypt":/etc/letsencrypt \
# -v /var/lib/letsencrypt:/var/lib/letsencrypt \
# -v "$REMOTE_HOME/letsencrypt_logs":/var/log/letsencrypt \
# certbot/certbot certonly -v --noninteractive --agree-tos --key-type ecdsa --standalone --expand -d "$DOMAIN_NAME" -d "$PRIMARY_WWW_FQDN" -d "$BTCPAY_USER_FQDN" -d "$NEXTCLOUD_FQDN" -d "$GITEA_FQDN" -d "$NOSTR_FQDN" --email "$CERTIFICATE_EMAIL_ADDRESS"
# iterate over each domain and call certbot
for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do
export DOMAIN_NAME="$DOMAIN_NAME"
export SITE_PATH="$SITES_PATH/$DOMAIN_NAME"
@ -55,4 +31,3 @@ elif [ "$VPS_HOSTING_TARGET" = lxd ]; then
certbot/certbot certonly -v --noninteractive --agree-tos --key-type ecdsa --standalone --expand -d "$DOMAIN_NAME" -d "$WWW_FQDN" -d "$BTCPAY_USER_FQDN" -d "$NEXTCLOUD_FQDN" -d "$GITEA_FQDN" -d "$NOSTR_FQDN" --email "$CERTIFICATE_EMAIL_ADDRESS"
done
fi

View File

@ -59,7 +59,7 @@ if docker stack list --format "{{.Name}}" | grep -q reverse-proxy; then
# wait for all docker containers to stop.
# TODO see if there's a way to check for this.
sleep 10
sleep 15
fi
# generate the certs and grab a backup

View File

@ -2,7 +2,6 @@
set -e
# TODO, ensure VPS_HOSTING_TARGET is in range.
export NEXTCLOUD_FQDN="$NEXTCLOUD_HOSTNAME.$DOMAIN_NAME"
export BTCPAY_FQDN="$BTCPAY_HOSTNAME.$DOMAIN_NAME"
export BTCPAY_USER_FQDN="$BTCPAY_HOSTNAME_IN_CERT.$DOMAIN_NAME"
@ -14,7 +13,6 @@ export CERTIFICATE_EMAIL_ADDRESS="$ADMIN_ACCOUNT_USERNAME@$DOMAIN_NAME"
export REMOTE_NEXTCLOUD_PATH="$REMOTE_HOME/nextcloud"
export REMOTE_GITEA_PATH="$REMOTE_HOME/gitea"
export BTC_CHAIN="$BTC_CHAIN"
export WWW_INSTANCE_TYPE="$WWW_INSTANCE_TYPE"
export BTCPAY_ADDITIONAL_HOSTNAMES="$BTCPAY_ADDITIONAL_HOSTNAMES"
BACKUP_TIMESTAMP="$(date +"%Y-%m")"

View File

@ -36,14 +36,6 @@ if ! snap list | grep -q lxd; then
sudo snap install lxd --candidate
fi
# TODO WORK ON GETTING RID OF THIS DEPENDENCY
if [ ! -f /usr/local/bin/docker-machine ]; then
# let's ensure docker-machine is available. This is only temporary though.
curl -L "https://github.com/docker/machine/releases/download/v0.16.2/docker-machine-$(uname -s)-$(uname -m)" >/tmp/docker-machine &&
chmod +x /tmp/docker-machine &&
sudo cp /tmp/docker-machine /usr/local/bin/docker-machine
fi
# make ss-deploy available to the user
if ! groups | grep -q docker; then
sudo groupadd docker