diff --git a/defaults.sh b/defaults.sh index 94c77dd..cb59daa 100755 --- a/defaults.sh +++ b/defaults.sh @@ -21,42 +21,10 @@ export NOSTR_HOSTNAME="relay" export SITE_LANGUAGE_CODES="en" export NOSTR_ACCOUNT_PUBKEY= -# used by 'aws' deployments only; planned deprecation -export DDNS_PASSWORD= - # this is where the html is sourced from. export SITE_HTML_PATH= export BTCPAY_ADDITIONAL_HOSTNAMES= -# enter your AWS Access Key and Secret Access Key here. -export AWS_ACCESS_KEY= -export AWS_SECRET_ACCESS_KEY= - -# if overridden, the app will be deployed to proxy $BTCPAY_HOSTNAME.$DOMAIN_NAME requests to the URL specified. -# this is useful when you want to oursource your BTCPAY fullnode/lightning node. -#export BTCPAY_HANDLER_URL= - - -export SMTP_SERVER="smtp.mailgun.org" -export SMTP_PORT="587" - -# default AWS region and AMI (free-tier AMI ubuntu 20.10) -export AWS_REGION="us-east-1" - -# AMI NAME: -# ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-20220420 -export AWS_AMI_ID="ami-09d56f8956ab235b3" -WWW_INSTANCE_TYPE="t2.small" -BTCPAY_INSTANCE_TYPE="t2.medium" - -# goal will be to keep any particular instance to run AT OR BELOW t2.medium. -# other options are t2.small, micro, nano; micro is the free-tier eligible. -# [1=vCPUs, 1=Mem(GiB)] -# nano [1,0.5], micro [1,1] (free-tier eligible), small [1,2], medium [2,4], large [2,8], xlarge [4,16], 2xlarge [8,32] - -export WWW_INSTANCE_TYPE="$WWW_INSTANCE_TYPE" -export BTCPAY_INSTANCE_TYPE="$BTCPAY_INSTANCE_TYPE" - export GHOST_MYSQL_PASSWORD= export GHOST_MYSQL_ROOT_PASSWORD= export NEXTCLOUD_MYSQL_PASSWORD= @@ -90,15 +58,6 @@ export NEXTCLOUD_SPACE_GB=10 # exit 1 # fi - - - -# TODO -# 1 add check for ~/.aws/credentials and stub one out -# 2 ensure install.sh has been run by checking for tor, docker-machine, lxd, wait-for-it, etc. -# 3 pretty much just run the install script if anything is awry -# 4 maybe check to ensure all the CNAME and A+ records are there first so we can quit before machine creation. - BTC_CHAIN=regtest export BTC_CHAIN="$BTC_CHAIN" diff --git a/deploy.sh b/deploy.sh index 549e274..332e04c 100755 --- a/deploy.sh +++ b/deploy.sh @@ -13,9 +13,7 @@ check_dependencies () { } # Check system's dependencies -check_dependencies wait-for-it dig rsync sshfs lxc docker-machine - -# TODO remove dependency on Docker-machine. That's what we use to provision VM on 3rd party vendors. Looking for LXD endpoint. +check_dependencies wait-for-it dig rsync sshfs lxc # let's check to ensure the management machine is on the Baseline ubuntu 21.04 if ! lsb_release -d | grep -q "Ubuntu 22.04"; then @@ -24,7 +22,6 @@ if ! lsb_release -d | grep -q "Ubuntu 22.04"; then fi DOMAIN_NAME= -VPS_HOSTING_TARGET=lxd RUN_CERT_RENEWAL=true SKIP_WWW=false RESTORE_WWW=false @@ -45,10 +42,6 @@ STOP_SERVICES=false # grab any modifications from the command line. for i in "$@"; do case $i in - --aws) - VPS_HOSTING_TARGET=aws - shift - ;; --restore-www) RESTORE_WWW=true BACKUP_APPS=false @@ -146,17 +139,6 @@ export BACKUP_BTCPAY="$BACKUP_BTCPAY" export MIGRATE_WWW="$MIGRATE_WWW" export MIGRATE_BTCPAY="$MIGRATE_BTCPAY" export RUN_CERT_RENEWAL="$RUN_CERT_RENEWAL" - -if [ "$VPS_HOSTING_TARGET" = aws ]; then - - if [ -z "$DOMAIN_NAME" ]; then - echo "ERROR: Please specify a domain name with --domain= when using --aws." - exit 1 - fi - - CLUSTER_NAME="docker-machine" -fi - export CLUSTER_NAME="$CLUSTER_NAME" export CLUSTER_PATH="$CLUSTERS_DIR/$CLUSTER_NAME" @@ -173,27 +155,27 @@ if [ ! -f "$CLUSTER_PATH/authorized_keys" ]; then exit 1 fi -if [ "$VPS_HOSTING_TARGET" = lxd ]; then - CLUSTER_DEFINITION="$CLUSTER_PATH/cluster_definition" - export CLUSTER_DEFINITION="$CLUSTER_DEFINITION" - ######################################### - if [ ! -f "$CLUSTER_DEFINITION" ]; then - echo "ERROR: The cluster defintion could not be found. You may need to re-run 'ss-cluster create'." - exit 1 - fi - - source "$CLUSTER_DEFINITION" +CLUSTER_DEFINITION="$CLUSTER_PATH/cluster_definition" +export CLUSTER_DEFINITION="$CLUSTER_DEFINITION" - ###########################3 - # # This section is done to the management machine. We deploy a registry pull through cache on port 5000 - # if ! docker volume list | grep -q registry_data; then - # docker volume create registry_data - # fi +######################################### +if [ ! -f "$CLUSTER_DEFINITION" ]; then + echo "ERROR: The cluster defintion could not be found. You may need to re-run 'ss-cluster create'." + exit 1 +fi + +source "$CLUSTER_DEFINITION" - # if the registry URL isn't defined, then we just use the upstream dockerhub. - # recommended to run a registry cache on your management machine though. - if [ -n "$REGISTRY_URL" ]; then +###########################3 +# # This section is done to the management machine. We deploy a registry pull through cache on port 5000 +# if ! docker volume list | grep -q registry_data; then +# docker volume create registry_data +# fi + +# if the registry URL isn't defined, then we just use the upstream dockerhub. +# recommended to run a registry cache on your management machine though. +if [ -n "$REGISTRY_URL" ]; then cat > "$CLUSTER_PATH/registry.yml" < /dev/null 2>&1 - fi - else - if wait-for-it -t 5 "$FQDN:443"; then - xdg-open "https://$FQDN" > /dev/null 2>&1 - fi + if wait-for-it -t 5 "$PRIMARY_WWW_FQDN:80"; then + xdg-open "http://$PRIMARY_WWW_FQDN" > /dev/null 2>&1 fi fi diff --git a/deployment/deploy_vms.sh b/deployment/deploy_vms.sh index 77d5230..f197c45 100755 --- a/deployment/deploy_vms.sh +++ b/deployment/deploy_vms.sh @@ -43,31 +43,19 @@ function prepare_host { } -# when set to true, this flag indicates that a new VPS was created during THIS script run. -if [ "$VPS_HOSTING_TARGET" = aws ]; then - # let's create the remote VPS if needed. - if ! docker-machine ls -q --filter name="$FQDN" | grep -q "$FQDN"; then +ssh-keygen -f "$SSH_HOME/known_hosts" -R "$FQDN" - ./provision_vps.sh +# if the machine doesn't exist, we create it. +if ! lxc list --format csv | grep -q "$LXD_VM_NAME"; then - prepare_host - fi -elif [ "$VPS_HOSTING_TARGET" = lxd ]; then - ssh-keygen -f "$SSH_HOME/known_hosts" -R "$FQDN" - - # if the machine doesn't exist, we create it. - if ! lxc list --format csv | grep -q "$LXD_VM_NAME"; then - - # create a base image if needed and instantiate a VM. - if [ -z "$MAC_ADDRESS_TO_PROVISION" ]; then - echo "ERROR: You MUST define a MAC Address for all your machines by setting WWW_SERVER_MAC_ADDRESS, BTCPAYSERVER_MAC_ADDRESS in your site defintion." - echo "INFO: IMPORTANT! You MUST have DHCP Reservations for these MAC addresses. You also need records established the DNS." - exit 1 - fi - - ./provision_lxc.sh + # create a base image if needed and instantiate a VM. + if [ -z "$MAC_ADDRESS_TO_PROVISION" ]; then + echo "ERROR: You MUST define a MAC Address for all your machines by setting WWW_SERVER_MAC_ADDRESS, BTCPAYSERVER_MAC_ADDRESS in your site defintion." + echo "INFO: IMPORTANT! You MUST have DHCP Reservations for these MAC addresses. You also need records established the DNS." + exit 1 fi - prepare_host - + ./provision_lxc.sh fi + +prepare_host diff --git a/deployment/provision_lxc.sh b/deployment/provision_lxc.sh index 43ebdcf..f641fff 100755 --- a/deployment/provision_lxc.sh +++ b/deployment/provision_lxc.sh @@ -5,39 +5,6 @@ cd "$(dirname "$0")" ./stub_lxc_profile.sh "$LXD_VM_NAME" -function run_ddns { - # now that the VM has an IP, we can update the DNS record. TODO add additional DNS providers here; namecheap only atm. - DDNS_STRING="$VPS_HOSTNAME" - if [ "$VPS_HOSTNAME" = www ]; then - # next update our DDNS record. TODO enable local/remote name provider. - DDNS_STRING="@" - fi - - # if the DNS record is incorrect, we run DDNS to get it corrected yo. - if "$(getent hosts "$FQDN" | awk '{ print $1 }')" != "$IP_V4_ADDRESS"; then - curl "https://dynamicdns.park-your-domain.com/update?host=$DDNS_STRING&domain=$DOMAIN_NAME&password=$DDNS_PASSWORD&ip=$IP_V4_ADDRESS" - - DDNS_SLEEP_SECONDS=60 - while true; do - # we test the www CNAME here so we can be assured the underlying has corrected. - if [[ "$(getent hosts "$FQDN" | awk '{ print $1 }')" == "$IP_V4_ADDRESS" ]]; then - echo "" - echo "SUCCESS: The DNS appears to be configured correctly." - - # we're waiting here to allow dns records to stale out. - # this is necessary for certificate renewal; letsencrypt might have stale records - echo "INFO: Waiting $DDNS_SLEEP_SECONDS seconds to allow cached DNS records to expire." - sleep "$DDNS_SLEEP_SECONDS"; - - break; - fi - - printf "." && sleep 2; - done - fi - -} - # now let's create a new VM to work with. lxc init --profile="$LXD_VM_NAME" "$VM_NAME" "$LXD_VM_NAME" --vm @@ -49,11 +16,3 @@ lxc config device override "$LXD_VM_NAME" root size="${ROOT_DISK_SIZE_GB}GB" lxc start "$LXD_VM_NAME" ./wait_for_lxc_ip.sh "$LXD_VM_NAME" - -if [ "$VPS_HOSTING_TARGET" = aws ]; then - run_ddns - - # remove any existing SSH identities for the host, then add it back. - ssh-keygen -R "$IP_V4_ADDRESS" - -fi diff --git a/deployment/provision_vps.sh b/deployment/provision_vps.sh deleted file mode 100755 index 077cd92..0000000 --- a/deployment/provision_vps.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -set -eu -cd "$(dirname "$0")" - -if [ ! -f "$HOME/.aws/credentials" ]; then - - # TODO write a credential file baseline - echo "ERROR: Please update your '$HOME/.aws/credentials' file before continuing." - mkdir -p "$HOME/.aws" - touch "$HOME/.aws/credentials" - - # stub out a site_definition with new passwords. - cat >"$HOME/.aws/credentials" <> $REMOTE_HOME/.ssh/authorized_keys" - -# we have to ensure ubuntu is able to do sudo less docker commands. -docker-machine ssh "$FQDN" sudo usermod -aG docker ubuntu - -# we restart so dockerd starts with fresh group membership. -docker-machine ssh "$FQDN" sudo systemctl restart docker - -# TODO INSTALL DOCKER COMPOSE - -# let's wire up the DNS so subsequent ssh commands resolve to the VPS. -./run_ddns.sh - -# remove the SSH hostname from known_hosts as we'll -# todo why do we need this again? -ssh-keygen -f "$SSH_HOME/known_hosts" -R "$FQDN" diff --git a/deployment/run_ddns.sh b/deployment/run_ddns.sh index bbcf226..5e825b3 100755 --- a/deployment/run_ddns.sh +++ b/deployment/run_ddns.sh @@ -13,15 +13,6 @@ fi # wait for DNS to get setup. Pass in the IP address of the actual VPS. MACHINE_IP="$(docker-machine ip "$FQDN")" -if [ "$VPS_HOSTING_TARGET" = aws ]; then - - # wire DNS entries using namecheap DDNS API (via HTTPS rather than ddclient) - curl "https://dynamicdns.park-your-domain.com/update?host=$DDNS_STRING&domain=$DOMAIN_NAME&password=$DDNS_PASSWORD&ip=$MACHINE_IP" - - #install dependencies. - docker-machine ssh "$FQDN" sudo apt-get -qq install -y wait-for-it git rsync duplicity sshfs -fi - DDNS_SLEEP_SECONDS=60 while true; do # we test the www CNAME here so we can be assured the underlying has corrected. diff --git a/deployment/www/generate_certs.sh b/deployment/www/generate_certs.sh index d75ec49..13b0f98 100755 --- a/deployment/www/generate_certs.sh +++ b/deployment/www/generate_certs.sh @@ -6,53 +6,28 @@ set -e # let's do a refresh of the certificates. Let's Encrypt will not run if it's not time. docker pull certbot/certbot:latest -# when deploying to AWS, www exists on a separate IP address from btcpay, etc. -# thus, we structure the certificate accordingly. -if [ "$VPS_HOSTING_TARGET" = aws ]; then +# iterate over each domain and call certbot +for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do + export DOMAIN_NAME="$DOMAIN_NAME" + export SITE_PATH="$SITES_PATH/$DOMAIN_NAME" + + # source the site path so we know what features it has. + source ../../reset_env.sh + source "$SITE_PATH/site_definition" + source ../../domain_env.sh + + # with the lxd side, we are trying to expose ALL OUR services from one IP address, which terminates + # at a cachehing reverse proxy that runs nginx. + + ssh "$PRIMARY_WWW_FQDN" sudo mkdir -p "$REMOTE_HOME/letsencrypt/$DOMAIN_NAME/_logs" + docker run -it --rm \ --name certbot \ -p 80:80 \ -p 443:443 \ - -v "$REMOTE_HOME/letsencrypt":/etc/letsencrypt \ + -v "$REMOTE_HOME/letsencrypt/$DOMAIN_NAME":/etc/letsencrypt \ -v /var/lib/letsencrypt:/var/lib/letsencrypt \ - -v "$REMOTE_HOME/letsencrypt_logs":/var/log/letsencrypt \ - certbot/certbot certonly -v --noninteractive --agree-tos --key-type ecdsa --standalone --expand -d "$DOMAIN_NAME" -d "$FQDN" -d "$NEXTCLOUD_FQDN" -d "$GITEA_FQDN" --email "$CERTIFICATE_EMAIL_ADDRESS" + -v "$REMOTE_HOME/letsencrypt/$DOMAIN_NAME/_logs":/var/log/letsencrypt \ + certbot/certbot certonly -v --noninteractive --agree-tos --key-type ecdsa --standalone --expand -d "$DOMAIN_NAME" -d "$WWW_FQDN" -d "$BTCPAY_USER_FQDN" -d "$NEXTCLOUD_FQDN" -d "$GITEA_FQDN" -d "$NOSTR_FQDN" --email "$CERTIFICATE_EMAIL_ADDRESS" -elif [ "$VPS_HOSTING_TARGET" = lxd ]; then - # with the lxd side, we are trying to expose ALL OUR services from one IP address, which terminates - # at a cachehing reverse proxy that runs nginx. - # docker run -it --rm \ - # --name certbot \ - # -p 80:80 \ - # -p 443:443 \ - # -v "$REMOTE_HOME/letsencrypt":/etc/letsencrypt \ - # -v /var/lib/letsencrypt:/var/lib/letsencrypt \ - # -v "$REMOTE_HOME/letsencrypt_logs":/var/log/letsencrypt \ - # certbot/certbot certonly -v --noninteractive --agree-tos --key-type ecdsa --standalone --expand -d "$DOMAIN_NAME" -d "$PRIMARY_WWW_FQDN" -d "$BTCPAY_USER_FQDN" -d "$NEXTCLOUD_FQDN" -d "$GITEA_FQDN" -d "$NOSTR_FQDN" --email "$CERTIFICATE_EMAIL_ADDRESS" - - - for DOMAIN_NAME in ${DOMAIN_LIST//,/ }; do - export DOMAIN_NAME="$DOMAIN_NAME" - export SITE_PATH="$SITES_PATH/$DOMAIN_NAME" - - # source the site path so we know what features it has. - source ../../reset_env.sh - source "$SITE_PATH/site_definition" - source ../../domain_env.sh - - # with the lxd side, we are trying to expose ALL OUR services from one IP address, which terminates - # at a cachehing reverse proxy that runs nginx. - - ssh "$PRIMARY_WWW_FQDN" sudo mkdir -p "$REMOTE_HOME/letsencrypt/$DOMAIN_NAME/_logs" - - docker run -it --rm \ - --name certbot \ - -p 80:80 \ - -p 443:443 \ - -v "$REMOTE_HOME/letsencrypt/$DOMAIN_NAME":/etc/letsencrypt \ - -v /var/lib/letsencrypt:/var/lib/letsencrypt \ - -v "$REMOTE_HOME/letsencrypt/$DOMAIN_NAME/_logs":/var/log/letsencrypt \ - certbot/certbot certonly -v --noninteractive --agree-tos --key-type ecdsa --standalone --expand -d "$DOMAIN_NAME" -d "$WWW_FQDN" -d "$BTCPAY_USER_FQDN" -d "$NEXTCLOUD_FQDN" -d "$GITEA_FQDN" -d "$NOSTR_FQDN" --email "$CERTIFICATE_EMAIL_ADDRESS" - - done -fi +done diff --git a/deployment/www/stop_docker_stacks.sh b/deployment/www/stop_docker_stacks.sh index 8dd6f61..2552b11 100755 --- a/deployment/www/stop_docker_stacks.sh +++ b/deployment/www/stop_docker_stacks.sh @@ -59,7 +59,7 @@ if docker stack list --format "{{.Name}}" | grep -q reverse-proxy; then # wait for all docker containers to stop. # TODO see if there's a way to check for this. - sleep 10 + sleep 15 fi # generate the certs and grab a backup diff --git a/domain_env.sh b/domain_env.sh index 73d0211..8405e38 100755 --- a/domain_env.sh +++ b/domain_env.sh @@ -2,7 +2,6 @@ set -e -# TODO, ensure VPS_HOSTING_TARGET is in range. export NEXTCLOUD_FQDN="$NEXTCLOUD_HOSTNAME.$DOMAIN_NAME" export BTCPAY_FQDN="$BTCPAY_HOSTNAME.$DOMAIN_NAME" export BTCPAY_USER_FQDN="$BTCPAY_HOSTNAME_IN_CERT.$DOMAIN_NAME" @@ -14,7 +13,6 @@ export CERTIFICATE_EMAIL_ADDRESS="$ADMIN_ACCOUNT_USERNAME@$DOMAIN_NAME" export REMOTE_NEXTCLOUD_PATH="$REMOTE_HOME/nextcloud" export REMOTE_GITEA_PATH="$REMOTE_HOME/gitea" export BTC_CHAIN="$BTC_CHAIN" -export WWW_INSTANCE_TYPE="$WWW_INSTANCE_TYPE" export BTCPAY_ADDITIONAL_HOSTNAMES="$BTCPAY_ADDITIONAL_HOSTNAMES" BACKUP_TIMESTAMP="$(date +"%Y-%m")" diff --git a/install.sh b/install.sh index 78f5668..6ddf44e 100755 --- a/install.sh +++ b/install.sh @@ -36,14 +36,6 @@ if ! snap list | grep -q lxd; then sudo snap install lxd --candidate fi -# TODO WORK ON GETTING RID OF THIS DEPENDENCY -if [ ! -f /usr/local/bin/docker-machine ]; then - # let's ensure docker-machine is available. This is only temporary though. - curl -L "https://github.com/docker/machine/releases/download/v0.16.2/docker-machine-$(uname -s)-$(uname -m)" >/tmp/docker-machine && - chmod +x /tmp/docker-machine && - sudo cp /tmp/docker-machine /usr/local/bin/docker-machine -fi - # make ss-deploy available to the user if ! groups | grep -q docker; then sudo groupadd docker