#!/bin/bash
#
# Uptime check for every install on this server.
#
# `ping_sites` records its samples in the database of whichever install runs
# it. So a tenant panel can only show Service Health if that tenant's own
# install has been running the job.
#
# It was only ever crontabbed on one install. Every other panel therefore had a
# Service Health card with nothing behind it, and no error to explain why: the
# view returns sh_sample_count = 0 and the template quietly hides the chart.
# The card looked broken and was in fact never fed.
#
# Same shape as ft-backup-all.sh: find the installs rather than being told
# about them, so a tenant added next month is monitored without anyone
# remembering to edit a crontab.
#
# Install:
#   chmod 700 /root/ft-ping-all.sh
#   */5 * * * * /root/ft-ping-all.sh >> /var/log/ft-ping.log 2>&1
#
# Dry run, lists what it would run and touches nothing:
#   APPLY=0 /root/ft-ping-all.sh
#
# Each install pings only the sites in its own Tenant table, which for a tenant
# is itself. The fleet master's install has every site in it, so it keeps doing
# the fleet-wide check exactly as before. Nothing is checked twice by accident:
# the samples land in different databases.
#
set -uo pipefail

APPLY="${APPLY:-1}"
SETTINGS="${SETTINGS:-config.settings.production}"

# Sites to leave alone.
SKIP="paneltest"

found=0; ran=0; failed=0

for manage in /home/*/public_html/backend/manage.py; do
    [ -f "$manage" ] || continue
    backend="$(dirname "$manage")"
    user="$(echo "$backend" | cut -d/ -f3)"
    found=$((found + 1))

    case " $SKIP " in *" $user "*) continue ;; esac
    [ -x "$backend/venv/bin/python" ] || { echo "  no venv  $user"; continue; }

    if [ "$APPLY" != "1" ]; then
        echo "  would ping $user"
        continue
    fi

    # Run as the site user. As root it would create __pycache__ and any log
    # file owned by root inside the install, and the next request from
    # gunicorn cannot read them. That is how paneltest ended up restarting
    # 59,269 times over a single root-owned log file.
    if sudo -u "$user" env HOME="/home/$user" \
         "$backend/venv/bin/python" "$backend/manage.py" ping_sites \
         --settings="$SETTINGS" >/dev/null 2>&1; then
        ran=$((ran + 1))
    else
        echo "$(date '+%F %T') ping_sites failed for $user"
        failed=$((failed + 1))
    fi
done

# Only speak up when something is wrong. This runs every five minutes, and a
# job that writes 288 lines a day into a log teaches you to ignore that log.
if [ "$failed" -gt 0 ] || [ "$ran" -eq 0 ]; then
    echo "$(date '+%F %T') installs $found, pinged $ran, failed $failed"
    exit 1
fi
exit 0