#!/usr/bin/env python3 """ Deploy runner. Built by deploy/build_ship.py, always arrives as ship.py. python3 ship.py show what would change, write nothing python3 ship.py --apply back up, write, restart, verify, roll back on failure python3 ship.py --rollback put the last deploy back What it refuses to do: run against the wrong site, write a Python file that does not compile, write a template whose tags do not balance, or leave a broken site up. """ import base64 import datetime import difflib import glob import hashlib import json import os import re import shutil import subprocess import sys SITE = __SITE__ FILES = __FILES__ SOURCE = __SOURCE__ BASE = SITE['base'] STAMP = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') BACKUPS = os.path.join(BASE, 'deploy_backups') MODE = sys.argv[1] if len(sys.argv) > 1 else '--check' def say(msg=''): print(msg, flush=True) def run(cmd): """Run a shell command and hand back the result. Written the long way on purpose. The server's system python is 3.6 and subprocess.run only grew capture_output in 3.7, so the short form dies with a TypeError after the files are already written, which leaves the service unrestarted and the deploy half done. """ proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate() class Result(object): pass r = Result() r.returncode = proc.returncode r.stdout = out.decode('utf-8', 'replace') r.stderr = err.decode('utf-8', 'replace') return r def newest_backup(): dirs = sorted(glob.glob(os.path.join(BACKUPS, '*'))) return dirs[-1] if dirs else None def restore(folder): n = 0 for root, _dirs, names in os.walk(folder): for name in names: src = os.path.join(root, name) rel = os.path.relpath(src, folder) shutil.copyfile(src, os.path.join(BASE, rel)) n += 1 return n def migrate(): """Apply migrations before the restart, not after. A bundle that carries both a model and its migration has to migrate while the old code is still serving, otherwise the new model asks the database for a column that the migration has not added yet and every page 500s. Splitting that across two uploads is not a workflow, it is a trap. """ cmd = ('cd %s && sudo -u %s venv/bin/python manage.py migrate --noinput' % (BASE, SITE['user'])) say(' applying migrations') r = run(cmd) out = (r.stdout or '') + (r.stderr or '') for line in out.splitlines(): line = line.strip() if line.startswith('Applying') or line.startswith(' Applying'): say(' ' + line) if r.returncode != 0: tail = [l for l in out.splitlines() if l.strip() and 'profiling:' not in l][-6:] return False, ' / '.join(t.strip() for t in tail) if 'No migrations to apply' in out: say(' nothing new to apply') return True, '' def restart_and_check(): say(' restarting %s' % SITE['service']) r = run('systemctl restart %s' % SITE['service']) if r.returncode != 0: return False, (r.stderr or '').strip()[:200] run('sleep 4') r = run('systemctl is-active %s' % SITE['service']) if r.stdout.strip() != 'active': return False, 'service is %s' % r.stdout.strip() # Both halves, because they fail independently. A missing module imported # only by the panel leaves the public site answering 200 while every page # of the CMS is a 500, and a check that looks at one of them calls that a # successful deploy. for label, url in (('public', SITE.get('health')), ('panel', SITE.get('panel'))): if not url: continue r = run('curl -s -o /dev/null -w "%%{http_code}" -m 20 %s' % url) code = r.stdout.strip() if code not in ('200', '301', '302'): return False, '%s returned %s' % (label, code) say(' %s %s' % (label, code)) return True, 'both answered' # ---- identity ------------------------------------------------------------ if not os.path.isdir(BASE): sys.exit(' REFUSING: %s does not exist on this machine' % BASE) env_path = os.path.join(BASE, '.env') if os.path.isfile(env_path): env = open(env_path, errors='ignore').read() if SITE['marker'] not in env: sys.exit(' REFUSING: %s is not mentioned in %s, wrong site' % (SITE['marker'], env_path)) else: say(' note: no .env found, skipping the site identity check') say(' site : %s' % SITE['service']) say(' target : %s' % BASE) say(' built : from %s' % SOURCE) say() # ---- rollback ------------------------------------------------------------ if MODE == '--rollback': folder = newest_backup() if not folder: sys.exit(' nothing to roll back') n = restore(folder) say(' restored %d files from %s' % (n, os.path.basename(folder))) ok, detail = restart_and_check() say(' %s after rollback (%s)' % ('healthy' if ok else 'STILL BROKEN', detail)) sys.exit(0 if ok else 1) # ---- compare ------------------------------------------------------------- changed, identical, missing = [], [], [] for rel, meta in sorted(FILES.items()): live = os.path.join(BASE, rel) new = base64.b64decode(meta['b64']) if not os.path.exists(live): missing.append(rel) say(' NEW %s (%d bytes)' % (rel, meta['size'])) continue cur = open(live, 'rb').read() if hashlib.sha256(cur).hexdigest() == meta['sha']: identical.append(rel) say(' same %s' % rel) continue changed.append(rel) diff = list(difflib.unified_diff( cur.decode(errors='replace').splitlines(), new.decode(errors='replace').splitlines(), fromfile='live/' + rel, tofile='new/' + rel, lineterm='', n=2)) adds = sum(1 for d in diff if d.startswith('+') and not d.startswith('+++')) dels = sum(1 for d in diff if d.startswith('-') and not d.startswith('---')) say(' CHANGED %s (+%d -%d)' % (rel, adds, dels)) for line in diff[:400]: say(' ' + line) if len(diff) > 400: say(' ... %d more diff lines' % (len(diff) - 400)) say() say(' %d to write, %d already current, %d new' % (len(changed), len(identical), len(missing))) if not changed and not missing: say(' nothing to do') sys.exit(0) if MODE != '--apply': say() say(' Read the diff above. Nothing was written.') say(' Re-run with --apply when it is what you expect.') sys.exit(0) # ---- validate ------------------------------------------------------------ for rel in changed + missing: new = base64.b64decode(FILES[rel]['b64']).decode(errors='replace') if rel.endswith('.py'): try: compile(new, rel, 'exec') except SyntaxError as exc: sys.exit(' REFUSING: %s does not compile, %s' % (rel, exc)) if rel.endswith('.html'): for a, b in ((r'{%\s*if\b', r'{%\s*endif\s*%}'), (r'{%\s*for\b', r'{%\s*endfor'), (r'{%\s*block\b', r'{%\s*endblock')): if len(re.findall(a, new)) != len(re.findall(b, new)): sys.exit(' REFUSING: %s has unbalanced template tags' % rel) if '{#' in new: sys.exit(' REFUSING: %s contains a template comment' % rel) say(' all files validate') # ---- write --------------------------------------------------------------- folder = os.path.join(BACKUPS, STAMP) for rel in changed: dest = os.path.join(folder, rel) os.makedirs(os.path.dirname(dest), exist_ok=True) shutil.copyfile(os.path.join(BASE, rel), dest) say(' backed up %d files to deploy_backups/%s' % (len(changed), STAMP)) for rel in changed + missing: live = os.path.join(BASE, rel) os.makedirs(os.path.dirname(live), exist_ok=True) open(live, 'wb').write(base64.b64decode(FILES[rel]['b64'])) shutil.chown(live, SITE['user'], SITE['user']) say(' wrote %s' % rel) if any('migrations/' in rel for rel in changed + missing): ok, detail = migrate() if not ok: say() say(' MIGRATION FAILED: %s' % detail) say(' rolling the files back, the database was not left half done') restore(folder) for rel in missing: p = os.path.join(BASE, rel) if os.path.exists(p): os.remove(p) restart_and_check() sys.exit(1) ok, detail = restart_and_check() if ok: say() say(' done, site healthy (%s)' % detail) sys.exit(0) say() say(' FAILED: %s' % detail) say(' rolling back') restore(folder) for rel in missing: p = os.path.join(BASE, rel) if os.path.exists(p): os.remove(p) ok2, detail2 = restart_and_check() say(' %s after rollback (%s)' % ('recovered' if ok2 else 'STILL BROKEN', detail2)) sys.exit(1)