#!/usr/bin/env python3 """ Deploy runner. Built by deploy/build_ship.py, always arrives as ship.py. python3 ship.py show what would change, write nothing python3 ship.py --apply back up, write, restart, verify, roll back on failure python3 ship.py --rollback put the last deploy back What it refuses to do: run against the wrong site, write a Python file that does not compile, write a template whose tags do not balance, or leave a broken site up. """ import base64 import datetime import difflib import glob import hashlib import json import os import re import shutil import subprocess import sys SITE = { "user": "finaltarget", "base": "/home/finaltarget/public_html/backend", "service": "finaltarget", "health": "https://finaltarget.com/", "panel": "https://panel.finaltarget.com/auth/login/", "marker": "finaltarget" } FILES = { "templates/public/auction/list.html": { "b64": "eyUgZXh0ZW5kcyAicHVibGljL2Jhc2UuaHRtbCIgJX0KeyUgbG9hZCBzdGF0aWMgaHVtYW5pemUgJX0KeyUgYmxvY2sgdGl0bGUgJX1QcmVtaXVtIERvbWFpbnMg4oCUIHt7IHNpdGUuc2l0ZV9uYW1lIH19eyUgZW5kYmxvY2sgJX0KeyUgYmxvY2sgbWV0YV9kZXNjcmlwdGlvbiAlfVByZW1pdW0gZG9tYWluIG5hbWVzIGF2YWlsYWJsZSB0byBidXkgb3IgcmVzZXJ2ZS4gU2hvcnQsIG1lbW9yYWJsZSBhbmQgcmVhZHkgdG8gdHJhbnNmZXIueyUgZW5kYmxvY2sgJX0KCnslIGJsb2NrIGV4dHJhX2hlYWQgJX0KPHN0eWxlPgojbG9hZC1tb3JlLWJ0biB7CiAgICBkaXNwbGF5OiBub25lOwogICAgbWFyZ2luOiAycmVtIGF1dG8gMDsKICAgIGJhY2tncm91bmQ6IHRyYW5zcGFyZW50OyBjb2xvcjogIzAzMzE1MjsKICAgIGJvcmRlcjogMnB4IHNvbGlkICMwMzMxNTI7IGJvcmRlci1yYWRpdXM6IDk5OXB4OwogICAgcGFkZGluZzogLjdyZW0gMi41cmVtOyBmb250LXdlaWdodDogNzAwOyBmb250LXNpemU6IC45NXJlbTsKICAgIGN1cnNvcjogcG9pbnRlcjsgdHJhbnNpdGlvbjogYWxsIC4yczsKfQojbG9hZC1tb3JlLWJ0bjpob3ZlciB7IGJhY2tncm91bmQ6ICMwMzMxNTI7IGNvbG9yOiAjZmZmOyB9Ci5kb20tc3RhdHVzIHsKICAgIGRpc3BsYXk6aW5saW5lLWJsb2NrO2JvcmRlci1yYWRpdXM6OTlweDtwYWRkaW5nOi4ycmVtIC42cmVtOwogICAgZm9udC1zaXplOi42NXJlbTtmb250LXdlaWdodDo3MDA7bWFyZ2luLXRvcDouMzVyZW07Cn0KLmRvbS1zdGF0dXMucmVzZXJ2ZWQgeyBiYWNrZ3JvdW5kOiNmZmY0ZTU7Y29sb3I6I2ExNWMwMDtib3JkZXI6MXB4IHNvbGlkICNhMTVjMDAzMzsgfQouZG9tLXN0YXR1cy5zb2xkICAgICB7IGJhY2tncm91bmQ6I2YxZjFmMTtjb2xvcjojNjY2O2JvcmRlcjoxcHggc29saWQgIzY2NjY2NjMzOyB9Cjwvc3R5bGU+CnslIGVuZGJsb2NrICV9Cgp7JSBibG9jayBjb250ZW50ICV9CjxkaXYgY2xhc3M9InNlY3Rpb24tZmlyc3QtaW5uZXIiPgoKICA8ZGl2IGNsYXNzPSJzZWN0aW9uIGNvbXBhY3QtYm90dG9tIj4KICAgIDxkaXYgY2xhc3M9Imhlcm8tY29udGFpbmVyIj4KICAgICAgPGRpdiBjbGFzcz0iYmFubmVyLWlubmVyIiBzdHlsZT0ianVzdGlmeS1jb250ZW50OmZsZXgtc3RhcnQ7YWxpZ24taXRlbXM6ZmxleC1lbmQ7ZmxleC13cmFwOndyYXAiPgogICAgICAgIDxkaXYgY2xhc3M9ImJhbm5lci1pbm5lci1jb250ZW50IiBzdHlsZT0idGV4dC1hbGlnbjpsZWZ0Ij4KICAgICAgICAgIDxoMT5QcmVtaXVtIERvbWFpbnM8L2gxPgogICAgICAgIDwvZGl2PgogICAgICA8L2Rpdj4KICAgIDwvZGl2PgogIDwvZGl2PgoKICA8ZGl2IGNsYXNzPSJzZWN0aW9uIj4KICAgIDxkaXYgY2xhc3M9Imhlcm8tY29udGFpbmVyIj4KICAgICAgPGRpdiBjbGFzcz0iZC1mbGV4IGZsZXgtY29sdW1uIGdzcGFjZS0yIj4KCiAgICAgICAgeyUgaWYgbGlzdGluZ3MgJX0KICAgICAgICA8ZGl2IGNsYXNzPSJyb3cgcm93LWNvbHMtbGctNCByb3ctY29scy1tZC0yIHJvdy1jb2xzLTEgZ3JpZC1zcGFjZXItMiIgaWQ9InNob3AtZ3JpZCI+CiAgICAgICAgICB7JSBmb3IgcCBpbiBsaXN0aW5ncyAlfQogICAgICAgICAgPGRpdiBjbGFzcz0iY29sIiBzdHlsZT0ieyUgaWYgZm9ybG9vcC5jb3VudGVyID4gMTYgJX1kaXNwbGF5Om5vbmV7JSBlbmRpZiAlfSIgZGF0YS1pbmRleD0ie3sgZm9ybG9vcC5jb3VudGVyIH19Ij4KICAgICAgICAgICAgPGRpdiBjbGFzcz0iY2FyZCBjYXJkLXByaWNpbmciPgoKICAgICAgICAgICAgICA8ZGl2IGNsYXNzPSJzdWItaGVhZGluZyB0ZXh0LWNlbnRlciBqdXN0aWZ5LWNvbnRlbnQtY2VudGVyIj4KICAgICAgICAgICAgICAgIDxpIGNsYXNzPSJ7eyBwLmljb25fY2xhc3N8ZGVmYXVsdDonZmEtc29saWQgZmEtZ2xvYmUnIH19Ij48L2k+CiAgICAgICAgICAgICAgICA8aDYgY2xhc3M9ImZvbnQtZmFtaWx5LTEgYWNjZW50LWNvbG9yIHRleHQtY2VudGVyIj5QcmVtaXVtIERvbWFpbjwvaDY+CiAgICAgICAgICAgICAgPC9kaXY+CgogICAgICAgICAgICAgIDxkaXYgY2xhc3M9ImNhcmQgcHJpY2UtY29udGFpbmVyIj4KICAgICAgICAgICAgICAgIDxkaXYgY2xhc3M9InByaWNlLXdyYXBwZXIiPgogICAgICAgICAgICAgICAgICA8c3BhbiBjbGFzcz0icHJpY2UiPnt7IHAubmFtZSB9fTwvc3Bhbj4KICAgICAgICAgICAgICAgICAgeyUgaWYgcC5hdWN0aW9uX3N0YXR1cyA9PSAncmVzZXJ2ZWQnICV9CiAgICAgICAgICAgICAgICAgIDxzcGFuIGNsYXNzPSJkb20tc3RhdHVzIHJlc2VydmVkIj5SZXNlcnZlZDwvc3Bhbj4KICAgICAgICAgICAgICAgICAgeyUgZWxpZiBwLmF1Y3Rpb25fc3RhdHVzID09ICdzb2xkJyAlfQogICAgICAgICAgICAgICAgICA8c3BhbiBjbGFzcz0iZG9tLXN0YXR1cyBzb2xkIj5Tb2xkPC9zcGFuPgogICAgICAgICAgICAgICAgICB7JSBlbGlmIHAuYmFkZ2UgJX0KICAgICAgICAgICAgICAgICAgPHNwYW4gc3R5bGU9ImRpc3BsYXk6aW5saW5lLWJsb2NrO2JhY2tncm91bmQ6e3sgcC5iYWRnZV9iZyB9fTtjb2xvcjp7eyBwLmJhZGdlX2NvbG9yIH19O2JvcmRlcjoxcHggc29saWQge3sgcC5iYWRnZV9jb2xvciB9fTMzO2JvcmRlci1yYWRpdXM6OTlweDtwYWRkaW5nOi4ycmVtIC42cmVtO2ZvbnQtc2l6ZTouNjVyZW07Zm9udC13ZWlnaHQ6NzAwO21hcmdpbi10b3A6LjM1cmVtIj57eyBwLmJhZGdlIH19PC9zcGFuPgogICAgICAgICAgICAgICAgICB7JSBlbmRpZiAlfQogICAgICAgICAgICAgICAgPC9kaXY+CiAgICAgICAgICAgICAgPC9kaXY+CgogICAgICAgICAgICAgIHslIGlmIHAuc2FsZV9tZXRob2QgPT0gJ29mZmVyJyAlfQogICAgICAgICAgICAgIDxhIGhyZWY9InslIHVybCAnYXVjdGlvbl9kZXRhaWwnIHAuc2x1ZyAlfSIgY2xhc3M9ImJ0biBidG4tYWNjZW50Ij4KICAgICAgICAgICAgICAgIHslIGlmIHAuYXVjdGlvbl9taW5fb2ZmZXIgJX1PZmZlcnMgb3ZlciAmcG91bmQ7e3sgcC5hdWN0aW9uX21pbl9vZmZlcnxpbnRjb21tYSB9fXslIGVsc2UgJX1NYWtlIGFuIEVucXVpcnl7JSBlbmRpZiAlfQogICAgICAgICAgICAgIDwvYT4KICAgICAgICAgICAgICB7JSBlbHNlICV9CiAgICAgICAgICAgICAgPGEgaHJlZj0ieyUgdXJsICdhdWN0aW9uX2RldGFpbCcgcC5zbHVnICV9IiBjbGFzcz0iYnRuIGJ0bi1hY2NlbnQiPgogICAgICAgICAgICAgICAgeyUgaWYgcC5wcmljZSAlfUJ1eSBOb3csICZwb3VuZDt7eyBwLnByaWNlfGludGNvbW1hIH19eyUgZWxzZSAlfVZpZXcgRGV0YWlsc3slIGVuZGlmICV9CiAgICAgICAgICAgICAgPC9hPgogICAgICAgICAgICAgIHslIGVuZGlmICV9CgogICAgICAgICAgICA8L2Rpdj4KICAgICAgICAgIDwvZGl2PgogICAgICAgICAgeyUgZW5kZm9yICV9CiAgICAgICAgPC9kaXY+CgogICAgICAgIDxkaXYgc3R5bGU9InRleHQtYWxpZ246Y2VudGVyIj4KICAgICAgICAgIDxidXR0b24gaWQ9ImxvYWQtbW9yZS1idG4iIGRhdGEtc2hvd249IjE2Ij5Mb2FkIE1vcmU8L2J1dHRvbj4KICAgICAgICA8L2Rpdj4KCiAgICAgICAgeyUgZWxzZSAlfQogICAgICAgIDxkaXYgY2xhc3M9InRleHQtY2VudGVyIHB5LTUiPgogICAgICAgICAgPHAgY2xhc3M9InNlY29uZGFyeS1hY2NlbnQiPk5vIGRvbWFpbnMgYXZhaWxhYmxlIHlldC4gQ2hlY2sgYmFjayBzb29uLjwvcD4KICAgICAgICA8L2Rpdj4KICAgICAgICB7JSBlbmRpZiAlfQoKICAgICAgPC9kaXY+CiAgICA8L2Rpdj4KICA8L2Rpdj4KCjwvZGl2Pgp7JSBlbmRibG9jayAlfQoKeyUgYmxvY2sgZXh0cmFfanMgJX0KPHNjcmlwdD4KKGZ1bmN0aW9uICgpIHsKICB2YXIgYnRuICAgID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoJ2xvYWQtbW9yZS1idG4nKTsKICBpZiAoIWJ0bikgcmV0dXJuOwogIHZhciBjYXJkcyAgPSBBcnJheS5mcm9tKGRvY3VtZW50LnF1ZXJ5U2VsZWN0b3JBbGwoJyNzaG9wLWdyaWQgLmNvbCcpKTsKICB2YXIgdG90YWwgID0gY2FyZHMubGVuZ3RoOwogIHZhciBzaG93biAgPSAxNjsKCiAgZnVuY3Rpb24gdXBkYXRlKCkgewogICAgaWYgKHRvdGFsID4gc2hvd24pIHsKICAgICAgYnRuLnN0eWxlLmRpc3BsYXkgPSAnaW5saW5lLWJsb2NrJzsKICAgICAgYnRuLmRhdGFzZXQuc2hvd24gPSBzaG93bjsKICAgIH0gZWxzZSB7CiAgICAgIGJ0bi5zdHlsZS5kaXNwbGF5ID0gJ25vbmUnOwogICAgfQogIH0KCiAgdXBkYXRlKCk7CgogIGJ0bi5hZGRFdmVudExpc3RlbmVyKCdjbGljaycsIGZ1bmN0aW9uICgpIHsKICAgIHZhciBuZXh0ID0gc2hvd24gKyAxNjsKICAgIGNhcmRzLmZvckVhY2goZnVuY3Rpb24gKGNhcmQsIGkpIHsKICAgICAgaWYgKGkgPj0gc2hvd24gJiYgaSA8IG5leHQpIGNhcmQuc3R5bGUuZGlzcGxheSA9ICdmbGV4JzsKICAgIH0pOwogICAgc2hvd24gPSBuZXh0OwogICAgdXBkYXRlKCk7CiAgfSk7Cn0oKSk7Cjwvc2NyaXB0Pgp7JSBlbmRibG9jayAlfQo=", "sha": "cbc3cdea79f8fe322687e57b9da0257b5f3d6f51fcfab9c2ac5b32d29e43ee8e", "size": 4457 } } SOURCE = "The Panel" BASE = SITE['base'] STAMP = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') BACKUPS = os.path.join(BASE, 'deploy_backups') MODE = sys.argv[1] if len(sys.argv) > 1 else '--check' def say(msg=''): print(msg, flush=True) def run(cmd): """Run a shell command and hand back the result. Written the long way on purpose. The server's system python is 3.6 and subprocess.run only grew capture_output in 3.7, so the short form dies with a TypeError after the files are already written, which leaves the service unrestarted and the deploy half done. """ proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate() class Result(object): pass r = Result() r.returncode = proc.returncode r.stdout = out.decode('utf-8', 'replace') r.stderr = err.decode('utf-8', 'replace') return r def newest_backup(): dirs = sorted(glob.glob(os.path.join(BACKUPS, '*'))) return dirs[-1] if dirs else None def restore(folder): n = 0 for root, _dirs, names in os.walk(folder): for name in names: src = os.path.join(root, name) rel = os.path.relpath(src, folder) shutil.copyfile(src, os.path.join(BASE, rel)) n += 1 return n def migrate(): """Apply migrations before the restart, not after. A bundle that carries both a model and its migration has to migrate while the old code is still serving, otherwise the new model asks the database for a column that the migration has not added yet and every page 500s. Splitting that across two uploads is not a workflow, it is a trap. """ cmd = ('cd %s && sudo -u %s venv/bin/python manage.py migrate --noinput' % (BASE, SITE['user'])) say(' applying migrations') r = run(cmd) out = (r.stdout or '') + (r.stderr or '') for line in out.splitlines(): line = line.strip() if line.startswith('Applying') or line.startswith(' Applying'): say(' ' + line) if r.returncode != 0: tail = [l for l in out.splitlines() if l.strip() and 'profiling:' not in l][-6:] return False, ' / '.join(t.strip() for t in tail) if 'No migrations to apply' in out: say(' nothing new to apply') return True, '' def restart_and_check(): say(' restarting %s' % SITE['service']) r = run('systemctl restart %s' % SITE['service']) if r.returncode != 0: return False, (r.stderr or '').strip()[:200] run('sleep 4') r = run('systemctl is-active %s' % SITE['service']) if r.stdout.strip() != 'active': return False, 'service is %s' % r.stdout.strip() # Both halves, because they fail independently. A missing module imported # only by the panel leaves the public site answering 200 while every page # of the CMS is a 500, and a check that looks at one of them calls that a # successful deploy. for label, url in (('public', SITE.get('health')), ('panel', SITE.get('panel'))): if not url: continue r = run('curl -s -o /dev/null -w "%%{http_code}" -m 20 %s' % url) code = r.stdout.strip() if code not in ('200', '301', '302'): return False, '%s returned %s' % (label, code) say(' %s %s' % (label, code)) return True, 'both answered' # ---- identity ------------------------------------------------------------ if not os.path.isdir(BASE): sys.exit(' REFUSING: %s does not exist on this machine' % BASE) env_path = os.path.join(BASE, '.env') if os.path.isfile(env_path): env = open(env_path, errors='ignore').read() if SITE['marker'] not in env: sys.exit(' REFUSING: %s is not mentioned in %s, wrong site' % (SITE['marker'], env_path)) else: say(' note: no .env found, skipping the site identity check') say(' site : %s' % SITE['service']) say(' target : %s' % BASE) say(' built : from %s' % SOURCE) say() # ---- rollback ------------------------------------------------------------ if MODE == '--rollback': folder = newest_backup() if not folder: sys.exit(' nothing to roll back') n = restore(folder) say(' restored %d files from %s' % (n, os.path.basename(folder))) ok, detail = restart_and_check() say(' %s after rollback (%s)' % ('healthy' if ok else 'STILL BROKEN', detail)) sys.exit(0 if ok else 1) # ---- compare ------------------------------------------------------------- changed, identical, missing = [], [], [] for rel, meta in sorted(FILES.items()): live = os.path.join(BASE, rel) new = base64.b64decode(meta['b64']) if not os.path.exists(live): missing.append(rel) say(' NEW %s (%d bytes)' % (rel, meta['size'])) continue cur = open(live, 'rb').read() if hashlib.sha256(cur).hexdigest() == meta['sha']: identical.append(rel) say(' same %s' % rel) continue changed.append(rel) diff = list(difflib.unified_diff( cur.decode(errors='replace').splitlines(), new.decode(errors='replace').splitlines(), fromfile='live/' + rel, tofile='new/' + rel, lineterm='', n=2)) adds = sum(1 for d in diff if d.startswith('+') and not d.startswith('+++')) dels = sum(1 for d in diff if d.startswith('-') and not d.startswith('---')) say(' CHANGED %s (+%d -%d)' % (rel, adds, dels)) for line in diff[:400]: say(' ' + line) if len(diff) > 400: say(' ... %d more diff lines' % (len(diff) - 400)) say() say(' %d to write, %d already current, %d new' % (len(changed), len(identical), len(missing))) if not changed and not missing: say(' nothing to do') sys.exit(0) if MODE != '--apply': say() say(' Read the diff above. Nothing was written.') say(' Re-run with --apply when it is what you expect.') sys.exit(0) # ---- validate ------------------------------------------------------------ for rel in changed + missing: new = base64.b64decode(FILES[rel]['b64']).decode(errors='replace') if rel.endswith('.py'): try: compile(new, rel, 'exec') except SyntaxError as exc: sys.exit(' REFUSING: %s does not compile, %s' % (rel, exc)) if rel.endswith('.html'): for a, b in ((r'{%\s*if\b', r'{%\s*endif\s*%}'), (r'{%\s*for\b', r'{%\s*endfor'), (r'{%\s*block\b', r'{%\s*endblock')): if len(re.findall(a, new)) != len(re.findall(b, new)): sys.exit(' REFUSING: %s has unbalanced template tags' % rel) if '{#' in new: sys.exit(' REFUSING: %s contains a template comment' % rel) say(' all files validate') # ---- write --------------------------------------------------------------- folder = os.path.join(BACKUPS, STAMP) for rel in changed: dest = os.path.join(folder, rel) os.makedirs(os.path.dirname(dest), exist_ok=True) shutil.copyfile(os.path.join(BASE, rel), dest) say(' backed up %d files to deploy_backups/%s' % (len(changed), STAMP)) for rel in changed + missing: live = os.path.join(BASE, rel) os.makedirs(os.path.dirname(live), exist_ok=True) open(live, 'wb').write(base64.b64decode(FILES[rel]['b64'])) shutil.chown(live, SITE['user'], SITE['user']) say(' wrote %s' % rel) if any('migrations/' in rel for rel in changed + missing): ok, detail = migrate() if not ok: say() say(' MIGRATION FAILED: %s' % detail) say(' rolling the files back, the database was not left half done') restore(folder) for rel in missing: p = os.path.join(BASE, rel) if os.path.exists(p): os.remove(p) restart_and_check() sys.exit(1) ok, detail = restart_and_check() if ok: say() say(' done, site healthy (%s)' % detail) sys.exit(0) say() say(' FAILED: %s' % detail) say(' rolling back') restore(folder) for rel in missing: p = os.path.join(BASE, rel) if os.path.exists(p): os.remove(p) ok2, detail2 = restart_and_check() say(' %s after rollback (%s)' % ('recovered' if ok2 else 'STILL BROKEN', detail2)) sys.exit(1)