From 9f43c1a09e374a0e8b94182c8f22bc0e21e656e7 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 22 Jul 2024 16:29:58 -0700 Subject: [PATCH] did it --- devenv/sync.py | 145 +++++++++++-------------------------------------- 1 file changed, 33 insertions(+), 112 deletions(-) diff --git a/devenv/sync.py b/devenv/sync.py index 24fc87847c5e2..600b0118ef8a3 100644 --- a/devenv/sync.py +++ b/devenv/sync.py @@ -4,8 +4,6 @@ import configparser import functools import os -import shlex -import subprocess from libdevinfra.jobs import Job, Task, run_jobs @@ -13,65 +11,6 @@ from devenv.lib import colima, config, fs, limactl, proc, venv, volta -# TODO: need to replace this with a nicer process executor in devenv.lib -def run_procs( - repo: str, - reporoot: str, - venv_path: str, - _procs: tuple[tuple[str, tuple[str, ...], dict[str, str]], ...], -) -> bool: - procs: list[tuple[str, tuple[str, ...], subprocess.Popen[bytes]]] = [] - - for name, cmd, extra_env in _procs: - print(f"⏳ {name}") - if constants.DEBUG: - proc.xtrace(cmd) - env = { - **constants.user_environ, - **proc.base_env, - "VIRTUAL_ENV": venv_path, - "VOLTA_HOME": f"{reporoot}/.devenv/bin/volta-home", - "PATH": f"{venv_path}/bin:{reporoot}/.devenv/bin:{proc.base_path}", - } - if extra_env: - env = {**env, **extra_env} - procs.append( - ( - name, - cmd, - subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=env, - cwd=reporoot, - ), - ) - ) - - all_good = True - for name, final_cmd, p in procs: - out, _ = p.communicate() - if p.returncode != 0: - all_good = False - print( - f""" -❌ {name} - -failed command (code p.returncode): - {shlex.join(final_cmd)} - -Output: -{out.decode()} - -""" - ) - else: - print(f"✅ {name}") - - return all_good - - def main(context: dict[str, str]) -> int: repo = context["repo"] reporoot = context["reporoot"] @@ -154,8 +93,6 @@ def main(context: dict[str, str]) -> int: ) jp3 = Task( name="install dependencies", - # could opt out of syncing python if FRONTEND_ONLY but only if repo-local devenv - # and pre-commit were moved to inside devenv and not the sentry venv func=functools.partial( proc.run, ( @@ -174,32 +111,48 @@ def main(context: dict[str, str]) -> int: ) jp4 = Task( name="install editable", - # could opt out of syncing python if FRONTEND_ONLY but only if repo-local devenv - # and pre-commit were moved to inside devenv and not the sentry venv func=functools.partial( proc.run, (f"{venv_dir}/bin/python3", "-m", "tools.fast_editable", "--path", "."), stdout=True, ), ) - - # jp5 shoudl start devservices iff freontend-only - + jp5 = Task( + name="init sentry config", + func=functools.partial( + proc.run, + (f"{venv_dir}/bin/sentry", "init", "--dev", "--no-clobber"), + stdout=True, + ), + ) jpc1 = Task( name="install pre-commit dependencies", - # could opt out of syncing python if FRONTEND_ONLY but only if repo-local devenv - # and pre-commit were moved to inside devenv and not the sentry venv func=functools.partial( proc.run, (f"{venv_dir}/bin/pre-commit", "install", "--install-hooks", "-f"), - # pathprepend=f"{venv_path}/bin:{reporoot}/.devenv/bin", + stdout=True, + ), + ) + jm1 = Task( + name="bring up redis and postgres", + func=functools.partial( + proc.run, + (f"{venv_dir}/bin/sentry", "devservices", "up", "redis", "postgres"), + stdout=True, + ), + ) + jm2 = Task( + name="apply migrations", + func=functools.partial( + proc.run, + ("make", "apply-migrations"), + pathprepend=f"{venv_dir}/bin:{reporoot}/.devenv/bin", + env={"VIRTUAL_ENV": venv_dir}, stdout=True, ), ) jt1 = Task( name="install js dependencies", - # could opt out of syncing python if FRONTEND_ONLY but only if repo-local devenv - # and pre-commit were moved to inside devenv and not the sentry venv func=functools.partial( proc.run, ( @@ -218,54 +171,22 @@ def main(context: dict[str, str]) -> int: ), ) - jp = Job(name="python dependencies", tasks=(jp1, jp2, jp3, jp4)) + jp = Job(name="python dependencies", tasks=(jp1, jp2, jp3, jp4, jp5)) + jm = Job(name="sentry migrations", tasks=(jm1, jm2)) jpc = Job(name="pre-commit dependencies", tasks=(jpc1,)) jt = Job(name="javascript dependencies", tasks=(jt1,)) # after python deps are installed we can install pre-commit deps jp3.spawn_jobs = (jpc,) - with concurrent.futures.ThreadPoolExecutor() as tpe: - run_jobs((jp, jt), tpe) - - fs.ensure_symlink("../../config/hooks/post-merge", f"{reporoot}/.git/hooks/post-merge") - - if not os.path.exists(f"{constants.home}/.sentry/config.yml") or not os.path.exists( - f"{constants.home}/.sentry/sentry.conf.py" - ): - proc.run((f"{venv_dir}/bin/sentry", "init", "--dev")) - # Frontend engineers don't necessarily always have devservices running and # can configure to skip them to save on local resources if FRONTEND_ONLY: print("Skipping python migrations since SENTRY_DEVENV_FRONTEND_ONLY is set.") - return 0 - - # TODO: check healthchecks for redis and postgres to short circuit this - proc.run( - ( - f"{venv_dir}/bin/{repo}", - "devservices", - "up", - "redis", - "postgres", - ), - pathprepend=f"{reporoot}/.devenv/bin", - exit=True, - ) + else: + jp5.spawn_jobs = (jm,) - if run_procs( - repo, - reporoot, - venv_dir, - ( - ( - "python migrations", - ("make", "apply-migrations"), - {}, - ), - ), - ): - return 0 + with concurrent.futures.ThreadPoolExecutor() as tpe: + run_jobs((jp, jt), tpe) - return 1 + fs.ensure_symlink("../../config/hooks/post-merge", f"{reporoot}/.git/hooks/post-merge")