WIP: offline work (#165)

* add some cli commands

* support settings file

* pin black

* handle wandb.init(mode=)

* fix up wandb sync

tests/utils/mock_backend.py

@@ -49,6 +49,7 @@ def __init__(self, mode=None):
  self.config = {}
  self.files = {}
  self.mocker = _get_mock_module(get_config())
+ self._internal_pid = None
 
  def _hack_set_run(self, run):
  self._run = run

tox.ini

@@ -94,15 +94,15 @@ commands=
 basepython=python3
 skip_install = true
 deps=
- black
+ black==19.10b0
 commands=
  black wandb/
 
 [testenv:black]
 basepython=python3
 skip_install = true
 deps=
- black
+ black==19.10b0
 commands=
  black --check wandb/
 

wandb/backend/backend.py

@@ -17,12 +17,13 @@
 
 
 class Backend(object):
- def __init__(self, mode=None):
+ def __init__(self):
  self._done = False
  self.record_q = None
  self.result_q = None
  self.wandb_process = None
  self.interface = None
+ self._internal_pid = None
  self._wl = wandb.setup(_warn=False)
 
  def _hack_set_run(self, run):
@@ -77,6 +78,7 @@ def ensure_launched(
 
  # Start the process with __name__ == "__main__" workarounds
  self.wandb_process.start()
+ self._internal_pid = self.wandb_process.pid
 
  # Undo temporary changes from: __name__ == "__main__"
  if save_mod_name:

wandb/cli/cli.py

@@ -18,6 +18,7 @@
 # pycreds has a find_executable that works in windows
 from dockerpycreds.utils import find_executable
 import six
+from six.moves import configparser
 import wandb
 from wandb import Config
 from wandb import env, util
@@ -1244,3 +1245,31 @@ def magic_run(cmd, globals, locals):
  )
  magic_run(prep, globs, None)
  magic_run(code, globs, None)
+
+
+@cli.command("on", help="Ensure W&B is enabled in this directory")
+@display_error
+def on():
+ api = InternalApi()
+ try:
+ api.clear_setting("disabled", persist=True)
+ except configparser.Error:
+ pass
+ click.echo(
+ "W&B enabled, running your script from this directory will now sync to the cloud."
+ )
+
+
+@cli.command("off", help="Disable W&B in this directory, useful for testing")
+@display_error
+def off():
+ api = InternalApi()
+ try:
+ api.set_setting("disabled", "true", persist=True)
+ click.echo(
+ "W&B disabled, running your script from this directory will only write metadata locally."
+ )
+ except configparser.Error:
+ click.echo(
+ "Unable to write config, copy and paste the following in your terminal to turn off W&B:\nexport WANDB_MODE=dryrun"
+ )

wandb/sdk/wandb_init.py

@@ -283,6 +283,7 @@ def _log_setup(self, settings):
  settings.log_internal = settings._path_convert(
  settings.log_dir_spec, settings.log_internal_spec
  )
+ settings._sync_dir = settings._path_convert(settings.sync_dir_spec)
  settings.sync_file = settings._path_convert(
  settings.sync_dir_spec, settings.sync_file_spec
  )
@@ -354,7 +355,7 @@ def init(self):
  stdout_master_fd, stdout_slave_fd = lib_console.win32_create_pipe()
  stderr_master_fd, stderr_slave_fd = lib_console.win32_create_pipe()
 
- backend = Backend(mode=s.mode)
+ backend = Backend()
  backend.ensure_launched(
  settings=s,
  stdout_fd=stdout_master_fd,
@@ -363,7 +364,7 @@ def init(self):
  )
  backend.server_connect()
  # Make sure we are logged in
- _login(_backend=backend, _disable_warning=True)
+ _login(_backend=backend, _disable_warning=True, _settings=self.settings)
 
  # resuming needs access to the server, check server_status()?
 
@@ -446,8 +447,7 @@ def init(
  config_exclude_keys=None,
  config_include_keys=None,
  anonymous: Optional[str] = None,
- disabled: bool = None,
- offline: bool = None,
+ mode: Optional[str] = None,
  allow_val_change: bool = None,
  resume: Optional[Union[bool, str]] = None,
  force=None,

wandb/sdk/wandb_login.py

@@ -25,7 +25,12 @@ def login(anonymous=None, key=None, relogin=None):
 
 
 def _login(
- anonymous=None, key=None, relogin=None, _backend=None, _disable_warning=None
+ anonymous=None,
+ key=None,
+ relogin=None,
+ _backend=None,
+ _disable_warning=None,
+ _settings=None,
 ):
  """Log in to W&B.
 
@@ -62,7 +67,7 @@ def _login(
  # you must make sure the anonymous setting (and any other settings) are
  # already properly set up there.
  wl = wandb.setup(settings=settings, _warn=False)
- settings = wl.settings()
+ settings = _settings or wl.settings()
 
  if settings.offline:
  return

wandb/sdk/wandb_run.py

@@ -838,6 +838,8 @@ def _display_run(self):
  click.style(run_url, underline=True, fg="blue"),
  )
  )
+ if not self._settings.offline:
+ wandb.termlog("Run `wandb off` to turn off syncing.")
 
  def _redirect(self, stdout_slave_fd, stderr_slave_fd):
  console = self._settings.console
@@ -960,7 +962,19 @@ def _console_stop(self):
  self._output_writer = None
 
  def _on_start(self):
+ if self._settings.offline:
+ wandb.termlog("Offline run mode, not syncing to the cloud.")
  wandb.termlog("Tracking run with wandb version {}".format(wandb.__version__))
+ if self._settings.offline:
+ wandb.termlog(
+ (
+ "W&B is disabled in this directory. "
+ "Run `wandb on` to enable cloud syncing."
+ )
+ )
+ wandb.termlog(
+ "Run data is saved locally in {}".format(self._settings._sync_dir)
+ )
  if self._run_obj:
  if self.resumed:
  run_state_str = "Resuming run"
@@ -1031,6 +1045,15 @@ def _on_finish(self):
 
  self._console_stop()
  print("")
+ pid = self._backend._internal_pid
+ wandb.termlog("Waiting for W&B process to finish, PID {}".format(pid))
+ if not self._exit_code:
+ wandb.termlog("Program ended successfully.")
+ else:
+ msg = "Program failed with code {}. ".format(self._exit_code)
+ if not self._settings.offline:
+ msg += " Press ctrl-c to abort syncing."
+ wandb.termlog(msg)
 
  if self._settings.offline:
  self._backend.interface.publish_exit(self._exit_code)
@@ -1087,6 +1110,13 @@ def _on_final(self):
  self._settings.log_internal
  )
  )
+ if self._settings.offline:
+ wandb.termlog("You can sync this run to the cloud by running:")
+ wandb.termlog(
+ click.style(
+ "wandb sync {}".format(self._settings.sync_file), fg="yellow"
+ )
+ )
 
  self._show_summary()
  self._show_history()

wandb/sdk/wandb_settings.py

@@ -66,7 +66,7 @@
  show_warnings=2,
  summary_warnings=5,
  # old mode field (deprecated in favor of WANDB_OFFLINE=true)
- _mode=Field(str, ("dryrun", "run",)),
+ _mode=Field(str, ("dryrun", "run", "offline", "online",)),
  # problem: TODO(jhr): Not implemented yet, needs new name?
  _problem=Field(str, ("fatal", "warn", "silent",)),
  console="auto",
@@ -237,6 +237,7 @@ def __init__( # pylint: disable=unused-argument
  # sync_symlink_sync_spec="{wandb_dir}/sync",
  # sync_symlink_offline_spec="{wandb_dir}/offline",
  sync_symlink_latest_spec="{wandb_dir}/latest-run",
+ _sync_dir=None, # computed
  sync_file=None, # computed
  log_dir_spec="{wandb_dir}/{run_mode}-{timespec}-{run_id}/logs",
  log_user_spec="debug-{timespec}-{run_id}.log",
@@ -264,6 +265,7 @@ def __init__( # pylint: disable=unused-argument
  save_code=None,
  program_relpath=None,
  git_remote=None,
+ dev_prod=None, # in old settings files, TODO: support?
  host=None,
  username=None,
  docker=None,
@@ -422,8 +424,18 @@ def update(self, __d=None, _setter=None, **kwargs):
  self.__dict__.update({k: v for k, v in d.items() if v is not None})
  self.__dict__.update({k: v for k, v in kwargs.items() if v is not None})
 
+ def _reinfer_settings_from_env(self):
+ """As settings change we might want to run this again."""
+ # figure out if we are in offline mode
+ # (disabled is how it is stored in settings files)
+ if self.disabled:
+ self.offline = True
+ if self.mode in ("dryrun", "offline"):
+ self.offline = True
+
  def _infer_settings_from_env(self):
  """Modify settings based on environment (for runs and cli)."""
+
  d = {}
  d["jupyter"] = _get_python_type() != "python"
  d["_kaggle"] = _is_kaggle()
@@ -444,10 +456,6 @@ def _infer_settings_from_env(self):
  # console = "off"
  u["console"] = console
 
- # convert wandb mode to "offline"
- if self.mode == "dryrun":
- self.offline = True
-
  # For code saving, only allow env var override if value from server is true, or
  # if no preference was specified.
  if (self.save_code is True or self.save_code is None) and os.getenv(
@@ -490,6 +498,7 @@ def _infer_settings_from_env(self):
  u["_except_exit"] = True
 
  self.update(u)
+ self._reinfer_settings_from_env()
 
  def _infer_run_settings_from_env(self):
  """Modify settings based on environment (for runs only)."""
@@ -593,3 +602,4 @@ def apply_init(self, args):
  wandb.util.mkdir_exists_ok(self.wandb_dir)
  with open(self.resume_fname, "w") as f:
  f.write(json.dumps({"run_id": self.run_id}))
+ self._reinfer_settings_from_env()

wandb/sdk_py27/wandb_init.py

@@ -283,6 +283,7 @@ def _log_setup(self, settings):
  settings.log_internal = settings._path_convert(
  settings.log_dir_spec, settings.log_internal_spec
  )
+ settings._sync_dir = settings._path_convert(settings.sync_dir_spec)
  settings.sync_file = settings._path_convert(
  settings.sync_dir_spec, settings.sync_file_spec
  )
@@ -354,7 +355,7 @@ def init(self):
  stdout_master_fd, stdout_slave_fd = lib_console.win32_create_pipe()
  stderr_master_fd, stderr_slave_fd = lib_console.win32_create_pipe()
 
- backend = Backend(mode=s.mode)
+ backend = Backend()
  backend.ensure_launched(
  settings=s,
  stdout_fd=stdout_master_fd,
@@ -363,7 +364,7 @@ def init(self):
  )
  backend.server_connect()
  # Make sure we are logged in
- _login(_backend=backend, _disable_warning=True)
+ _login(_backend=backend, _disable_warning=True, _settings=self.settings)
 
  # resuming needs access to the server, check server_status()?
 
@@ -446,8 +447,7 @@ def init(
  config_exclude_keys=None,
  config_include_keys=None,
  anonymous = None,
- disabled = None,
- offline = None,
+ mode = None,
  allow_val_change = None,
  resume = None,
  force=None,

wandb/sdk_py27/wandb_login.py

@@ -25,7 +25,12 @@ def login(anonymous=None, key=None, relogin=None):
 
 
 def _login(
- anonymous=None, key=None, relogin=None, _backend=None, _disable_warning=None
+ anonymous=None,
+ key=None,
+ relogin=None,
+ _backend=None,
+ _disable_warning=None,
+ _settings=None,
 ):
  """Log in to W&B.
 
@@ -62,7 +67,7 @@ def _login(
  # you must make sure the anonymous setting (and any other settings) are
  # already properly set up there.
  wl = wandb.setup(settings=settings, _warn=False)
- settings = wl.settings()
+ settings = _settings or wl.settings()
 
  if settings.offline:
  return

wandb/sdk_py27/wandb_run.py

@@ -838,6 +838,8 @@ def _display_run(self):
  click.style(run_url, underline=True, fg="blue"),
  )
  )
+ if not self._settings.offline:
+ wandb.termlog("Run `wandb off` to turn off syncing.")
 
  def _redirect(self, stdout_slave_fd, stderr_slave_fd):
  console = self._settings.console
@@ -960,7 +962,19 @@ def _console_stop(self):
  self._output_writer = None
 
  def _on_start(self):
+ if self._settings.offline:
+ wandb.termlog("Offline run mode, not syncing to the cloud.")
  wandb.termlog("Tracking run with wandb version {}".format(wandb.__version__))
+ if self._settings.offline:
+ wandb.termlog(
+ (
+ "W&B is disabled in this directory. "
+ "Run `wandb on` to enable cloud syncing."
+ )
+ )
+ wandb.termlog(
+ "Run data is saved locally in {}".format(self._settings._sync_dir)
+ )
  if self._run_obj:
  if self.resumed:
  run_state_str = "Resuming run"
@@ -1031,6 +1045,15 @@ def _on_finish(self):
 
  self._console_stop()
  print("")
+ pid = self._backend._internal_pid
+ wandb.termlog("Waiting for W&B process to finish, PID {}".format(pid))
+ if not self._exit_code:
+ wandb.termlog("Program ended successfully.")
+ else:
+ msg = "Program failed with code {}. ".format(self._exit_code)
+ if not self._settings.offline:
+ msg += " Press ctrl-c to abort syncing."
+ wandb.termlog(msg)
 
  if self._settings.offline:
  self._backend.interface.publish_exit(self._exit_code)
@@ -1087,6 +1110,13 @@ def _on_final(self):
  self._settings.log_internal
  )
  )
+ if self._settings.offline:
+ wandb.termlog("You can sync this run to the cloud by running:")
+ wandb.termlog(
+ click.style(
+ "wandb sync {}".format(self._settings.sync_file), fg="yellow"
+ )
+ )
 
  self._show_summary()
  self._show_history()