Lock warning (#506)

allenai · Jan 18, 2023 · 8d29321 · 8d29321
1 parent 521de99
commit 8d29321
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - You can now add arguments to steps without invalidating the cache. See `Step.SKIP_DEFAULT_ARGUMENTS`.
 - Fixed integration status messages in `tango info` command.
 
+### Fixed
+
+- Warnings about locks are now reliably printed every 30 seconds
+
 
 ## [v1.1.0](https://github.com/allenai/tango/releases/tag/v1.1.0) - 2022-12-01
 

diff --git a/tango/integrations/beaker/common.py b/tango/integrations/beaker/common.py
@@ -168,15 +168,16 @@ def acquire(self, timeout=None, poll_interval: float = 2.0, log_interval: float
                     self._beaker.dataset.delete(self._lock_dataset_name)
                     continue
 
-                if last_logged is None or last_logged - start >= log_interval:
+                now = time.monotonic()
+                if last_logged is None or now - last_logged >= log_interval:
                     logger.warning(
                         "Waiting to acquire lock dataset for step '%s':\n\n%s\n\n"
                         "This probably means the step is being run elsewhere, but if you're sure it isn't "
                         "you can just delete the lock dataset.",
                         self._step_id,
                         self.lock_dataset_url,
                     )
-                    last_logged = time.monotonic()
+                    last_logged = now
                 time.sleep(poll_interval)
                 continue
             else: