Feature/ben/expose visibility timeout

CHANGELOG.md

@@ -16,7 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - The default rabbitmq vhost is now <user> instead of /<user>.
-- Changed default task timeout from 2 hours to 24 hours.
+- Changed default visibility timeout from 2 hours to 24 hours. Exposed this in the config
+  file.
 - The monitor function will now check the queues to determine when
   to exit.
 

merlin/celery.py

@@ -47,6 +47,7 @@
     broker,
     results_backend,
 )
+from merlin.config.configfile import CONFIG
 from merlin.log_formatter import FORMATS
 from merlin.router import route_for_task
 
@@ -100,10 +101,10 @@
     redis_max_connections=100000,
 )
 
-# Set a 24-hour timeout to acknowledge a task before it's available to grab
-# again.
+# Set a timeout to acknowledge a task before it's available to grab
+# again (default 24 hours).
 app.conf.broker_transport_options = {
-    "visibility_timeout": 86400,
+    "visibility_timeout": CONFIG.celery.visibility_timeout,
     "max_connections": 100,
 }
 

merlin/data/celery/app.yaml

@@ -1,3 +1,6 @@
+celery:
+    visibility_timeout: 86400
+
 broker:
     # can be redis, redis+sock, or rabbitmq
     name: rabbitmq

merlin/data/celery/app_redis.yaml

@@ -1,3 +1,6 @@
+celery:
+    visibility_timeout: 86400
+
 broker:
     # can be redis, redis+sock, or rabbitmq
     name: redis

merlin/examples/workflows/feature_demo/scripts/learn.py

@@ -2,7 +2,6 @@
 import sys
 
 import numpy as np
-
 from sklearn.ensemble import RandomForestRegressor
 
 

merlin/examples/workflows/hello/make_samples.py

@@ -1,8 +1,7 @@
 import argparse
 
-import numpy as np
-
 import names
+import numpy as np
 
 
 # argument parsing

merlin/examples/workflows/openfoam_wf/scripts/combine_outputs.py

@@ -2,7 +2,6 @@
 import glob
 
 import numpy as np
-
 import Ofpp
 
 

merlin/examples/workflows/openfoam_wf/scripts/learn.py

@@ -2,7 +2,6 @@
 
 import matplotlib.pyplot as plt
 import numpy as np
-
 from joblib import dump
 from sklearn.ensemble import RandomForestRegressor
 from sklearn.metrics import mean_squared_error

merlin/examples/workflows/openfoam_wf_no_docker/scripts/combine_outputs.py

@@ -2,7 +2,6 @@
 import glob
 
 import numpy as np
-
 import Ofpp
 
 

merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py

@@ -2,7 +2,6 @@
 
 import matplotlib.pyplot as plt
 import numpy as np
-
 from joblib import dump
 from sklearn.ensemble import RandomForestRegressor
 from sklearn.metrics import mean_squared_error

merlin/study/study.py

@@ -383,7 +383,7 @@ def expanded_spec(self):
             # rewrite provenance spec to correct samples.generate.cmd and samples.file
             if self.restart_dir is None:
                 self.write_expanded_spec(self.expanded_filepath)
-        
+
         LOG.info(f"Study workspace is '{self.workspace}'.")
         return result