Merge pull request #4868 from golemfactory/dump_docker_logs

Dump docker logs

golem/core/common.py

@@ -7,7 +7,8 @@
 from calendar import timegm
 from datetime import datetime
 from functools import wraps
-from typing import Any, Callable, cast, List, TypeVar
+from pathlib import Path
+from typing import Any, Callable, cast, List, TypeVar, Optional
 
 import pytz
 
@@ -238,6 +239,12 @@ def wrapper(*args, **kwargs):
     return decorator
 
 
+def get_log_dir(data_dir: Optional[str] = None) -> Path:
+    if data_dir is None:
+        data_dir = simpleenv.get_local_datadir("default")
+    return Path(data_dir) / 'logs'
+
+
 # pylint: disable=too-many-branches,too-many-locals
 def config_logging(
         suffix='',
@@ -252,9 +259,7 @@ def config_logging(
     except ImportError:
         from loggingconfig import LOGGING
 
-    if datadir is None:
-        datadir = simpleenv.get_local_datadir("default")
-    logdir_path = os.path.join(datadir, 'logs')
+    logdir_path = get_log_dir(datadir)
 
     for formatter in LOGGING.get('formatters', {}).values():
         formatter['format'] = f"{formatter_prefix}{formatter['format']}"
@@ -284,14 +289,11 @@ def config_logging(
             LOGGING['loggers']['twisted']['level'] = 'WARNING'
 
     try:
-        if not os.path.exists(logdir_path):
-            os.makedirs(logdir_path)
+        logdir_path.mkdir(parents=True, exist_ok=True)
 
         logging.config.dictConfig(LOGGING)
     except (ValueError, PermissionError) as e:
-        sys.stderr.write(
-            "Can't configure logging in: {} Got: {}\n".format(logdir_path, e)
-        )
+        sys.stderr.write(f"Can't configure logging in {logdir_path} Got: {e}\n")
         return  # Avoid consequent errors
     logging.captureWarnings(True)
 

golem/envs/__init__.py

@@ -20,6 +20,7 @@
 CounterUsage = Any
 
 EnvId = str
+RuntimeId = str
 
 
 class RuntimeEventType(Enum):
@@ -143,7 +144,10 @@ def __exit__(self, *_, **__) -> None:
         self.close()
 
 
-class RuntimeOutput(Iterable[Union[str, bytes]], ABC):
+RuntimeOutput = Iterable[Union[str, bytes]]
+
+
+class RuntimeOutputBase(RuntimeOutput, ABC):
     """ A handle for reading output (either stdout or stderr) from a running
         Runtime. Yielded items are output lines. Output could be either raw
         (bytes) or decoded (str). """
@@ -161,6 +165,12 @@ class Runtime(ABC):
     """ A runnable object representing some particular computation. Tied to a
         particular Environment that was used to create this object. """
 
+    @abstractmethod
+    def id(self) -> Optional[RuntimeId]:
+        """ Get unique identifier of this Runtime. Might not be available if the
+            Runtime is not yet prepared. """
+        raise NotImplementedError
+
     @abstractmethod
     def prepare(self) -> Deferred:
         """ Prepare the Runtime to be started. Assumes current status is
@@ -403,6 +413,7 @@ def supported(cls) -> EnvSupportStatus:
         """ Is the Environment supported on this machine? """
         raise NotImplementedError
 
+    @abstractmethod
     def status(self) -> EnvStatus:
         """ Get current status of the Environment. """
         raise NotImplementedError
@@ -423,9 +434,8 @@ def run_benchmark(self) -> Deferred:
         """ Get the general performance score for this environment. """
         raise NotImplementedError
 
-    @classmethod
     @abstractmethod
-    def parse_prerequisites(cls, prerequisites_dict: Dict[str, Any]) \
+    def parse_prerequisites(self, prerequisites_dict: Dict[str, Any]) \
             -> Prerequisites:
         """ Build Prerequisites struct from supplied dictionary. Returned value
             is of appropriate type for calling install_prerequisites(). """
@@ -438,9 +448,8 @@ def install_prerequisites(self, prerequisites: Prerequisites) -> Deferred:
             Returns boolean indicating whether installation was successful. """
         raise NotImplementedError
 
-    @classmethod
     @abstractmethod
-    def parse_config(cls, config_dict: Dict[str, Any]) -> EnvConfig:
+    def parse_config(self, config_dict: Dict[str, Any]) -> EnvConfig:
         """ Build config struct from supplied dictionary. Returned value
             is of appropriate type for calling update_config(). """
         raise NotImplementedError
@@ -455,6 +464,7 @@ def update_config(self, config: EnvConfig) -> None:
         """ Update configuration. Assumes current status is 'DISABLED'. """
         raise NotImplementedError
 
+    @abstractmethod
     def listen(
             self,
             event_type: EnvEventType,

golem/envs/docker/cpu.py

@@ -34,8 +34,10 @@
     EnvSupportStatus,
     Prerequisites,
     RuntimeBase,
+    RuntimeId,
     RuntimeInput,
     RuntimeOutput,
+    RuntimeOutputBase,
     RuntimePayload,
     RuntimeStatus,
     BenchmarkResult)
@@ -74,7 +76,7 @@ def from_dict(cls, data: Dict[str, Any]) -> 'DockerCPUConfig':
         return cls(work_dirs=work_dirs, **data)
 
 
-class DockerOutput(RuntimeOutput):
+class DockerOutput(RuntimeOutputBase):
 
     def __init__(
             self, raw_output: Iterable[bytes], encoding: Optional[str] = None
@@ -231,6 +233,9 @@ def _update_status_loop(self) -> None:
         self._logger.info("Runtime is no longer running. "
                           "Stopping status update thread.")
 
+    def id(self) -> Optional[RuntimeId]:
+        return self._container_id
+
     def prepare(self) -> Deferred:
         self._change_status(
             from_status=RuntimeStatus.CREATED,