Skip to content

Commit

Permalink
docs: Make docstring compatible with pydocstyle
Browse files Browse the repository at this point in the history
  • Loading branch information
giffels committed Feb 27, 2024
1 parent 61531a6 commit 0fea500
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 26 deletions.
43 changes: 18 additions & 25 deletions reana_job_controller/compute4punch_job_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
# REANA is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""Compute4PUNCH Job Manager."""

import base64
import logging
import os
Expand All @@ -26,7 +28,7 @@


class Compute4PUNCHJobManager(JobManager):
"""Compute4PUNCH Job Manager"""
"""Compute4PUNCH Job Manager."""

C4P_WORKSPACE_PATH = ""
"""Absolute path on the Compute4PUNCH head node used for submission"""
Expand Down Expand Up @@ -169,6 +171,13 @@ def get_logs(cls, backend_job_id: str, **kwargs) -> str:
def get_outputs(cls, c4p_connection, src, dest) -> None:
"""
Transfer job outputs from Compute4PUNCH to local REANA workspace.
:param c4p_connection: SSH connection to Compute4PUNCH
:type c4p_connection: SSHClient
:param src: Source directory
:type src: str
:param dest: Destination directory
:type dest: str
"""
sftp_client = c4p_connection.ssh_client.open_sftp()
sftp_client.chdir(src)
Expand All @@ -191,9 +200,7 @@ def stop(self, backend_job_id: str) -> None:

@property
def c4p_home_path(self) -> str:
"""
Determine and return the Compute4PUNCH home directory on Compute4PUNCH
"""
"""Determine and return the Compute4PUNCH home directory on Compute4PUNCH."""
if not self.C4P_HOME_PATH:
# Since the JobMonitor entirely rely on class variables to get corresponding
# paths on Compute4PUNCH, the class variable C4P_HOME_PATH needs to be
Expand All @@ -205,9 +212,7 @@ def c4p_home_path(self) -> str:

@property
def c4p_abs_workspace_path(self) -> str:
"""
Determine and return the absolute Compute4PUNCH workspace path
"""
"""Determine and return the absolute Compute4PUNCH workspace path."""
if not self.C4P_WORKSPACE_PATH:
# Since the JobMonitor entirely rely on class variables to get corresponding
# paths on Compute4PUNCH, the class variable C4P_WORKSPACE_PATH needs to be
Expand All @@ -219,15 +224,11 @@ def c4p_abs_workspace_path(self) -> str:

@property
def c4p_rel_workspace_path(self) -> str:
"""
Determine and return the relative Compute4PUNCH workspace path
"""
"""Determine and return the relative Compute4PUNCH workspace path."""
return os.path.join("reana/workflows", self.workflow_uuid)

def _create_c4p_job_description(self) -> None:
"""
Create job description for Compute4PUNCH
"""
"""Create job description for Compute4PUNCH."""
job_inputs = ",".join(self._get_inputs())
job_outputs = "." # download everything from remote job
job_description_template = [
Expand Down Expand Up @@ -257,9 +258,7 @@ def _create_c4p_job_description(self) -> None:
)

def _create_c4p_job_execution_script(self) -> None:
"""
Create job execution script for Compute4PUNCH
"""
"""Create job execution script for Compute4PUNCH."""
# The workflow workspace does not exist on Compute4PUNCH,
# therefore replace it with CONDOR_JOB_IWD
self.cmd = self.cmd.replace(self.workflow_workspace, "$_CONDOR_JOB_IWD")
Expand All @@ -274,9 +273,7 @@ def _create_c4p_job_execution_script(self) -> None:
)

def _create_c4p_workspace_environment(self) -> None:
"""
Create workspace environment for REANA @ Compute4PUNCH
"""
"""Create workspace environment for REANA @ Compute4PUNCH."""
self.c4p_connection.exec_command(f"mkdir -p {self.c4p_abs_workspace_path}")
self.c4p_connection.exec_command(
os.path.join(
Expand Down Expand Up @@ -331,9 +328,7 @@ def _encode_cmd(cmd: str) -> str:
return f"echo {encoded_cmd} | base64 -d | bash"

def _get_inputs(self) -> Iterable:
"""
Collect all input files in the local REANA workspace.
"""
"""Collect all input files in the local REANA workspace."""
skipped_input_files = (".job.ad", ".machine.ad", ".chirp.config")
return filter(
lambda x: x not in skipped_input_files,
Expand All @@ -343,9 +338,7 @@ def _get_inputs(self) -> Iterable:
)

def _upload_job_inputs(self) -> None:
"""
Upload job inputs to Compute4PUNCH
"""
"""Upload job inputs to Compute4PUNCH."""
sftp_client = self.c4p_connection.ssh_client.open_sftp()
sftp_client.chdir(self.c4p_rel_workspace_path)

Expand Down
19 changes: 19 additions & 0 deletions reana_job_controller/job_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -576,6 +576,13 @@ def watch_jobs(self, job_db, app=None):


def query_c4p_jobs(*backend_job_ids: str, ssh_client: SSHClient):
"""
Query status information of backend jobs on Compute4PUNCH.
:param backend_job_ids: List of job ids to query on Compute4PUNCH
:type backend_job_ids: str
:param ssh_client: SSH client used to communicate with Compute4PUNCH
"""
attributes = ("JobStatus", "ClusterId", "ProcId", "ExitCode", "HoldReasonCode")
attributes_string = " ".join(attributes)

Expand Down Expand Up @@ -607,6 +614,18 @@ def query_c4p_jobs(*backend_job_ids: str, ssh_client: SSHClient):
def filter_jobs_to_watch(
id, job_db, compute_backend, statuses_to_skip=("finished", "failed", "stopped")
):
"""
Filter jobs to watch for job completion.
:param id: REANA job id
:type id: str
:param job_db: REANA job database
:type job_db: JOB_DB
:param compute_backend: REANA compute backend used
:type compute_backend: str
:param statuses_to_skip: REANA job statuses to skip
:type statuses_to_skip: tuple[str]
"""
return job_db[id]["compute_backend"] == compute_backend and not (
job_db[id]["deleted"] or job_db[id]["status"] in statuses_to_skip
)
Expand Down
11 changes: 10 additions & 1 deletion reana_job_controller/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def csv_parser(
skip_trailing_space: bool = False,
):
"""
Parses CSV formatted input
Parse CSV formatted input.
:param input_csv: CSV formatted input
:type input_csv: str
Expand Down Expand Up @@ -116,6 +116,15 @@ def csv_parser(


def motley_cue_auth_strategy_factory(hostname):
"""
Paramiko auth strategy factory that provides oauth based ssh token authentication.
This auth strategy has been developed against the motley cue implementation of
oauth based ssh token authentication on the server side.
:param hostname: hostname of the ssh node
:type hostname: str
"""
# Using a factory to avoid a general dependency on libmytoken, paramiko and pyjwt
from libmytoken import get_access_token_from_jwt_mytoken
from paramiko.auth_strategy import AuthSource
Expand Down

0 comments on commit 0fea500

Please sign in to comment.