From 040ca74fe7bd81f7c90c95a23ffca5c6c2b786df Mon Sep 17 00:00:00 2001 From: pneerincx Date: Wed, 24 Nov 2021 17:10:26 +0100 Subject: [PATCH 1/9] Small fix for determining SCRIPT_NAME. --- roles/shared_storage/templates/set_isilon_quota.zsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/shared_storage/templates/set_isilon_quota.zsh b/roles/shared_storage/templates/set_isilon_quota.zsh index 8660bbbe2..2bde40845 100755 --- a/roles/shared_storage/templates/set_isilon_quota.zsh +++ b/roles/shared_storage/templates/set_isilon_quota.zsh @@ -10,7 +10,7 @@ declare pfs_base_path='/ifs/rekencluster' declare pfs_name_prefix='umcgst' declare TMPDIR="${TMPDIR:-/tmp}" # Default to /tmp if ${TMPDIR} was not defined. declare SCRIPT_NAME -SCRIPT_NAME="$(basename "${0}" '.bash')" +SCRIPT_NAME="$(basename "${0}" '.zsh')" export TMPDIR export SCRIPT_NAME declare mixed_stdouterr='' # global variable to capture output from commands for reporting in custom log messages. From e3ce20ee839adcbba4ca0d0afb23f30a6c8c895f Mon Sep 17 00:00:00 2001 From: pneerincx Date: Wed, 24 Nov 2021 17:11:43 +0100 Subject: [PATCH 2/9] Added example /etc/local/crontab.local for applying/updating quota settings on an Isilon node. --- roles/shared_storage/files/crontab.local | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 roles/shared_storage/files/crontab.local diff --git a/roles/shared_storage/files/crontab.local b/roles/shared_storage/files/crontab.local new file mode 100644 index 000000000..720705ba4 --- /dev/null +++ b/roles/shared_storage/files/crontab.local @@ -0,0 +1,14 @@ +# +# Quota management for our Isilon file systems +# +# This contab.local file must be installed on one of the Isilon nodes in +# /etc/local/crontab.local +# Important notes: +# * This file may get lost when Isilon OneFS is updated, +# so check after each Isilon update and reploy is necessary! +# * This means the cronjob will run on a single node and there is no automagic failover; +# If the nodes with the crontab.smbtime fails, the job will not run. +# * There is also a /etc/mcp/override/crontab.smbtime, that will survive Isilon updates, +# but adding a cron entry there, means it will run on each Isilon node, which is not what we want. +# +30 9,11,13,15,17,19,21 * * * root /bin/zsh /ifs/scripts/set_isilon_quota.zsh -a \ No newline at end of file From d15fb4be22bedb99528899c40548b4deddd3da0d Mon Sep 17 00:00:00 2001 From: pneerincx Date: Thu, 25 Nov 2021 09:40:27 +0100 Subject: [PATCH 3/9] Added missing newline character to crontab. --- roles/shared_storage/files/crontab.local | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/shared_storage/files/crontab.local b/roles/shared_storage/files/crontab.local index 720705ba4..b709bed7b 100644 --- a/roles/shared_storage/files/crontab.local +++ b/roles/shared_storage/files/crontab.local @@ -10,5 +10,7 @@ # If the nodes with the crontab.smbtime fails, the job will not run. # * There is also a /etc/mcp/override/crontab.smbtime, that will survive Isilon updates, # but adding a cron entry there, means it will run on each Isilon node, which is not what we want. +# * We currently do not have access from our Ansible control hosts to the Isilon backend, +# so this crontab.local cannot be deployed with Ansible: deploy manually instead! # -30 9,11,13,15,17,19,21 * * * root /bin/zsh /ifs/scripts/set_isilon_quota.zsh -a \ No newline at end of file +30 9,11,13,15,17,19,21 * * * root /bin/zsh /ifs/scripts/set_isilon_quota.zsh -a From 8d99e216400d94401a99ca5f1bf730c26ae47348 Mon Sep 17 00:00:00 2001 From: pneerincx Date: Wed, 24 Nov 2021 17:10:26 +0100 Subject: [PATCH 4/9] Small fix for determining SCRIPT_NAME. --- roles/shared_storage/templates/set_isilon_quota.zsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/shared_storage/templates/set_isilon_quota.zsh b/roles/shared_storage/templates/set_isilon_quota.zsh index 8660bbbe2..2bde40845 100755 --- a/roles/shared_storage/templates/set_isilon_quota.zsh +++ b/roles/shared_storage/templates/set_isilon_quota.zsh @@ -10,7 +10,7 @@ declare pfs_base_path='/ifs/rekencluster' declare pfs_name_prefix='umcgst' declare TMPDIR="${TMPDIR:-/tmp}" # Default to /tmp if ${TMPDIR} was not defined. declare SCRIPT_NAME -SCRIPT_NAME="$(basename "${0}" '.bash')" +SCRIPT_NAME="$(basename "${0}" '.zsh')" export TMPDIR export SCRIPT_NAME declare mixed_stdouterr='' # global variable to capture output from commands for reporting in custom log messages. From c89986b0b1e4949126d994c47f3179fae55ef982 Mon Sep 17 00:00:00 2001 From: pneerincx Date: Wed, 24 Nov 2021 17:11:43 +0100 Subject: [PATCH 5/9] Added example /etc/local/crontab.local for applying/updating quota settings on an Isilon node. --- roles/shared_storage/files/crontab.local | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 roles/shared_storage/files/crontab.local diff --git a/roles/shared_storage/files/crontab.local b/roles/shared_storage/files/crontab.local new file mode 100644 index 000000000..720705ba4 --- /dev/null +++ b/roles/shared_storage/files/crontab.local @@ -0,0 +1,14 @@ +# +# Quota management for our Isilon file systems +# +# This contab.local file must be installed on one of the Isilon nodes in +# /etc/local/crontab.local +# Important notes: +# * This file may get lost when Isilon OneFS is updated, +# so check after each Isilon update and reploy is necessary! +# * This means the cronjob will run on a single node and there is no automagic failover; +# If the nodes with the crontab.smbtime fails, the job will not run. +# * There is also a /etc/mcp/override/crontab.smbtime, that will survive Isilon updates, +# but adding a cron entry there, means it will run on each Isilon node, which is not what we want. +# +30 9,11,13,15,17,19,21 * * * root /bin/zsh /ifs/scripts/set_isilon_quota.zsh -a \ No newline at end of file From 1c391b536de8407210ce2d9804ae58ad5ae088d3 Mon Sep 17 00:00:00 2001 From: pneerincx Date: Thu, 25 Nov 2021 09:40:27 +0100 Subject: [PATCH 6/9] Added missing newline character to crontab. --- roles/shared_storage/files/crontab.local | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/shared_storage/files/crontab.local b/roles/shared_storage/files/crontab.local index 720705ba4..b709bed7b 100644 --- a/roles/shared_storage/files/crontab.local +++ b/roles/shared_storage/files/crontab.local @@ -10,5 +10,7 @@ # If the nodes with the crontab.smbtime fails, the job will not run. # * There is also a /etc/mcp/override/crontab.smbtime, that will survive Isilon updates, # but adding a cron entry there, means it will run on each Isilon node, which is not what we want. +# * We currently do not have access from our Ansible control hosts to the Isilon backend, +# so this crontab.local cannot be deployed with Ansible: deploy manually instead! # -30 9,11,13,15,17,19,21 * * * root /bin/zsh /ifs/scripts/set_isilon_quota.zsh -a \ No newline at end of file +30 9,11,13,15,17,19,21 * * * root /bin/zsh /ifs/scripts/set_isilon_quota.zsh -a From 6d9bb2ec43ef1aaffb2a5c9c7b15ed1a0ed93f5e Mon Sep 17 00:00:00 2001 From: pneerincx Date: Mon, 29 Nov 2021 17:14:16 +0100 Subject: [PATCH 7/9] Improved crontabs for managing quota on Isilon. --- roles/shared_storage/files/crontab.local | 5 +++-- roles/shared_storage/files/crontab.smbtime | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 roles/shared_storage/files/crontab.smbtime diff --git a/roles/shared_storage/files/crontab.local b/roles/shared_storage/files/crontab.local index b709bed7b..7013bb882 100644 --- a/roles/shared_storage/files/crontab.local +++ b/roles/shared_storage/files/crontab.local @@ -5,7 +5,7 @@ # /etc/local/crontab.local # Important notes: # * This file may get lost when Isilon OneFS is updated, -# so check after each Isilon update and reploy is necessary! +# so check after each Isilon update and redeploy if necessary! # * This means the cronjob will run on a single node and there is no automagic failover; # If the nodes with the crontab.smbtime fails, the job will not run. # * There is also a /etc/mcp/override/crontab.smbtime, that will survive Isilon updates, @@ -13,4 +13,5 @@ # * We currently do not have access from our Ansible control hosts to the Isilon backend, # so this crontab.local cannot be deployed with Ansible: deploy manually instead! # -30 9,11,13,15,17,19,21 * * * root /bin/zsh /ifs/scripts/set_isilon_quota.zsh -a +#min hour mday month wday who command +30 * * * * root zsh /ifs/scripts/set_isilon_quota.zsh -a diff --git a/roles/shared_storage/files/crontab.smbtime b/roles/shared_storage/files/crontab.smbtime new file mode 100644 index 000000000..849a6666c --- /dev/null +++ b/roles/shared_storage/files/crontab.smbtime @@ -0,0 +1,14 @@ +# +# Quota management for our Isilon file systems +# +# This contab.smbtime file must be installed on one of the Isilon nodes in +# /etc/mcp/override/crontab.smbtime +# Important notes: +# * This file should survive an Isilon OneFS update, +# but check after each Isilon update to make sure and redeploy if necessary! +# * We use the isi_ropc command to make sure the script will Run Once Per Cluster. +# * We currently do not have access from our Ansible control hosts to the Isilon backend, +# so this crontab.smbtime cannot be deployed with Ansible: deploy manually instead! +# +#min hour mday month wday who command +30 * * * * root isi_ropc -s zsh /ifs/scripts/set_isilon_quota.zsh -a From a5b14563e50f874f13d1022197c452453fe2daf7 Mon Sep 17 00:00:00 2001 From: pneerincx Date: Wed, 1 Dec 2021 15:04:26 +0100 Subject: [PATCH 8/9] Use isi_ropc for redundancy and added code to send quota reports by email. --- roles/shared_storage/files/crontab.smbtime | 1 + roles/shared_storage/files/send_email.py | 65 +++++++++++++++++++ .../send_isilon_quota_report_wrapper.zsh | 23 +++++++ 3 files changed, 89 insertions(+) create mode 100644 roles/shared_storage/files/send_email.py create mode 100644 roles/shared_storage/files/send_isilon_quota_report_wrapper.zsh diff --git a/roles/shared_storage/files/crontab.smbtime b/roles/shared_storage/files/crontab.smbtime index 849a6666c..359c4b32e 100644 --- a/roles/shared_storage/files/crontab.smbtime +++ b/roles/shared_storage/files/crontab.smbtime @@ -12,3 +12,4 @@ # #min hour mday month wday who command 30 * * * * root isi_ropc -s zsh /ifs/scripts/set_isilon_quota.zsh -a +10 * * * * root isi_ropc -s zsh /ifs/scripts/send_isilon_quota_report_wrapper.zsh \ No newline at end of file diff --git a/roles/shared_storage/files/send_email.py b/roles/shared_storage/files/send_email.py new file mode 100644 index 000000000..0a6e1b836 --- /dev/null +++ b/roles/shared_storage/files/send_email.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# +# Helper script to send mail using the Isilon libraries. +# Original from: https://groups.google.com/g/isilon-user-group/c/4XM0_hmHTr0/m/sRos52H2HAAJ +# + +import sys +from optparse import OptionParser +import socket +from isi.app.lib.emailer import Emailer, EmailAttachmentFromFile + +# Emailer.send_email(to_addresses(list), message(string), from_address=None(string), subject=None(string), +# attachments=None(list), headers=None(list), charset="us-ascii"(string)) + +def main(): + usage = '%prog: [-f sender] -t recipient [ -t recipient ... ] [-s subject] [-b body] [-a attachment]' + argparser = OptionParser(usage = usage, description = 'Send email from a cluser node') + argparser.add_option('-f', '--from', '--sender', dest='sender', + help="email sender (From:)") + argparser.add_option('-t', '--to', '--recipients', dest='recipients', + action = 'append', help="email recipient (To:)") + argparser.add_option('-s', '--subject', dest='subject', + help="email subject (Subject:)") + argparser.add_option('-b', '--body', dest='body', + help="email body (default stdin)") + argparser.add_option('-a', '--attachment', '--file', dest='attfiles', + action = 'append', help="attachment filename") + (options, args) = argparser.parse_args() + if options.sender is None: + fqdn = socket.getfqdn() + sender = "root@%s" % fqdn + else: + sender = options.sender + if options.recipients is None: + argparser.error("Unable to send mail without at least one recipient"); + sys.exit(1); + else: + recipients = options.recipients + if options.subject is None: + subject = 'No subject specified' + else: + subject = options.subject + if options.body is None: + lines = sys.stdin.readlines() + body = ''.join(lines) + else: + body = options.body + if options.attfiles is None: + atts = None + else: + atts = [] + for attfile in options.attfiles: + att = EmailAttachmentFromFile(attfile) + atts.append(att) + try: + Emailer.send_email(recipients, body, sender, subject, attachments = atts) + except: + print('Error sending email.') + sys.exit(1) + + sys.exit(0) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/roles/shared_storage/files/send_isilon_quota_report_wrapper.zsh b/roles/shared_storage/files/send_isilon_quota_report_wrapper.zsh new file mode 100644 index 000000000..524a03450 --- /dev/null +++ b/roles/shared_storage/files/send_isilon_quota_report_wrapper.zsh @@ -0,0 +1,23 @@ +#!/bin/zsh +# +# Wrapper script to: +# * Generate a quota report with "isi quota quotas list" command and +# * Send the generated report by email using a custom sendmail.py, +# which is required as Isilon systems are Debian based, +# but lack a regular Linux command line mail client. +# + +declare -a physical_file_systems +physical_file_systems=( + 'umcgst10' # On UMCG Research Isilon +) + +for pfs in "${physical_file_systems}"; do + isi quota quotas list \ + --recurse-path-children \ + --path "/ifs/rekencluster/${pfs}/groups/" \ + | python /ifs/scripts/send_email.py \ + -f "cron@$(hostname)" \ + -t hpc.helpdesk@umcg.nl \ + -s "HPC report: Quota vs. disk usage for file system ${pfs}." +done From 1fe3c030a7267c8342d52ee4d891970444524b10 Mon Sep 17 00:00:00 2001 From: pneerincx Date: Wed, 1 Dec 2021 15:06:38 +0100 Subject: [PATCH 9/9] Use isi_ropc for redundancy and added code to send quota reports by email. --- roles/shared_storage/files/crontab.local | 17 ----------------- roles/shared_storage/files/crontab.smbtime | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 roles/shared_storage/files/crontab.local diff --git a/roles/shared_storage/files/crontab.local b/roles/shared_storage/files/crontab.local deleted file mode 100644 index 7013bb882..000000000 --- a/roles/shared_storage/files/crontab.local +++ /dev/null @@ -1,17 +0,0 @@ -# -# Quota management for our Isilon file systems -# -# This contab.local file must be installed on one of the Isilon nodes in -# /etc/local/crontab.local -# Important notes: -# * This file may get lost when Isilon OneFS is updated, -# so check after each Isilon update and redeploy if necessary! -# * This means the cronjob will run on a single node and there is no automagic failover; -# If the nodes with the crontab.smbtime fails, the job will not run. -# * There is also a /etc/mcp/override/crontab.smbtime, that will survive Isilon updates, -# but adding a cron entry there, means it will run on each Isilon node, which is not what we want. -# * We currently do not have access from our Ansible control hosts to the Isilon backend, -# so this crontab.local cannot be deployed with Ansible: deploy manually instead! -# -#min hour mday month wday who command -30 * * * * root zsh /ifs/scripts/set_isilon_quota.zsh -a diff --git a/roles/shared_storage/files/crontab.smbtime b/roles/shared_storage/files/crontab.smbtime index 359c4b32e..b1156aad5 100644 --- a/roles/shared_storage/files/crontab.smbtime +++ b/roles/shared_storage/files/crontab.smbtime @@ -12,4 +12,4 @@ # #min hour mday month wday who command 30 * * * * root isi_ropc -s zsh /ifs/scripts/set_isilon_quota.zsh -a -10 * * * * root isi_ropc -s zsh /ifs/scripts/send_isilon_quota_report_wrapper.zsh \ No newline at end of file +45 5 * * 0 root isi_ropc -s zsh /ifs/scripts/send_isilon_quota_report_wrapper.zsh