diff --git a/group_vars/gearshift_cluster/vars.yml b/group_vars/gearshift_cluster/vars.yml index 860fd2ada..53d74de23 100644 --- a/group_vars/gearshift_cluster/vars.yml +++ b/group_vars/gearshift_cluster/vars.yml @@ -55,6 +55,19 @@ ldap_domains: user_expiration_date: loginExpirationTime group_member: memberUid group_object_class: groupofnames + # + # LDAP is case insensitive, but we use lowercase only for quota field names, + # so we can use simple literal strings in comparisons as opposed to regexes + # to handle differences in UPPERCASE vs. lowercase. + # + # The UPPERCASE "LFS" in {{ ldap_domains[domain]['group_quota_*_limit_template'] }} Ansible variables + # is a required placeholder that will get replaced with the value of the Logical File System (LFS) + # for which we will try to fetch quota limits from the LDAP. + # E.g. with + # group_quota_soft_limit_template: 'ruggroupumcgquotaLFSsoft' + # the fieldname/key used to lookup the soft quota limit for the prm01 LFS is + # ruggroupumcgquotaprm01soft + # group_quota_soft_limit_template: ruggroupumcgquotaLFSsoft group_quota_hard_limit_template: ruggroupumcgquotaLFS network_public_external_id: vlan16 @@ -457,6 +470,8 @@ lfs_mounts: - lfs: prm03 pfs: umcgst02 quota_type: 'group' + #quota_type: 'project' + #quota_pid_increment: 200000 # Value added to GID to create unique PID for a group on the PFS hosting this LFS. groups: - name: umcg-aad - name: umcg-as diff --git a/group_vars/nibbler_cluster/vars.yml b/group_vars/nibbler_cluster/vars.yml index f2a7226c3..4e5fe3e92 100644 --- a/group_vars/nibbler_cluster/vars.yml +++ b/group_vars/nibbler_cluster/vars.yml @@ -82,6 +82,19 @@ ldap_domains: user_expiration_date: loginExpirationTime group_member: memberUid group_object_class: groupofnames + # + # LDAP is case insensitive, but we use lowercase only for quota field names, + # so we can use simple literal strings in comparisons as opposed to regexes + # to handle differences in UPPERCASE vs. lowercase. + # + # The UPPERCASE "LFS" in {{ ldap_domains[domain]['group_quota_*_limit_template'] }} Ansible variables + # is a required placeholder that will get replaced with the value of the Logical File System (LFS) + # for which we will try to fetch quota limits from the LDAP. + # E.g. with + # group_quota_soft_limit_template: 'ruggroupumcgquotaLFSsoft' + # the fieldname/key used to lookup the soft quota limit for the prm01 LFS is + # ruggroupumcgquotaprm01soft + # group_quota_soft_limit_template: ruggroupumcgquotaLFSsoft group_quota_hard_limit_template: ruggroupumcgquotaLFS pam_weblogin: diff --git a/group_vars/talos_cluster/vars.yml b/group_vars/talos_cluster/vars.yml index 6c0ce8366..38a11dd1a 100644 --- a/group_vars/talos_cluster/vars.yml +++ b/group_vars/talos_cluster/vars.yml @@ -59,6 +59,19 @@ ldap_domains: user_expiration_date: loginExpirationTime group_member: memberUid group_object_class: groupofnames + # + # LDAP is case insensitive, but we use lowercase only for quota field names, + # so we can use simple literal strings in comparisons as opposed to regexes + # to handle differences in UPPERCASE vs. lowercase. + # + # The UPPERCASE "LFS" in {{ ldap_domains[domain]['group_quota_*_limit_template'] }} Ansible variables + # is a required placeholder that will get replaced with the value of the Logical File System (LFS) + # for which we will try to fetch quota limits from the LDAP. + # E.g. with + # group_quota_soft_limit_template: 'ruggroupumcgquotaLFSsoft' + # the fieldname/key used to lookup the soft quota limit for the prm01 LFS is + # ruggroupumcgquotaprm01soft + # group_quota_soft_limit_template: ruggroupumcgquotaLFSsoft group_quota_hard_limit_template: ruggroupumcgquotaLFS network_public_external_id: vlan16 diff --git a/roles/shared_storage/defaults/main.yml b/roles/shared_storage/defaults/main.yml index 2d8068ee8..5cb673860 100644 --- a/roles/shared_storage/defaults/main.yml +++ b/roles/shared_storage/defaults/main.yml @@ -1,22 +1,5 @@ --- # -# Defaults for names of keys used to fetch quota limits for groups from an LDAP. -# -# LDAP is case insensitive, but we use lowercase only for field names, -# so we can use simple literal strings in comparisons as opposed to regexes -# to handle differences in UPPERCASE vs. lowercase. -# -# The UPPERCASE "LFS" in {{ ldap_group_quota_*_limit_template }} Ansible variables -# is a required placeholder that will get replaced with the value of the Logical File System (LFS) -# for which we will try to fetch quota limits from the LDAP. -# E.g. with ldap_group_quota_soft_limit_template='ruggroupumcgquotaLFSsoft', -# the fieldname/key to lookup the soft quota limit for the LFS prm01 is -# ruggroupumcgquotaprm01soft -# -ldap_group_object_class: 'groupofnames' -ldap_group_quota_soft_limit_template: 'ruggroupumcgquotaLFSsoft' -ldap_group_quota_hard_limit_template: 'ruggroupumcgquotaLFS' -# # Lustre 'project' quota is preferred, but older Lustre versions may not support it; # Fall-back is 'group' quota, which will work with older versions too. # Setting is global for all Lustre file systems mounted on a cluster. diff --git a/roles/shared_storage/templates/ldapquota.bash b/roles/shared_storage/templates/ldapquota.bash index 689ae3313..d9b9b59e4 100755 --- a/roles/shared_storage/templates/ldapquota.bash +++ b/roles/shared_storage/templates/ldapquota.bash @@ -26,28 +26,12 @@ set -o pipefail # Fail when any command in series of piped commands failed as op umask 0077 -# -# LDAP is case insensitive, but we use lowercase only for field names, -# so we can use simple literal strings in comparisons as opposed to regexes -# to handle differences in UPPERCASE vs. lowercase. -# -# The UPPERCASE "LFS" in ${ldap_group_quota_*_limit_template} variables -# is a required placeholder that will get replaced with the value of the Logical File System (LFS) -# for which we will try to fetch quota limits from the LDAP. -# E.g. with ldap_group_quota_soft_limit_template='ruggroupumcgquotaLFSsoft', -# the fieldname/key to lookup the soft quota limit for the LFS prm01 is -# ruggroupumcgquotaprm01soft -# -declare ldap_group_object_class='{{ ldap_group_object_class }}' -declare ldap_group_quota_soft_limit_template='{{ ldap_group_quota_soft_limit_template }}' -declare ldap_group_quota_hard_limit_template='{{ ldap_group_quota_hard_limit_template }}' -declare -A ldap_quota_limits=() - # # Quota settings for groups: # For Lustre file systems we prefer "project quota" for group folders, # but we'll use "group quota" when project quota are not supported (yet). # +declare -A ldap_quota_limits=() declare -A quota_types=( {% for lfs_item in lfs_mounts | selectattr('lfs', 'search', '((tmp)|(rsc)|(prm)|(dat))[0-9]+$') %} {% if lfs_item['quota_type'] is defined %} @@ -76,7 +60,7 @@ export TMPDIR export SCRIPT_NAME declare mixed_stdouterr='' # global variable to capture output from commands for reporting in custom log messages. declare ldif_dir="${TMPDIR}/ldifs" -declare config_file='/etc/ssh/ldap.conf' +declare ldap_config_file='/etc/openldap/readonly-ldapsearch-credentials.bash' # # Initialise Log4Bash logging with defaults. @@ -409,115 +393,118 @@ function getQuotaFromLDAP () { local _group="${2}" local _ldap_attr_regex='([^: ]{1,})(:{1,2}) ([^:]{1,})' local _ldif_file="${ldif_dir}/${_group}.ldif" - local _ldif_record - local -a _ldif_records - local _ldap_group_quota_soft_limit_key="${ldap_group_quota_soft_limit_template/LFS/${_lfs}}" - local _ldap_group_quota_hard_limit_key="${ldap_group_quota_hard_limit_template/LFS/${_lfs}}" + local _ldap # # Query LDAP # - log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "Retrieving data from LDAP..." - ldapsearch -LLL -o ldif-wrap=no \ - -H "${LDAP_HOST}" \ - -D "${LDAP_USER}" \ - -w "${LDAP_PASS}" \ - -b "${LDAP_SEARCH_BASE}" \ - "(&(ObjectClass=${ldap_group_object_class})(cn:dn:=${_group}))" \ - "${_ldap_group_quota_soft_limit_key}" \ - "${_ldap_group_quota_hard_limit_key}" \ - 2>&1 >"${_ldif_file}" \ - || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "ldapsearch failed." - # - # Parse query results. - # - while IFS= read -r -d '' _ldif_record; do - _ldif_records+=("${_ldif_record}") - done < <(sed 's/^$/\x0/' "${_ldif_file}") \ - || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "Parsing LDIF file (${_ldif_file}) into records failed." - # - # Loop over records in the array and create a faked-multi-dimensional hash. - # - for _ldif_record in "${_ldif_records[@]}"; do + for _ldap in "${domain_names[@]}"; do + local _uri="${domain_configs[${_ldap}_uri]}" + local _search_base="${domain_configs[${_ldap}_search_base]}" + local _bind_dn="${domain_configs[${_ldap}_bind_dn]}" + local _bind_pw="${domain_configs[${_ldap}_bind_pw]}" + local _group_object_class="${domain_configs[${_ldap}_group_object_class]}" + local _group_quota_soft_limit_template="${domain_configs[${_ldap}_group_quota_soft_limit_template]}" + local _group_quota_hard_limit_template="${domain_configs[${_ldap}_group_quota_hard_limit_template]}" + local _group_quota_soft_limit_key="${_group_quota_soft_limit_template/LFS/${_lfs}}" + local _group_quota_hard_limit_key="${_group_quota_hard_limit_template/LFS/${_lfs}}" + ldapsearch -LLL -o ldif-wrap=no \ + -H "${_uri}" \ + -D "${_bind_dn}" \ + -w "${_bind_pw}" \ + -b "${_search_base}" \ + "(&(ObjectClass=${_group_object_class})(cn:dn:=${_group}))" \ + "${_group_quota_soft_limit_key}" \ + "${_group_quota_hard_limit_key}" \ + 2>&1 >"${_ldif_file}" \ + || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "ldapsearch failed." # - # Remove trailing white space like the new line character. - # And skip blank lines. + # Parse query results. # - _ldif_record="${_ldif_record%%[[:space:]]}" - [[ "${_ldif_record}" == '' ]] && continue - log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "LDIF record contains: ${_ldif_record}" + local _ldif_record + local -a _ldif_records + while IFS= read -r -d '' _ldif_record; do + _ldif_records+=("${_ldif_record}") + done < <(sed 's/^$/\x0/' "${_ldif_file}") \ + || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "Parsing LDIF file (${_ldif_file}) into records failed." # - # Parse record's key:value pairs. + # Loop over records in the array and create a faked-multi-dimensional hash. # - local -A _directory_record_attributes=() - local _ldif_line - while IFS=$'\n' read -r _ldif_line; do - [[ "${_ldif_line}" == '' ]] && continue # Skip blank lines. - log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' "LDIF key:value pair contains: ${_ldif_line}." - if [[ "${_ldif_line}" =~ ${_ldap_attr_regex} ]]; then - local _key="${BASH_REMATCH[1],,}" # Convert key on-the-fly to lowercase. - local _sep="${BASH_REMATCH[2]}" - local _value="${BASH_REMATCH[3]}" - # - # Check if value was base64 encoded (double colon as separator) - # or plain text (single colon as separator) and decode if necessary. - # - if [[ "${_sep}" == '::' ]]; then - log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' " decoding base64 encoded value..." - _value="$(printf '%s' "${_value}" | base64 -di)" + for _ldif_record in "${_ldif_records[@]}"; do + # + # Remove trailing white space like the new line character. + # And skip blank lines. + # + _ldif_record="${_ldif_record%%[[:space:]]}" + [[ "${_ldif_record}" == '' ]] && continue + log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "LDIF record contains: ${_ldif_record}" + # + # Parse record's key:value pairs. + # + local -A _directory_record_attributes=() + local _ldif_line + while IFS=$'\n' read -r _ldif_line; do + [[ "${_ldif_line}" == '' ]] && continue # Skip blank lines. + log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' "LDIF key:value pair contains: ${_ldif_line}." + if [[ "${_ldif_line}" =~ ${_ldap_attr_regex} ]]; then + local _key="${BASH_REMATCH[1],,}" # Convert key on-the-fly to lowercase. + local _sep="${BASH_REMATCH[2]}" + local _value="${BASH_REMATCH[3]}" + # + # Check if value was base64 encoded (double colon as separator) + # or plain text (single colon as separator) and decode if necessary. + # + if [[ "${_sep}" == '::' ]]; then + log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' " decoding base64 encoded value..." + _value="$(printf '%s' "${_value}" | base64 -di)" + fi + # + # This may be a multi-valued attribute and therefore check if key already exists; + # When key already exists make sure we append instead of overwriting the existing value(s)! + # + if [[ -n "${_directory_record_attributes[${_key}]+isset}" ]]; then + _directory_record_attributes["${_key}"]="${_directory_record_attributes["${_key}"]} ${_value}" + else + _directory_record_attributes["${_key}"]="${_value}" + fi + log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' " key contains: ${_key}." + log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' " value contains: ${_value}." + else + log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" '1' "Failed to parse LDIF key:value pair (${_ldif_line})." fi + done < <(printf '%s\n' "${_ldif_record}") || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "Parsing LDIF record failed." + # + # Get Quota from processed LDIF record if this the right group. + # + local _ldap_group + if [[ -n "${_directory_record_attributes['dn']+isset}" ]]; then # - # This may be a multi-valued attribute and therefore check if key already exists; - # When key already exists make sure we append instead of overwriting the existing value(s)! + # Parse cn from dn. # - if [[ -n "${_directory_record_attributes[${_key}]+isset}" ]]; then - _directory_record_attributes["${_key}"]="${_directory_record_attributes["${_key}"]} ${_value}" - else - _directory_record_attributes["${_key}"]="${_value}" - fi - log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' " key contains: ${_key}." - log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' " value contains: ${_value}." + _ldap_group=$(dn2cn "${_directory_record_attributes['dn']}") + log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "Found group ${_ldap_group} in dn attribute." else - log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" '1' "Failed to parse LDIF key:value pair (${_ldif_line})." + log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" '1' "dn attribute missing for ${_ldif_record}" + fi + if [[ "${_ldap_group}" == "${_group}" ]]; then + log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "Group from ldap record matches the group we were looking for: ${_ldap_group}." + else + log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "Skipping LDAP group ${_ldap_group} that does not match the LFS group ${_group} we were looking for." + continue fi - done < <(printf '%s\n' "${_ldif_record}") || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "Parsing LDIF record failed." - # - # Get Quota from processed LDIF record if this the right group. - # - local _ldap_group - if [[ -n "${_directory_record_attributes['dn']+isset}" ]]; then # - # Parse cn from dn. + # Get quota values for this group on this LFS. # - _ldap_group=$(dn2cn "${_directory_record_attributes['dn']}") - log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "Found group ${_ldap_group} in dn attribute." - else - log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" '1' "dn attribute missing for ${_ldif_record}" - fi - if [[ "${_ldap_group}" == "${_group}" ]]; then - log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "Group from ldap record matches the group we were looking for: ${_ldap_group}." - else - log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "Skipping LDAP group ${_ldap_group} that does not match the LFS group ${_group} we were looking for." - continue - fi - # - # Get quota values for this group on this LFS. - # - if [[ -n "${_directory_record_attributes["${_ldap_group_quota_soft_limit_key}"]+isset}" && \ - -n "${_directory_record_attributes["${_ldap_group_quota_hard_limit_key}"]+isset}" ]]; then - ldap_quota_limits['soft']="${_directory_record_attributes["${_ldap_group_quota_soft_limit_key}"]}" - ldap_quota_limits['hard']="${_directory_record_attributes["${_ldap_group_quota_hard_limit_key}"]}" - return - else - log4Bash 'WARN' "${LINENO}" "${FUNCNAME:-main}" '0' " Quota values missing for group ${_ldap_group} on LFS ${_lfs}." - log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' " Search keys were ${_ldap_group_quota_soft_limit_key} and ${_ldap_group_quota_hard_limit_key}." - if [[ "${_lfs}" == 'rsc01' ]]; then - log4Bash 'WARN' "${LINENO}" "${FUNCNAME:-main}" '0' " Temporary hack: use hard coded values for LFS rsc01 for all groups." - ldap_quota_limits['soft']='1024' - ldap_quota_limits['hard']='2048' + if [[ -n "${_directory_record_attributes["${_group_quota_soft_limit_key}"]+isset}" && \ + -n "${_directory_record_attributes["${_group_quota_hard_limit_key}"]+isset}" ]]; then + ldap_quota_limits['soft']="${_directory_record_attributes["${_group_quota_soft_limit_key}"]}" + ldap_quota_limits['hard']="${_directory_record_attributes["${_group_quota_hard_limit_key}"]}" return + else + log4Bash 'WARN' "${LINENO}" "${FUNCNAME:-main}" '0' " Quota values missing for group ${_ldap_group} on LFS ${_lfs}." + log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' " Search keys were ${_group_quota_soft_limit_key} and ${_group_quota_hard_limit_key}." + continue fi - continue - fi + done done } @@ -571,21 +558,14 @@ while getopts ":l:ah" opt; do done # -# Get credentials from config file. -# -# LDAP_USER='some_account' -# LDAP_PASS='some_passwd' -# LDAP_SEARCH_BASE='ou=groups,ou=some_org_unit,o=some_org' +# Parse LDAP config file. # -if [[ -r "${config_file}" && -f "${config_file}" ]]; then - log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "Fetching ldapsearch credentials from config file ${config_file} ..." - LDAP_HOST="$(awk '$1 == "uri" {print $2}' "${config_file}")" || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "Failed to parse LDAP_HOST from ${config_file}." - LDAP_USER="$(awk '$1 == "binddn" {print $2}' "${config_file}")" || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "Failed to parse LDAP_USER from ${config_file}." - LDAP_PASS="$(awk '$1 == "bindpw" {print $2}' "${config_file}")" || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "Failed to parse LDAP_PASS from ${config_file}." - LDAP_SEARCH_BASE="$(awk '$1 == "base" {print $2}' "${config_file}")" || log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" "${?}" "Failed to parse LDAP_SEARCH_BASE from ${config_file}." - log4Bash 'TRACE' "${LINENO}" "${FUNCNAME:-main}" '0' "Using LDAP credentials: LDAP_HOST=${LDAP_HOST} | LDAP_USER=${LDAP_USER} | LDAP_PASS=${LDAP_PASS} | LDAP_SEARCH_BASE=${LDAP_SEARCH_BASE}" +if [[ -e "${ldap_config_file}" && -r "${ldap_config_file}" ]]; then + log4Bash 'DEBUG' "${LINENO}" "${FUNCNAME:-main}" '0' "Fetching ldapsearch credentials from config file ${ldap_config_file} ..." + # shellcheck source=/dev/null + source "${ldap_config_file}" else - log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" '1' "Config file ${config_file} missing or not accessible." + log4Bash 'FATAL' "${LINENO}" "${FUNCNAME:-main}" '1' "Config file ${ldap_config_file} missing or not readable." fi if [ "${apply_settings}" -eq 1 ]; then diff --git a/roles/sssd/templates/readonly-ldapsearch-credentials.bash b/roles/sssd/templates/readonly-ldapsearch-credentials.bash index 0ef03d697..59b0eeead 100644 --- a/roles/sssd/templates/readonly-ldapsearch-credentials.bash +++ b/roles/sssd/templates/readonly-ldapsearch-credentials.bash @@ -16,5 +16,7 @@ declare -A domain_configs=( [{{ ldap_domain }}_user_expiration_date]='{{ ldap_config['user_expiration_date'] | default('loginExpirationTime') }}' [{{ ldap_domain }}_user_expiration_regex]='{{ ldap_config['user_expiration_regex'] | default('^([0-9]{4})([0-9]{2})([0-9]{2}).+Z$') }}' [{{ ldap_domain }}_group_object_class]='{{ ldap_config['group_object_class'] | default('posixGroup') }}' + [{{ ldap_domain }}_group_quota_soft_limit_template]='{{ ldap_config['group_quota_soft_limit_template'] | default('quotaLFSsoft') }}' + [{{ ldap_domain }}_group_quota_hard_limit_template]='{{ ldap_config['group_quota_hard_limit_template'] | default('quotaLFShard') }}' {% endfor %} ) \ No newline at end of file