diff --git a/group_vars/copperfist_cluster/ip_addresses.yml b/group_vars/copperfist_cluster/ip_addresses.yml new file mode 100644 index 000000000..d2b5fc8d3 --- /dev/null +++ b/group_vars/copperfist_cluster/ip_addresses.yml @@ -0,0 +1,10 @@ +--- +ip_addresses: + copperfist: + vlan983: + address: 172.23.41.226 + netmask: /23 + vlan13: + address: 129.125.55.14 + netmask: /24 +... diff --git a/group_vars/copperfist_cluster/secrets.yml b/group_vars/copperfist_cluster/secrets.yml new file mode 100644 index 000000000..01fbb6fa4 --- /dev/null +++ b/group_vars/copperfist_cluster/secrets.yml @@ -0,0 +1,66 @@ +$ANSIBLE_VAULT;1.2;AES256;copperfist_cluster +63366232376339363063366130646337633837346332616337666164653637383439623532363839 +3037393935643166333564626262303838613739636261300a386465346631306565306364373333 +31376666663634613265353465316230646434653733306463323665643034366637643562623931 +3535653135653839340a336435383133303937613933323039663139393639353265366431356161 +66353563316235653837356430356439666261633066383862663838373434313336346639346139 +63333433373535636536383438383761303566623335316332303665356564623331653638333766 +31346461346561666338643032613034653932343863626234373539323036616463356562396232 +33636266626137383166636632633236366235663139336462333135656562386337376332313336 +61396638313935323235636635343865646232303833313435366331366134383732323431363338 +61306330366666353130316162643232306336383833366339353534363463626432623462373363 +64333236316532346532633835646137653838663038626164303938383236323030646232326666 +62613435383137396262633336343039383366336364376666386637393765653538323766663536 +31373632333161386635646639383131633865346532333764343564666631656464363964353031 +31323865303863356563623531303836383838333233323063633763306566633039346665636237 +64643463356436306132393639313565633461363933316461313762356366633363613361306435 +35626263646163633166613537643937333731343634396264383235366530323334366363323064 +63643033663461636437616265666539333766383038353563663439636131343431653434353464 +63303362386336663566303631333733373337303065383066363466653138653064353231623261 +33313531623061316166353431663135643331653065626335383432346236306462623935346236 +61383533323833626264363030306333633630316339373039373264306337333132333163323335 +64636436326634646366363233323163356161633063663539313163656631626339353731353562 +30386465386636383563313461653832653965633263333133323531326634376662373662656336 +61663530653166663062383332616163613464613461336431336331343231633433323266323935 +65653662373539653964666636656533663764613666656665663633653931316562616465333534 +63623336626231306130393332666636313339636233643039323561373931333735646139323864 +37333037346239623532646438303234303364356330313664326132383739643562343239353865 +38396536626634613366393030373530646361376135613038643539323635663461623134646562 +31326536376363383639376430633135616430383736353036383533373838626263303338356165 +33303235346266636364336139333064333964663564626336643832663061616635656464656262 +35626538613033666664626364663335646430373738666265636333663063323164643039323663 +31373931656233323530646538353635323338613835653766323332663431383464636532643539 +37363439303566366266653931313936303661623534386465363230653337356338363861306539 +64333130653565373261313738333763323737303561333365393935616164386632333731333331 +32653238633932343734313233313231383934303036353164643164346433666630363663353361 +37656635623839383865616166326361316165646137386661363762393131373937616331643835 +61373231336137616532623338363832636538376138343139663361393839643264633730623365 +35306530383531323562623063346162636633663962333333626435353833326332393734393965 +64346261333466636136343730313637383466633830656134633636356230373036323632303665 +62376633366364633032326133333336373430363434393038633364353831633535363661626432 +66386164363263613337666132346634616532356635373464656539623461326636303937373534 +65353636363733333462393734366163363966653633333963646265663132326662623139393832 +31653336373664613965373934643132663732396562396561383932616332313464333165303961 +37383235363636313063306432363661343139366530663035663131336266306564333865653136 +30353536353238376430653734373164323234326234353863656433316165333466313037643632 +63616162303431363963303439663766366538386538313234306666363932343836303266643639 +37336263313330626632363765323661363762376162366239336335636436303466626136333436 +62326465656363303830633065343236306366336663653463393062386539383737346234653964 +32633261336337663137623966373064313838646533383935653739636465376664363236636432 +64663032383532646262316430373335383361386530353936656561386263343265616432313939 +32363861323034303866373430373761613330653135623438666233326663643132653162636466 +38626262363736663561313039633736386338396562623831333131373735633261393535343230 +39656431316666663338623866303134663536643961353431343135373461666332633266333638 +65376335353136336434656436353738356532646531663161613461616235353437393138396635 +37643165356431633635393565366164323363626662646334313431303331383838303434373330 +61316263383836373237336266643065653130383763343434373363646664663330336435613534 +31666230613933313666616664386433373265643833373033623034386634393837643766666337 +39386434633038636461633235373530653564346566636665306435333130626434343063313132 +30303236663334383265663536376666316536313830326661626661313331383134356138336366 +66386536373337636466303831613331623338653365353662613935643963616562343239343564 +61656334356662376363633237643930323130613832336461373839346234633362626230623139 +32633532343365353063316433386339653137663431373037313637643061633064333333353530 +63313763643133313265383862623534323431313039663831326465306564643263396361313563 +30396136643430643935346334313462353535363136356338343030376633306139643465333838 +39663730393238653235313433656661386632653464643539633265343032353933343439626438 +36316534396364343563313466316533383537356632383137326361343764333735 diff --git a/group_vars/copperfist_cluster/vars.yml b/group_vars/copperfist_cluster/vars.yml new file mode 100644 index 000000000..1ca7b64e4 --- /dev/null +++ b/group_vars/copperfist_cluster/vars.yml @@ -0,0 +1,272 @@ +--- +slurm_cluster_name: 'copperfist' +stack_domain: '' # Only add hpc.rug.nl domain when jumphost is registered in DNS. +stack_name: "{{ slurm_cluster_name }}_cluster" # stack_name must match the name of the folder that contains this vars.yml file. +stack_prefix: 'cf' +slurm_version: '20.11.8-1.el7.umcg' +slurm_partitions: + - name: regular # Must be in sync with group listed in Ansible inventory. + default: yes + nodes: copperfist # Must be in sync with Ansible hostnames listed in inventory. + max_nodes_per_job: "{% if slurm_allow_jobs_to_span_nodes is defined and slurm_allow_jobs_to_span_nodes is true %}{{ groups['regular']|list|length }}{% else %}1{% endif %}" + max_cores_per_node: "{{ groups['regular'] | map('extract', hostvars, 'slurm_max_cpus_per_node') | first }}" + max_mem_per_node: "{{ groups['regular'] | map('extract', hostvars, 'slurm_max_mem_per_node') | first }}" + local_disk: "{{ groups['regular'] | map('extract', hostvars, 'slurm_local_disk') | first | default(0, true) }}" + features: "{{ groups['regular'] | map('extract', hostvars, 'slurm_features') | first | default('none') }}" + extra_options: 'TRESBillingWeights="CPU=1.0,Mem=0.25G"' +repo_manager: 'none' +figlet_font: 'ogre' +motd: | + ========================================================= + Welcome to {{ slurm_cluster_name | capitalize }} + ========================================================= +additional_etc_hosts: + - group: docs_library + nodes: + - name: docs_on_merlin + network: vlan16 +use_ldap: yes +create_ldap: no +use_sssd: yes +ldap_domains: + default_domain: + uri: ldaps://172.23.40.249 + search_base: ou=gd,o=asds + schema: rfc2307 + min_id: 50100000 + max_id: 55999999 + user_object_class: posixAccount + user_name: uid + user_ssh_public_key: sshPublicKey + user_member_of: groupMembership + group_member: memberUid + group_object_class: groupofnames + group_quota_soft_limit_template: ruggroupumcgquotaLFSsoft + group_quota_hard_limit_template: ruggroupumcgquotaLFS +ssh_host_signer_hostnames: "{{ ansible_fqdn }},{{ ansible_hostname }},{{ inventory_hostname }}" +totp: + machines: "{{ groups['jumphost'] }}" + excluded: + - 'LOCAL' + - "{{ all.ip_addresses['umcg']['net1']['address'] }}{{ all.ip_addresses['umcg']['net1']['netmask'] }}" + - "{{ all.ip_addresses['umcg']['net2']['address'] }}{{ all.ip_addresses['umcg']['net2']['netmask'] }}" + - "{{ all.ip_addresses['umcg']['net3']['address'] }}{{ all.ip_addresses['umcg']['net3']['netmask'] }}" + - "{{ all.ip_addresses['umcg']['net4']['address'] }}{{ all.ip_addresses['umcg']['net4']['netmask'] }}" +nameservers: [ + '8.8.4.4', # Google DNS. + '8.8.8.8', # Google DNS. +] +network_private_management_id: "vlan983" +network_private_management_cidr: "172.23.41.226/23" +#network_private_storage_id: "{{ stack_prefix }}_internal_storage" +#network_private_storage_cidr: "10.10.2.0/24" + +iptables_allow_icmp_inbound: + - "{{ all.ip_addresses['umcg']['net1'] }}" + - "{{ all.ip_addresses['umcg']['net2'] }}" + - "{{ all.ip_addresses['umcg']['net3'] }}" + - "{{ all.ip_addresses['umcg']['net4'] }}" + - "{{ all.ip_addresses['rug']['bwp_net'] }}" + - "{{ all.ip_addresses['rug']['operator'] }}" + - "{{ all.ip_addresses['gcc']['cloud_net'] }}" + - "{{ wingedhelix_cluster.ip_addresses['porch']['vlan16'] }}" +iptables_allow_ssh_inbound: + - "{{ all.ip_addresses['umcg']['net1'] }}" + - "{{ all.ip_addresses['umcg']['net2'] }}" + - "{{ all.ip_addresses['umcg']['net3'] }}" + - "{{ all.ip_addresses['umcg']['net4'] }}" + - "{{ wingedhelix_cluster.ip_addresses['porch']['vlan16'] }}" +iptables_allow_ssh_outbound: + - "{{ wingedhelix_cluster.ip_addresses['porch']['vlan16'] }}" + +local_backups: # list of folders for cron to make daily backup + - name: apps # don't modify after once deployed! + src_path: '/apps' + frequency: + - { name: 'daily', hour: '5', minute: '47', day: '*', weekday: '*', month: '*', keep: '60', disabled: 'false' } + +local_admin_groups: + - 'admin' + - 'docker' +local_admin_users: + - 'egon' + - 'ger' + - 'gerben' + - 'henkjan' + - 'kim' + - 'marieke' + - 'marloes' + - 'morris' + - 'pieter' + - 'robin' + - 'sandi' + - 'wim' +data_transfer_only_group: 'umcg-sftp-only' +envsync_user: 'umcg-envsync' +envsync_group: 'umcg-depad' +functional_admin_group: 'umcg-funad' +hpc_env_prefix: '/apps' +regular_groups: + - "{{ envsync_group }}" + - "{{ functional_admin_group }}" + - 'umcg-atd' + - 'umcg-gap' + - 'umcg-gd' + - 'umcg-genomescan' + - 'umcg-gsad' + - 'umcg-gst' + - 'umcg-vipt' +regular_users: + - user: "{{ envsync_user }}" + groups: ["{{ envsync_group }}"] + - user: 'umcg-atd-ateambot' + groups: ['umcg-atd'] + sudoers: '%umcg-atd' + - user: 'umcg-atd-dm' + groups: ['umcg-atd'] + sudoers: '%umcg-atd' + - user: 'umcg-gap-ateambot' + groups: ['umcg-gap'] + sudoers: '%umcg-gap' + - user: 'umcg-gap-dm' + groups: ['umcg-gap'] + sudoers: '%umcg-gap' + - user: 'umcg-gd-ateambot' + groups: ['umcg-gd'] + sudoers: '%umcg-gd' + - user: 'umcg-gd-dm' + groups: ['umcg-gd'] + sudoers: '%umcg-gd' + - user: 'umcg-genomescan-ateambot' + groups: ['umcg-genomescan'] + sudoers: '%umcg-genomescan' + - user: 'umcg-genomescan-dm' + groups: ['umcg-genomescan'] + sudoers: '%umcg-genomescan' + - user: 'umcg-gsad-ateambot' + groups: ['umcg-gsad'] + sudoers: '%umcg-gsad' + - user: 'umcg-gsad-dm' + groups: ['umcg-gsad'] + sudoers: '%umcg-gsad' + - user: 'umcg-gst-ateambot' + groups: ['umcg-gst'] + sudoers: '%umcg-gst' + - user: 'umcg-gst-dm' + groups: ['umcg-gst'] + sudoers: '%umcg-gst' + - user: 'umcg-vipt-dm' + groups: ['umcg-vipt'] + sudoers: '%umcg-vipt' +# +# Shared storage related variables +# +pfs_mounts: + - pfs: local_raid + device: /data # needs to be already mounted on system (f.e. /dev/sdc1 > /data) + source: '/mnt' + type: 'none' + rw_options: 'bind' + ro_options: 'bind,ro' + machines: "{{ groups['sys_admin_interface'] }}" + - pfs: 'medgen_zincfinger$' + source: '//storage3.umcg.nl' + type: cifs # checked with cat /proc/filesystem + rw_options: 'vers=3.0,mfsymlinks,rw,soft,perm,dir_mode=02750,file_mode=0640' + ro_options: 'vers=3.0,mfsymlinks,ro,soft,perm,dir_mode=02750,file_mode=0640' + machines: "{{ groups['chaperone'] }}" + - pfs: 'medgen_leucinezipper$' + source: '//storage3.umcg.nl' + type: cifs # checked with cat /proc/filesystem + rw_options: 'vers=3.0,mfsymlinks,rw,soft,perm,dir_mode=02750,file_mode=0640' + ro_options: 'vers=3.0,mfsymlinks,ro,soft,perm,dir_mode=02750,file_mode=0640' + machines: "{{ groups['chaperone'] }}" + - pfs: 'medgen_wingedhelix$' + source: '//storage3.umcg.nl' + type: cifs # checked with cat /proc/filesystem + rw_options: 'vers=3.0,mfsymlinks,rw,soft,perm,dir_mode=02750,file_mode=0640' + ro_options: 'vers=3.0,mfsymlinks,ro,soft,perm,dir_mode=02750,file_mode=0640' + machines: "{{ groups['chaperone'] }}" +lfs_mounts: + - lfs: home + pfs: local_raid + rw_machines: "{{ groups['cluster'] }}" + - lfs: tmp06 + pfs: local_raid + groups: + - name: umcg-atd + - name: umcg-gap + - name: umcg-gd + - name: umcg-genomescan + - name: umcg-gsad + - name: umcg-gst + - name: umcg-vipt + rw_machines: "{{ groups['user_interface'] + groups['compute_vm'] }}" + - lfs: prm05 + pfs: 'medgen_zincfinger$' + groups: + - name: umcg-atd + - name: umcg-gap + - name: umcg-gd + - name: umcg-gsad + - name: umcg-gst + - name: umcg-vipt + rw_machines: "{{ groups['chaperone'] }}" + - lfs: dat05 + pfs: 'medgen_zincfinger$' + groups: + - name: umcg-atd + - name: umcg-gap + - name: umcg-gd + - name: umcg-genomescan + - name: umcg-gsad + - name: umcg-gst + - name: umcg-vipt + rw_machines: "{{ groups['chaperone'] }}" + - lfs: prm06 + pfs: 'medgen_leucinezipper$' + groups: + - name: umcg-atd + - name: umcg-gap + - name: umcg-gd + - name: umcg-gsad + - name: umcg-gst + - name: umcg-vipt + rw_machines: "{{ groups['chaperone'] }}" + - lfs: dat06 + pfs: 'medgen_leucinezipper$' + groups: + - name: umcg-atd + - name: umcg-gap + - name: umcg-gd + - name: umcg-genomescan + - name: umcg-gsad + - name: umcg-gst + - name: umcg-vipt + rw_machines: "{{ groups['chaperone'] }}" + - lfs: prm07 + pfs: 'medgen_wingedhelix$' + groups: + - name: umcg-atd + - name: umcg-gap + - name: umcg-gd + - name: umcg-gsad + - name: umcg-gst + - name: umcg-vipt + rw_machines: "{{ groups['chaperone'] }}" + - lfs: dat07 + pfs: 'medgen_wingedhelix$' + groups: + - name: umcg-atd + - name: umcg-gap + - name: umcg-gd + - name: umcg-genomescan + - name: umcg-gsad + - name: umcg-gst + - name: umcg-vipt + rw_machines: "{{ groups['chaperone'] }}" + - lfs: env06 + pfs: local_raid + ro_machines: "{{ groups['compute_vm'] + groups['user_interface'] }}" + rw_machines: "{{ groups['deploy_admin_interface'] }}" +... diff --git a/ssh-host-ca/copperfist-ca b/ssh-host-ca/copperfist-ca new file mode 100644 index 000000000..5b46e8f6f --- /dev/null +++ b/ssh-host-ca/copperfist-ca @@ -0,0 +1,29 @@ +$ANSIBLE_VAULT;1.2;AES256;copperfist_cluster +37653436303838383934366361383837313138393461663733663633346166366561613361623165 +6237363335336163363437303337613439616163643465650a633533646164653339383137356532 +63653736646364393531303934626137643962626462613839623035643263396462313962383339 +3532643837303036660a343332383864393937356236353539396232323131623034613535643765 +39633665393337363531313330653038643665326530656435373738636636376462333435336263 +31313431663465346233363363383566373766333037626439656464663463303762643431336636 +39396461386665353338323034366136353839393231626361373666633564656338393636323430 +65313163306332323464346539396362303138636231653837366630363433316137356331313938 +62353238386430373534326336303134313938396566343265303330313035653161323635643535 +63653561393066323566346334343138383036393664393563323765326637656637383436393264 +35353163653732303933313631663638396239646230326339313639316163346462643338633361 +39663234656461356664383964663365636138636432353834396238366663633430666330363738 +35666439323762633661623665616233633935356335363839383832346237353635623035656661 +66303965393764353833643139323934323430356330323436643936613937646463333665333764 +32613431333530303433623230663637323432396362356231323836363535336137303338373334 +65393630323330643433323234363138636632353566396664623361343336353233613664316437 +39386137323035386437323533393038396634636635343765663333643564623332633835303066 +39613330636662386330313939336266316435623333643466393937333136303136656665313836 +66383461643465356139363063326637343839373837373732393038333362363762636531353362 +34663238626330316136656233313263646439396462666532653931373762363031393536303034 +37386562363063663837393935386332373666306537356237356532363131633339656161343831 +66386132323963386339353563383631636332313538613430353162303861663532393066343465 +31633838643934373863316632383862383838343236363737353830306331643065653233633861 +63633737366531363836346136626363353132346234323233323830633234363233396130306533 +38663866313234356363666266326463346133666561623539643937383564316362313739396462 +64383132633266323161316566343630356265353334336233333033336463333933313835343434 +38333864333164363364313230353834353035633532346433633064346562376562356438333466 +38313637396337633665 diff --git a/ssh-host-ca/copperfist-ca.pub b/ssh-host-ca/copperfist-ca.pub new file mode 100644 index 000000000..2f2bbebfb --- /dev/null +++ b/ssh-host-ca/copperfist-ca.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOgd3uhXGZ1cYE+/EcT3Gd4AsY6rnX/zr0IYcod7vj+c CA key for Copperfist diff --git a/static_inventories/copperfist_cluster.yml b/static_inventories/copperfist_cluster.yml new file mode 100644 index 000000000..b96592a72 --- /dev/null +++ b/static_inventories/copperfist_cluster.yml @@ -0,0 +1,72 @@ +--- +all: + children: + openstack_api: + hosts: + localhost: + docs: + hosts: + docs_on_merlin: + sys_admin_interface: + hosts: + copperfist: + local_yum_repository: true # enable local yum repository + deploy_admin_interface: + hosts: + copperfist: + volumes: + - mount_point: '/apps' + device: '/mnt/env06/apps/' + mounted_owner: root + mounted_group: "{{ envsync_group }}" + mounted_mode: '2775' + mount_options: 'bind' + type: none + user_interface: + hosts: + copperfist: + slurm_sockets: 2 + slurm_cores_per_socket: 128 + slurm_real_memory: 515456 + slurm_local_disk: 0 + slurm_features: 'tmp06' # copperfist will replace Zinc-Finger + slurm_ethernet_interfaces: + - eth0 + - eth1 + compute_vm: + children: + regular: # Must be item from {{ slurm_partitions }} variable defined in group_vars/{{ stack_name }}/vars.yml + hosts: + copperfist: + vars: + slurm_sockets: 2 + slurm_cores_per_socket: 128 + slurm_real_memory: 515456 + slurm_max_cpus_per_node: "{{ slurm_sockets * slurm_cores_per_socket - 8 }}" + slurm_max_mem_per_node: "{{ slurm_real_memory - 8 * 2048 }}" + slurm_local_disk: 0 + slurm_features: 'tmp06' + slurm_ethernet_interfaces: + - eth0 + - eth1 + chaperone: + hosts: + cf-chaperone: + #ansible_host: uozkh[0-9].zkh.umcg.intra +administration: + children: + sys_admin_interface: + deploy_admin_interface: + user_interface: +cluster: + children: + compute_vm: + administration: +copperfist_cluster: + children: + openstack_api: + jumphost: + cluster: + chaperone: + docs: +...