From 404068260ad709dadadf4d9477ec49fb1bfb8786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andre=CC=81s=20Gonza=CC=81lez?= Date: Tue, 25 Oct 2022 15:00:04 -0300 Subject: [PATCH] feat: Add mongorestore additional options Add BACKUP_MONGORESTORE_ADDITIONAL_OPTIONS setting (defaults to none) to add options to the mongorestore command. By default, mongorestore will attempt to restore four collections concurrently. This causes the process to consume a lot of memory and potentially break the process in low memory environments, specially in K8s deployments. Set BACKUP_MONGORESTORE_ADDITIONAL_OPTIONS to '-j 1' to reduce memory requirements by restoring one collection at a time. Fixes: #54 --- CHANGELOG.md | 3 +++ README.md | 4 ++++ tutorbackup/patches/k8s-jobs | 2 ++ tutorbackup/patches/local-docker-compose-jobs-services | 1 + tutorbackup/plugin.py | 1 + tutorbackup/templates/backup/build/backup/restore_services.py | 1 + 6 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6163dd8..0d4bc49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## Unreleased +* [feature] Add the `BACKUP_MONGORESTORE_ADDITIONAL_OPTIONS` setting. + ## Version 1.2.0 (2022-09-21) * [feature] Add the `BACKUP_MONGODB_DATABASES` setting to select the diff --git a/README.md b/README.md index 210ce58..e901c7d 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,10 @@ service, some of these values may be required to set. * For a Ceph Object Gateway that doesn’t set [rgw_dns_name](https://docs.ceph.com/en/latest/radosgw/config-ref/#confval-rgw_dns_name), you will need `BACKUP_S3_ADDRESSING_STYLE: path`. +* When dealing with very large databases, restoring MongoDB may consume more memory than available and the process may break. + In this situation, you can set `BACKUP_MONGORESTORE_ADDITIONAL_OPTIONS` to `-j 1` to + [limit the number of collections](https://www.mongodb.com/docs/v4.2/reference/program/mongorestore/#cmdoption-mongorestore-numparallelcollections) + that are restored in parallel in order to save memory. ### Selecting databases to backup diff --git a/tutorbackup/patches/k8s-jobs b/tutorbackup/patches/k8s-jobs index 7928e3d..090783a 100644 --- a/tutorbackup/patches/k8s-jobs +++ b/tutorbackup/patches/k8s-jobs @@ -49,6 +49,8 @@ spec: {% endif %} - name: MONGODB_AUTHENTICATION_DATABASE value: '{{ BACKUP_MONGODB_AUTHENTICATION_DATABASE }}' + - name: MONGORESTORE_ADDITIONAL_OPTIONS + value: '{{ BACKUP_MONGORESTORE_ADDITIONAL_OPTIONS }}' - name: S3_SIGNATURE_VERSION value: '{{ BACKUP_S3_SIGNATURE_VERSION }}' - name: S3_ADDRESSING_STYLE diff --git a/tutorbackup/patches/local-docker-compose-jobs-services b/tutorbackup/patches/local-docker-compose-jobs-services index a9ea404..21e9139 100644 --- a/tutorbackup/patches/local-docker-compose-jobs-services +++ b/tutorbackup/patches/local-docker-compose-jobs-services @@ -19,6 +19,7 @@ backup-job: - MONGODB_PASSWORD={{ MONGODB_PASSWORD }} {% endif %} - MONGODB_AUTHENTICATION_DATABASE={{ BACKUP_MONGODB_AUTHENTICATION_DATABASE }} + - MONGORESTORE_ADDITIONAL_OPTIONS={{ BACKUP_MONGORESTORE_ADDITIONAL_OPTIONS }} volumes: - ../backup:/data/backup {% if ENABLE_HTTPS and ENABLE_WEB_PROXY %}- ../../data/caddy:/caddy{% endif %} diff --git a/tutorbackup/plugin.py b/tutorbackup/plugin.py index 98b5148..c769798 100644 --- a/tutorbackup/plugin.py +++ b/tutorbackup/plugin.py @@ -35,6 +35,7 @@ "MYSQL_DATABASES": [], "MONGODB_DATABASES": [], "MONGODB_AUTHENTICATION_DATABASE": "admin", + "MONGORESTORE_ADDITIONAL_OPTIONS": "", } } diff --git a/tutorbackup/templates/backup/build/backup/restore_services.py b/tutorbackup/templates/backup/build/backup/restore_services.py index badb34f..781a45a 100755 --- a/tutorbackup/templates/backup/build/backup/restore_services.py +++ b/tutorbackup/templates/backup/build/backup/restore_services.py @@ -66,6 +66,7 @@ def restore_mongodb(): cmd = ("mongorestore " "--stopOnError --drop " f"--host={host} --port={port} " + f"{ENV['MONGORESTORE_ADDITIONAL_OPTIONS']} " f"{dump_dir}" ) try: