[Feature Request] Only backup claimed chunks (Option)

[jusvit]

Server world's generally get really big and it would be nice to have some integration that would only backup the chunk MCA's that have been claimed by players so we could keep more backups, more frequently.

In the case that you need a backup, yes you would lose all the chunks which aren't claimed but it's a good compromise.

[jusvit]

I have a prototype which I got stumped on because there's no region-specific claiming foundations.

Here's one of my server's backup (6GB and growing):

The overworld (under /region) consisting of 532 regions (544,768) chunks, 300 chunks of which are actually required to be backed up (claimed by players).

Pretty nasty overall with raw file manipulating.

Basically we call this every file we "backup":

private static boolean shouldSkipFileBackup(File sourceDirectory, File evaluateFile) {

        // Only backup claimed chunks functionality
        if (ServerUtilitiesConfig.backups.only_backup_claimed_chunks){
            var evaluatePath = evaluateFile.getPath();

            // Only care about region file format (e.g. under regions)
            if (evaluatePath.endsWith(".mca")){

                // Extract region coordinates into 4 parts (2 specifically): r | RegionX | RegionZ | mca
                String fileName = evaluateFile.getName();
                int firstDot = fileName.indexOf('.');
                int secondDot = fileName.indexOf('.', firstDot + 1);

                int RegionX = Integer.parseInt(fileName.substring(firstDot + 1, secondDot));
                int RegionY = Integer.parseInt(fileName.substring(secondDot + 1, fileName.lastIndexOf('.')));


                // Extract dimension
                String relativePath = sourceDirectory.toPath().relativize(evaluateFile.toPath()).toString();

                int dimension = 0; // Default to overworld
                int dimIndex = relativePath.indexOf("DIM");
                if (dimIndex != -1) {
                    int endIndex = relativePath.indexOf(File.separator, dimIndex);
                    String dimString = relativePath.substring(dimIndex + 3, endIndex == -1 ? relativePath.length() : endIndex);
                    dimension = Integer.parseInt(dimString);
                }

                // We either need to check if a region contains a claimed chunk which we keep the entire region or go into the mca which is nasty...
                var chunkClaim = ClaimedChunks.instance.getChunk(new ChunkDimPos(chunkX, chunkZ, dimension));

                // If chunkClaim is null, we skip backing up this chunk.
                return chunkClaim == null;
            }
        }


        // This file will be backed up
        return false;
    }

[jusvit]

Also did a refactor of ThreadBackup.java but It didn't serve anything functionality-wise. It seems #100 already partially refactored it but could be further broken down.

Refactor

package serverutils.task.backup;

import static serverutils.ServerUtilitiesNotifications.BACKUP_END1;
import static serverutils.ServerUtilitiesNotifications.BACKUP_END2;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.zip.*;

import net.minecraft.util.EnumChatFormatting;
import net.minecraft.util.IChatComponent;

import serverutils.*;
import serverutils.data.ClaimedChunks;
import serverutils.lib.math.ChunkDimPos;
import serverutils.lib.util.*;

public class ThreadBackup extends Thread {
    private final File sourceDirectory;
    private final String customName;
    public volatile boolean isDone = false;

    public ThreadBackup(File sourceDirectory, String customName) {
        this.sourceDirectory = sourceDirectory;
        this.customName = customName;
        setPriority(Thread.NORM_PRIORITY + 2);
    }

    @Override
    public void run() {
        isDone = false;
        doBackup(sourceDirectory, customName);
        isDone = true;
    }

    public static void doBackup(File sourceDirectory, String customName) {
        String timestamp = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().getTime());
        String backupName = customName.isEmpty() ? timestamp : customName;
        File destinationFile = null;

        try {
            List<File> files = FileUtils.listTree(sourceDirectory);
            int totalFiles = files.size();

            ServerUtilities.LOGGER.info("Backing up {} files...", totalFiles);
            long startTime = System.currentTimeMillis();

            if (ServerUtilitiesConfig.backups.compression_level > 0) {
                destinationFile = compressBackup(sourceDirectory, backupName, files, totalFiles);
            } else {
                destinationFile = copyBackup(sourceDirectory, backupName, files, totalFiles);
            }

            ServerUtilities.LOGGER.info("Created {} from {}", destinationFile.getAbsolutePath(), sourceDirectory.getAbsolutePath());

            BackupTask.clearOldBackups();

            notifyBackupCompletion(startTime, destinationFile);

        } catch (Exception e) {
            handleBackupFailure(e, destinationFile);
        }
    }

    private static File compressBackup(File sourceDirectory, String backupName, List<File> files, int totalFiles) throws IOException {
        File destinationFile = FileUtils.newFile(new File(BackupTask.backupsFolder, backupName + ".zip"));

        try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(destinationFile))) {
            zos.setLevel(ServerUtilitiesConfig.backups.compression_level);
            byte[] buffer = new byte[4096];

            ServerUtilities.LOGGER.info("Compressing {} files!", totalFiles);

            for (int i = 0; i < totalFiles; i++) {
                File file = files.get(i);

                String filePath = file.getAbsolutePath();
                ZipEntry ze = new ZipEntry(sourceDirectory.getName() + File.separator +
                        filePath.substring(sourceDirectory.getAbsolutePath().length() + 1));

                logProgress(i, totalFiles, ze.getName());

                zos.putNextEntry(ze);
                try (FileInputStream fis = new FileInputStream(file)) {
                    int len;
                    while ((len = fis.read(buffer)) > 0) zos.write(buffer, 0, len);
                }
                zos.closeEntry();
            }
        }

        return destinationFile;
    }


    private static File copyBackup(File sourceDirectory, String backupName, List<File> files, int totalFiles) throws Exception {
        File destinationDirectory = new File(BackupTask.backupsFolder, backupName + File.separator + sourceDirectory.getName());
        destinationDirectory.mkdirs();

        String destPath = destinationDirectory.getAbsolutePath() + File.separator;
        String srcPath = sourceDirectory.getAbsolutePath();

        for (int i = 0; i < totalFiles; i++) {
            File file = files.get(i);

            logProgress(i, totalFiles, file.getName());

            File destFile = new File(destPath + (file.getAbsolutePath().replace(srcPath, "")));
            FileUtils.copyFile(file, destFile);
        }

        return destinationDirectory;
    }

    private static void logProgress(int current, int total, String fileName) {
        if (current == 0 || current == total - 1 || current % 100 == 0) {
            ServerUtilities.LOGGER.info("[{} | {}%]: {}",
                    current,
                    StringUtils.formatDouble00((current / (double) total) * 100D),
                    fileName);
        }
    }

    private static void notifyBackupCompletion(long startTime, File destinationFile) {
        String elapsedTime = StringUtils.getTimeString(System.currentTimeMillis() - startTime);
        if (ServerUtilitiesConfig.backups.display_file_size) {
            String backupSize = FileUtils.getSizeString(destinationFile);
            String totalSize = FileUtils.getSizeString(BackupTask.backupsFolder);
            ServerUtilitiesNotifications.backupNotification(
                    BACKUP_END2, "cmd.backup_end_2", elapsedTime,
                    backupSize.equals(totalSize) ? backupSize : (backupSize + " | " + totalSize));
        } else {
            ServerUtilitiesNotifications.backupNotification(BACKUP_END1, "cmd.backup_end_1", elapsedTime);
        }
    }

    private static void handleBackupFailure(Exception e, File destinationFile) {
        IChatComponent message = StringUtils.color(
                ServerUtilities.lang(null, "cmd.backup_fail", e.getClass().getName()),
                EnumChatFormatting.RED);
        ServerUtils.notifyChat(ServerUtils.getServer(), null, message);
        e.printStackTrace();
        if (destinationFile != null) FileUtils.delete(destinationFile);
    }
}

[Lyfts]

Minecraft already has a way to get an InputStream of a specific chunk from a region file, which is definitely the way to go.