From 2ed76d31b1601151d017b7d9d0912ba7d4e2ea63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Lidwin?= Date: Tue, 10 Sep 2024 17:21:52 +0200 Subject: [PATCH] [SAVES] Full save sync (automatic files deleting) (#25) * [SAVES] fully sync saves, by also deleting them * fix: support for dont sync marker * improv: use stdout for timestamps only --- gogdl/saves.py | 49 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/gogdl/saves.py b/gogdl/saves.py index e116d28..4aa313d 100644 --- a/gogdl/saves.py +++ b/gogdl/saves.py @@ -56,7 +56,8 @@ def __init__(self, api_handler, authorization_manager): self.logger = logging.getLogger("SAVES") self.session.headers.update( - {"User-Agent": "GOGGalaxyCommunicationService/2.0.4.164 (Windows_32bit)"} + {"User-Agent": "GOGGalaxyCommunicationService/2.0.13.27 (Windows_32bit) dont_sync_marker/true installation_source/gog", + "X-Object-Meta-User-Agent": "GOGGalaxyCommunicationService/2.0.13.27 (Windows_32bit) dont_sync_marker/true installation_source/gog"} ) self.credentials = dict() @@ -78,7 +79,8 @@ def create_directory_map(self, path: str) -> list: files.append(abs_path) return files - def get_relative_path(self, root: str, path: str) -> str: + @staticmethod + def get_relative_path(root: str, path: str) -> str: if not root.endswith("/") and not root.endswith("\\"): root = root + os.sep return path.replace(root, "") @@ -110,6 +112,7 @@ def sync(self, arguments, unknown_args): self.get_auth_token() cloud_files = self.get_cloud_files_list() + downloadable_cloud = [f for f in cloud_files if f.md5 != "aadd86936a80ee8a369579c3926f1b3c"] if len(local_files) > 0 and len(cloud_files) == 0: action = SyncAction.UPLOAD @@ -117,20 +120,23 @@ def sync(self, arguments, unknown_args): for f in local_files: self.upload_file(f) self.logger.info("Done") + sys.stdout.write(str(datetime.datetime.now().timestamp())) + sys.stdout.flush() return elif len(local_files) == 0 and len(cloud_files) > 0: self.logger.info("No files locally, downloading") action = SyncAction.DOWNLOAD - for f in cloud_files: + for f in downloadable_cloud: self.download_file(f) self.logger.info("Done") + sys.stdout.write(str(datetime.datetime.now().timestamp())) + sys.stdout.flush() return timestamp = float(arguments.timestamp) classifier = SyncClassifier.classify(local_files, cloud_files, timestamp) action = classifier.get_action() - # print(action) if prefered_action: if prefered_action == "forceupload": @@ -139,7 +145,7 @@ def sync(self, arguments, unknown_args): action = SyncAction.UPLOAD elif prefered_action == "forcedownload": self.logger.warning("Forcing download") - classifier.updated_cloud = cloud_files + classifier.updated_cloud = downloadable_cloud action = SyncAction.DOWNLOAD if prefered_action == "upload" and action == SyncAction.DOWNLOAD: self.logger.warning("Refused to upload files, newer files in the cloud") @@ -150,15 +156,21 @@ def sync(self, arguments, unknown_args): print(self.arguments.timestamp) return - # return if action == SyncAction.UPLOAD: self.logger.info("Uploading files") for f in classifier.updated_local: self.upload_file(f) + for f in classifier.not_existing_locally: + self.logger.info(f"DELETING IN CLOUD {f}") + self.delete_file(f) elif action == SyncAction.DOWNLOAD: self.logger.info("Downloading files") for f in classifier.updated_cloud: self.download_file(f) + for f in classifier.not_existing_remotely: + self.logger.info(f"DELETING LOCALLY {f.absolute_path}") + os.remove(f.absolute_path) + elif action == SyncAction.CONFLICT: self.logger.warning( "Files in conflict force downloading or uploading of files" @@ -200,8 +212,10 @@ def get_cloud_files_list(self): headers={"Accept": "application/json"}, ) - if not response.ok: + if response.status_code == 404: return [] + else: + response.raise_for_status() json_res = response.json() # print(json_res) @@ -286,7 +300,7 @@ def download_file(self, file: SyncFile, retries=3): # f.write(response.content) total = int(total) for data in response.iter_content( - chunk_size=max(int(total / 1000), 1024 * 1024) + chunk_size=max(int(total / 1000), 1024 * 1024) ): f.write(data) @@ -303,6 +317,11 @@ def download_file(self, file: SyncFile, retries=3): self.logger.warning(f"Incorrect LastModified header for file {file.relative_path} {response.headers.get('X-Object-Meta-LocalLastModified')} ; Ignoring...") pass + def commit_changes(self): + response = self.session.post(f"{constants.GOG_CLOUDSTORAGE}/v1/{self.credentials['user_id']}/{self.client_id}") + if not response.ok: + self.logger.error("Failed to commit") + class SyncClassifier: def __init__(self): @@ -310,6 +329,9 @@ def __init__(self): self.updated_local = list() self.updated_cloud = list() + self.not_existing_locally = list() + self.not_existing_remotely = list() + def get_action(self): if len(self.updated_local) == 0 and len(self.updated_cloud) > 0: self.action = SyncAction.DOWNLOAD @@ -329,13 +351,20 @@ def get_action(self): def classify(cls, local, cloud, timestamp): classifier = cls() - for f in local: + local_paths = [f.relative_path for f in local] + cloud_paths = [f.relative_path for f in cloud] + for f in local: + if f.relative_path not in cloud_paths: + classifier.not_existing_remotely.append(f) if f.update_ts > timestamp: classifier.updated_local.append(f) for f in cloud: - + if f.md5 == "aadd86936a80ee8a369579c3926f1b3c": + continue + if f.relative_path not in local_paths: + classifier.not_existing_locally.append(f) if f.update_ts > timestamp: classifier.updated_cloud.append(f)