[SAVES] Full save sync (automatic files deleting) (#25)

* [SAVES] fully sync saves, by also deleting them

* fix: support for dont sync marker

* improv: use stdout for timestamps only

gogdl/saves.py

@@ -56,7 +56,8 @@ def __init__(self, api_handler, authorization_manager):
         self.logger = logging.getLogger("SAVES")
 
         self.session.headers.update(
-            {"User-Agent": "GOGGalaxyCommunicationService/2.0.4.164 (Windows_32bit)"}
+            {"User-Agent": "GOGGalaxyCommunicationService/2.0.13.27 (Windows_32bit) dont_sync_marker/true installation_source/gog",
+             "X-Object-Meta-User-Agent": "GOGGalaxyCommunicationService/2.0.13.27 (Windows_32bit) dont_sync_marker/true installation_source/gog"}
         )
 
         self.credentials = dict()
@@ -78,7 +79,8 @@ def create_directory_map(self, path: str) -> list:
                 files.append(abs_path)
         return files
 
-    def get_relative_path(self, root: str, path: str) -> str:
+    @staticmethod
+    def get_relative_path(root: str, path: str) -> str:
         if not root.endswith("/") and not root.endswith("\\"):
             root = root + os.sep
         return path.replace(root, "")
@@ -110,27 +112,31 @@ def sync(self, arguments, unknown_args):
         self.get_auth_token()
 
         cloud_files = self.get_cloud_files_list()
+        downloadable_cloud = [f for f in cloud_files if f.md5 != "aadd86936a80ee8a369579c3926f1b3c"]
 
         if len(local_files) > 0 and len(cloud_files) == 0:
             action = SyncAction.UPLOAD
             self.logger.info("No files in cloud, uploading")
             for f in local_files:
                 self.upload_file(f)
             self.logger.info("Done")
+            sys.stdout.write(str(datetime.datetime.now().timestamp()))
+            sys.stdout.flush()
             return
         elif len(local_files) == 0 and len(cloud_files) > 0:
             self.logger.info("No files locally, downloading")
             action = SyncAction.DOWNLOAD
-            for f in cloud_files:
+            for f in downloadable_cloud:
                 self.download_file(f)
             self.logger.info("Done")
+            sys.stdout.write(str(datetime.datetime.now().timestamp()))
+            sys.stdout.flush()
             return
 
         timestamp = float(arguments.timestamp)
         classifier = SyncClassifier.classify(local_files, cloud_files, timestamp)
 
         action = classifier.get_action()
-        # print(action)
 
         if prefered_action:
             if prefered_action == "forceupload":
@@ -139,7 +145,7 @@ def sync(self, arguments, unknown_args):
                 action = SyncAction.UPLOAD
             elif prefered_action == "forcedownload":
                 self.logger.warning("Forcing download")
-                classifier.updated_cloud = cloud_files
+                classifier.updated_cloud = downloadable_cloud 
                 action = SyncAction.DOWNLOAD
             if prefered_action == "upload" and action == SyncAction.DOWNLOAD:
                 self.logger.warning("Refused to upload files, newer files in the cloud")
@@ -150,15 +156,21 @@ def sync(self, arguments, unknown_args):
                 print(self.arguments.timestamp)
                 return
 
-        # return
         if action == SyncAction.UPLOAD:
             self.logger.info("Uploading files")
             for f in classifier.updated_local:
                 self.upload_file(f)
+            for f in classifier.not_existing_locally:
+                self.logger.info(f"DELETING IN CLOUD {f}")
+                self.delete_file(f)
         elif action == SyncAction.DOWNLOAD:
             self.logger.info("Downloading files")
             for f in classifier.updated_cloud:
                 self.download_file(f)
+            for f in classifier.not_existing_remotely:
+                self.logger.info(f"DELETING LOCALLY {f.absolute_path}")
+                os.remove(f.absolute_path)
+
         elif action == SyncAction.CONFLICT:
             self.logger.warning(
                 "Files in conflict force downloading or uploading of files"
@@ -200,8 +212,10 @@ def get_cloud_files_list(self):
             headers={"Accept": "application/json"},
         )
 
-        if not response.ok:
+        if response.status_code == 404:
             return []
+        else:
+            response.raise_for_status()
 
         json_res = response.json()
         # print(json_res)
@@ -286,7 +300,7 @@ def download_file(self, file: SyncFile, retries=3):
             #     f.write(response.content)
             total = int(total)
             for data in response.iter_content(
-                chunk_size=max(int(total / 1000), 1024 * 1024)
+                    chunk_size=max(int(total / 1000), 1024 * 1024)
             ):
                 f.write(data)
 
@@ -303,13 +317,21 @@ def download_file(self, file: SyncFile, retries=3):
             self.logger.warning(f"Incorrect LastModified header for file {file.relative_path} {response.headers.get('X-Object-Meta-LocalLastModified')} ; Ignoring...")
             pass 
 
+    def commit_changes(self):
+        response = self.session.post(f"{constants.GOG_CLOUDSTORAGE}/v1/{self.credentials['user_id']}/{self.client_id}")
+        if not response.ok:
+            self.logger.error("Failed to commit")
+
 
 class SyncClassifier:
     def __init__(self):
         self.action = None
         self.updated_local = list()
         self.updated_cloud = list()
 
+        self.not_existing_locally = list()
+        self.not_existing_remotely = list()
+
     def get_action(self):
         if len(self.updated_local) == 0 and len(self.updated_cloud) > 0:
             self.action = SyncAction.DOWNLOAD
@@ -329,13 +351,20 @@ def get_action(self):
     def classify(cls, local, cloud, timestamp):
         classifier = cls()
 
-        for f in local:
+        local_paths = [f.relative_path for f in local]
+        cloud_paths = [f.relative_path for f in cloud]
 
+        for f in local:
+            if f.relative_path not in cloud_paths:
+                classifier.not_existing_remotely.append(f)
             if f.update_ts > timestamp:
                 classifier.updated_local.append(f)
 
         for f in cloud:
-
+            if f.md5 == "aadd86936a80ee8a369579c3926f1b3c":
+                continue
+            if f.relative_path not in local_paths:
+                classifier.not_existing_locally.append(f)
             if f.update_ts > timestamp:
                 classifier.updated_cloud.append(f)