traceability_index: improvements in the incremental generation

.gitignore

@@ -21,6 +21,7 @@ output.zip
 
 ### StrictDoc's default export path
 output/**
+Output/**
 developer/**/output/**
 tests/unit_server/**/output/**
 

strictdoc/backend/sdoc/models/document.py

@@ -54,7 +54,6 @@ def __init__(
         self.fragments_from_files: List = []
 
         self.ng_level: int = 0
-        self.ng_needs_generation = False
         self.ng_has_requirements = False
 
         self.meta: Optional[DocumentMeta] = None

strictdoc/core/document_iterator.py

@@ -14,7 +14,7 @@
 
 class DocumentCachingIterator:
     def __init__(self, document: SDocDocument) -> None:
-        assert isinstance(document, SDocDocument)
+        assert isinstance(document, SDocDocument), document
 
         self.document: SDocDocument = document
 

strictdoc/core/document_meta.py

@@ -1,4 +1,5 @@
 # mypy: disable-error-code="no-untyped-call,no-untyped-def"
+import os
 from typing import Optional
 
 from strictdoc.export.html.document_type import DocumentType
@@ -18,7 +19,7 @@ def __init__(
         input_doc_rel_path: SDocRelativePath,
         input_doc_dir_rel_path: SDocRelativePath,
         input_doc_assets_dir_rel_path: SDocRelativePath,
-        output_document_dir_full_path,
+        output_document_dir_full_path: str,
         output_document_dir_rel_path: SDocRelativePath,
     ):
         """
@@ -60,10 +61,14 @@ def __init__(
         self.input_doc_assets_dir_rel_path: SDocRelativePath = (
             input_doc_assets_dir_rel_path
         )
-        self.output_document_dir_full_path = output_document_dir_full_path
+        self.output_document_dir_full_path: str = output_document_dir_full_path
         self.output_document_dir_rel_path: SDocRelativePath = (
             output_document_dir_rel_path
         )
+        self.output_document_full_path: str = os.path.join(
+            self.output_document_dir_full_path,
+            self.get_html_doc_path(),
+        )
 
     # Paths
     def get_html_doc_path(self):

strictdoc/core/file_dependency_manager.py

@@ -0,0 +1,126 @@
+import datetime
+import os.path
+from copy import deepcopy
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Optional, Set
+
+from strictdoc.core.project_config import ProjectConfig
+from strictdoc.helpers.file_modification_time import (
+    get_file_modification_time,
+    set_file_modification_time,
+)
+from strictdoc.helpers.pickle import pickle_dump, pickle_load
+
+
+@dataclass
+class FileDependencyEntry:
+    path_to_file_output: str
+    dependencies: Set[str]
+
+
+@dataclass
+class FileDependencyManager:
+    path_to_cache_dir: str
+    path_to_cache: str
+    dependencies_now: Dict[str, FileDependencyEntry]
+    dependencies_prev: Dict[str, FileDependencyEntry]
+
+    @staticmethod
+    def create_from_cache(
+        project_config: ProjectConfig,
+    ) -> "FileDependencyManager":
+        path_to_cache_dir = project_config.get_path_to_cache_dir()
+        path_to_cache = os.path.join(path_to_cache_dir, "dependencies")
+        if not os.path.isfile(path_to_cache):
+            return FileDependencyManager(
+                path_to_cache_dir, path_to_cache, {}, {}
+            )
+
+        with open(path_to_cache, "rb") as cache_file:
+            cache_file_bytes = cache_file.read()
+
+        unpickled_content: Optional[FileDependencyManager] = pickle_load(
+            cache_file_bytes
+        )
+
+        if unpickled_content is not None:
+            assert isinstance(unpickled_content, FileDependencyManager), (
+                unpickled_content,
+            )
+            unpickled_content.dependencies_now.clear()
+            return unpickled_content
+
+        return FileDependencyManager(path_to_cache_dir, path_to_cache, {}, {})
+
+    def add_dependency(
+        self,
+        path_to_input_file: str,
+        path_to_output_file: str,
+        path_to_dependent_file: str,
+    ) -> None:
+        assert os.path.isfile(path_to_input_file), path_to_input_file
+        assert os.path.isfile(path_to_dependent_file), path_to_dependent_file
+
+        entry = self.dependencies_now.setdefault(
+            path_to_input_file, FileDependencyEntry(path_to_output_file, set())
+        )
+        entry.dependencies.add(path_to_dependent_file)
+
+    def save_to_cache(self) -> None:
+        self.dependencies_prev = deepcopy(self.dependencies_now)
+
+        pickled_content = pickle_dump(self)
+
+        Path(self.path_to_cache_dir).mkdir(exist_ok=True, parents=True)
+        with open(self.path_to_cache, "wb") as cache_file:
+            cache_file.write(pickled_content)
+
+    def resolve_modification_dates(
+        self, strictdoc_last_update: datetime.datetime
+    ) -> None:
+        visited_dependencies = set()
+
+        items_now, items_before = (
+            self.dependencies_now.items(),
+            self.dependencies_prev.items(),
+        )
+        for items_ in [items_now, items_before]:
+            # FIXME
+            print(f"CURRENT ITEM: {items_}")  # noqa: T201
+            for path_to_input_file_, entry_ in items_:
+                path_to_output_file = entry_.path_to_file_output
+
+                if (
+                    # The file has not been generated yet.
+                    not os.path.isfile(path_to_output_file)
+                    # The file used to exist but not anymore.
+                    or not os.path.isfile(path_to_input_file_)
+                    # The file is outdated compared to its HTML output artifact.
+                    or get_file_modification_time(path_to_input_file_)
+                    > get_file_modification_time(path_to_output_file)
+                    # The file is outdated compared to StrictDoc's own code (this
+                    # branch is development-only).
+                    or strictdoc_last_update
+                    > get_file_modification_time(path_to_output_file)
+                ):
+                    # If the file is not a no longer existing file, mark it for
+                    # regeneration.
+                    if os.path.isfile(path_to_input_file_):
+                        visited_dependencies.add(path_to_input_file_)
+
+                    for path_to_dependent_input_file_ in entry_.dependencies:
+                        if (
+                            path_to_dependent_input_file_
+                            in visited_dependencies
+                        ):
+                            continue
+
+                        print(f"DEP ITEM: {path_to_dependent_input_file_}")  # noqa: T201
+
+                        visited_dependencies.add(path_to_dependent_input_file_)
+
+        for dependency_ in visited_dependencies:
+            set_file_modification_time(dependency_, datetime.datetime.today())
+
+        self.save_to_cache()