Improvement: Integrate Versions into Projects API

Now we don't have to load any data for each project, greatly reducing
the number of requests and improving responsiveness.

We now also return hidden versions, but with the property "hidden" to
distinguish them in the UI, which fixes a bug where you couldn't look at
hidden versions because they weren't found, but they aren't returned by
default.

fixes: #367

Author: reglim

docat/docat/app.py

@@ -88,10 +88,10 @@ def update_index(index_db: TinyDB = Depends(get_index_db)):
 
 
 @app.get("/api/projects", response_model=Projects, status_code=status.HTTP_200_OK)
-def get_projects():
+def get_projects(include_hidden: bool = False):
     if not DOCAT_UPLOAD_FOLDER.exists():
         return Projects(projects=[])
-    return get_all_projects(DOCAT_UPLOAD_FOLDER)
+    return get_all_projects(DOCAT_UPLOAD_FOLDER, include_hidden)
 
 
 @app.get(
@@ -106,8 +106,8 @@ def get_projects():
     status_code=status.HTTP_200_OK,
     responses={status.HTTP_404_NOT_FOUND: {"model": ApiResponse}},
 )
-def get_project(project):
-    details = get_project_details(DOCAT_UPLOAD_FOLDER, project)
+def get_project(project, include_hidden: bool = False):
+    details = get_project_details(DOCAT_UPLOAD_FOLDER, project, include_hidden)
 
     if not details:
         return JSONResponse(status_code=status.HTTP_404_NOT_FOUND, content={"message": f"Project {project} does not exist"})

docat/docat/models.py

@@ -17,19 +17,20 @@ class ClaimResponse(ApiResponse):
     token: str
 
 
-class ProjectWithVersionCount(BaseModel):
+class ProjectVersion(BaseModel):
     name: str
-    logo: bool
-    versions: int
+    tags: list[str]
+    hidden: bool
 
 
-class Projects(BaseModel):
-    projects: list[ProjectWithVersionCount]
+class Project(BaseModel):
+    name: str
+    logo: bool
+    versions: list[ProjectVersion]
 
 
-class ProjectVersion(BaseModel):
-    name: str
-    tags: list[str]
+class Projects(BaseModel):
+    projects: list[Project]
 
 
 class ProjectDetail(BaseModel):

docat/docat/utils.py

@@ -11,7 +11,7 @@
 from bs4.element import Comment
 from tinydb import Query, TinyDB
 
-from docat.models import ProjectDetail, Projects, ProjectVersion, ProjectWithVersionCount
+from docat.models import Project, ProjectDetail, Projects, ProjectVersion
 
 NGINX_CONFIG_PATH = Path("/etc/nginx/locations.d")
 UPLOAD_FOLDER = "doc"
@@ -109,36 +109,32 @@ def is_forbidden_project_name(name: str) -> bool:
     return name in ["upload", "claim", "delete", "search", "help"]
 
 
-def get_all_projects(upload_folder_path: Path) -> Projects:
+def get_all_projects(upload_folder_path: Path, include_hidden: bool) -> Projects:
     """
     Returns all projects in the upload folder.
     """
+    projects: list[Project] = []
 
-    def count_not_hidden_versions(project) -> int:
-        path = upload_folder_path / project
-        versions = [
-            version
-            for version in (upload_folder_path / project).iterdir()
-            if (path / version).is_dir() and not (path / version).is_symlink() and not (path / version / ".hidden").exists()
-        ]
-        return len(versions)
+    for project in upload_folder_path.iterdir():
+        if not project.is_dir():
+            continue
 
-    projects: list[ProjectWithVersionCount] = []
+        details = get_project_details(upload_folder_path, project.name, include_hidden)
 
-    for project in upload_folder_path.iterdir():
-        if project.is_dir():
-            versions = count_not_hidden_versions(project)
-            if versions < 1:
-                continue
+        if details is None:
+            continue
 
-            project_name = str(project.relative_to(upload_folder_path))
-            project_has_logo = (upload_folder_path / project / "logo").exists()
-            projects.append(ProjectWithVersionCount(name=project_name, logo=project_has_logo, versions=versions))
+        if len(details.versions) < 1:
+            continue
+
+        project_name = str(project.relative_to(upload_folder_path))
+        project_has_logo = (upload_folder_path / project / "logo").exists()
+        projects.append(Project(name=project_name, logo=project_has_logo, versions=details.versions))
 
     return Projects(projects=projects)
 
 
-def get_project_details(upload_folder_path: Path, project_name: str) -> ProjectDetail | None:
+def get_project_details(upload_folder_path: Path, project_name: str, include_hidden: bool) -> ProjectDetail | None:
     """
     Returns all versions and tags for a project.
     """
@@ -149,16 +145,23 @@ def get_project_details(upload_folder_path: Path, project_name: str) -> ProjectD
 
     tags = [x for x in docs_folder.iterdir() if x.is_dir() and x.is_symlink()]
 
+    def should_include(name: str) -> bool:
+        if include_hidden:
+            return True
+
+        return not (docs_folder / name / ".hidden").exists()
+
     return ProjectDetail(
         name=project_name,
         versions=sorted(
             [
                 ProjectVersion(
                     name=str(x.relative_to(docs_folder)),
                     tags=[str(t.relative_to(docs_folder)) for t in tags if t.resolve() == x],
+                    hidden=(docs_folder / x.name / ".hidden").exists(),
                 )
                 for x in docs_folder.iterdir()
-                if x.is_dir() and not x.is_symlink() and not (docs_folder / x.name / ".hidden").exists()
+                if x.is_dir() and not x.is_symlink() and should_include(x.name)
             ],
             key=lambda k: k.name,
             reverse=True,
@@ -179,7 +182,7 @@ def index_all_projects(
     index_db.table("projects")
     index_db.table("files")
 
-    all_projects = get_all_projects(upload_folder_path).projects
+    all_projects = get_all_projects(upload_folder_path, include_hidden=False).projects
 
     for project in all_projects:
         update_version_index_for_project(upload_folder_path, index_db, project.name)
@@ -193,7 +196,7 @@ def update_file_index_for_project(upload_folder_path: Path, index_db: TinyDB, pr
     files_table = index_db.table("files")
     files_table.remove(Query().project == project)
 
-    project_details = get_project_details(upload_folder_path, project)
+    project_details = get_project_details(upload_folder_path, project, include_hidden=False)
 
     if not project_details:
         return
@@ -233,7 +236,7 @@ def update_version_index_for_project(upload_folder_path: Path, index_db: TinyDB,
     Project = Query()
     project_table.remove(Project.name == project)
 
-    details = get_project_details(upload_folder_path, project)
+    details = get_project_details(upload_folder_path, project, include_hidden=False)
 
     if not details:
         return

docat/tests/test_hide_show.py

@@ -6,7 +6,7 @@
 
 def test_hide(client_with_claimed_project):
     """
-    Tests that the version is no longer returned when getting the details after hiding
+    Tests that the version is marked as hidden when getting the details after hiding
     """
     # create a version
     create_response = client_with_claimed_project.post(
@@ -19,7 +19,7 @@ def test_hide(client_with_claimed_project):
     assert project_details_response.status_code == 200
     assert project_details_response.json() == {
         "name": "some-project",
-        "versions": [{"name": "1.0.0", "tags": []}],
+        "versions": [{"name": "1.0.0", "tags": [], "hidden": False}],
     }
 
     # hide the version
@@ -50,7 +50,7 @@ def test_hide_only_version_not_listed_in_projects(client_with_claimed_project):
     projects_response = client_with_claimed_project.get("/api/projects")
     assert projects_response.status_code == 200
     assert projects_response.json() == {
-        "projects": [{"name": "some-project", "logo": False, "versions": 1}],
+        "projects": [{"name": "some-project", "logo": False, "versions": [{"name": "1.0.0", "tags": [], "hidden": False}]}],
     }
 
     # hide the only version
@@ -189,7 +189,7 @@ def test_hide_fails_invalid_token(client_with_claimed_project):
 
 def test_show(client_with_claimed_project):
     """
-    Tests that the version is returned again after requesting show.
+    Tests that the version is no longer marked as hidden after requesting show.
     """
     # create a version
     create_response = client_with_claimed_project.post(
@@ -220,7 +220,7 @@ def test_show(client_with_claimed_project):
     assert project_details_response.status_code == 200
     assert project_details_response.json() == {
         "name": "some-project",
-        "versions": [{"name": "1.0.0", "tags": []}],
+        "versions": [{"name": "1.0.0", "tags": [], "hidden": False}],
     }
 
 
@@ -356,7 +356,7 @@ def test_show_fails_invalid_token(client_with_claimed_project):
 
 def test_hide_and_show_with_tag(client_with_claimed_project):
     """
-    Tests that the version is returned again after requesting show on a tag.
+    Tests that the version is no longer marked as hidden after requesting show on a tag.
     """
     # create a version
     create_response = client_with_claimed_project.post(
@@ -392,5 +392,5 @@ def test_hide_and_show_with_tag(client_with_claimed_project):
     assert project_details_response.status_code == 200
     assert project_details_response.json() == {
         "name": "some-project",
-        "versions": [{"name": "1.0.0", "tags": ["latest"]}],
+        "versions": [{"name": "1.0.0", "tags": ["latest"], "hidden": False}],
     }