Improvement: Integrate Versions and into Projects API

Now we don't have to load any data for each project, greatly reducing
the number of requests and improving responsiveness.

We now also return hidden versions, but with the property "hidden" to
distinguish them in the UI, which fixes a bug where you couldn't look at
hidden versions because they weren't found.

fixes: #367

Author: reglim

docat/docat/models.py

@@ -17,19 +17,20 @@ class ClaimResponse(ApiResponse):
     token: str
 
 
-class ProjectWithVersionCount(BaseModel):
+class ProjectVersion(BaseModel):
     name: str
-    logo: bool
-    versions: int
+    tags: list[str]
+    hidden: bool
 
 
-class Projects(BaseModel):
-    projects: list[ProjectWithVersionCount]
+class Project(BaseModel):
+    name: str
+    logo: bool
+    versions: list[ProjectVersion]
 
 
-class ProjectVersion(BaseModel):
-    name: str
-    tags: list[str]
+class Projects(BaseModel):
+    projects: list[Project]
 
 
 class ProjectDetail(BaseModel):

docat/docat/utils.py

@@ -11,7 +11,7 @@
 from bs4.element import Comment
 from tinydb import Query, TinyDB
 
-from docat.models import ProjectDetail, Projects, ProjectVersion, ProjectWithVersionCount
+from docat.models import Project, ProjectDetail, Projects, ProjectVersion
 
 NGINX_CONFIG_PATH = Path("/etc/nginx/locations.d")
 UPLOAD_FOLDER = "doc"
@@ -113,32 +113,28 @@ def get_all_projects(upload_folder_path: Path) -> Projects:
     """
     Returns all projects in the upload folder.
     """
+    projects: list[Project] = []
 
-    def count_not_hidden_versions(project) -> int:
-        path = upload_folder_path / project
-        versions = [
-            version
-            for version in (upload_folder_path / project).iterdir()
-            if (path / version).is_dir() and not (path / version).is_symlink() and not (path / version / ".hidden").exists()
-        ]
-        return len(versions)
+    for project in upload_folder_path.iterdir():
+        if not project.is_dir():
+            continue
 
-    projects: list[ProjectWithVersionCount] = []
+        details = get_project_details(upload_folder_path, project.name)
 
-    for project in upload_folder_path.iterdir():
-        if project.is_dir():
-            versions = count_not_hidden_versions(project)
-            if versions < 1:
-                continue
+        if details is None:
+            continue
+
+        if len(details.versions) < 1:
+            continue
 
-            project_name = str(project.relative_to(upload_folder_path))
-            project_has_logo = (upload_folder_path / project / "logo").exists()
-            projects.append(ProjectWithVersionCount(name=project_name, logo=project_has_logo, versions=versions))
+        project_name = str(project.relative_to(upload_folder_path))
+        project_has_logo = (upload_folder_path / project / "logo").exists()
+        projects.append(Project(name=project_name, logo=project_has_logo, versions=details.versions))
 
     return Projects(projects=projects)
 
 
-def get_project_details(upload_folder_path: Path, project_name: str) -> ProjectDetail | None:
+def get_project_details(upload_folder_path: Path, project_name: str, include_hidden: bool = True) -> ProjectDetail | None:
     """
     Returns all versions and tags for a project.
     """
@@ -156,9 +152,10 @@ def get_project_details(upload_folder_path: Path, project_name: str) -> ProjectD
                 ProjectVersion(
                     name=str(x.relative_to(docs_folder)),
                     tags=[str(t.relative_to(docs_folder)) for t in tags if t.resolve() == x],
+                    hidden=(docs_folder / x.name / ".hidden").exists(),
                 )
                 for x in docs_folder.iterdir()
-                if x.is_dir() and not x.is_symlink() and not (docs_folder / x.name / ".hidden").exists()
+                if x.is_dir() and not x.is_symlink() and (include_hidden or not (docs_folder / x.name / ".hidden").exists())
             ],
             key=lambda k: k.name,
             reverse=True,
@@ -193,7 +190,7 @@ def update_file_index_for_project(upload_folder_path: Path, index_db: TinyDB, pr
     files_table = index_db.table("files")
     files_table.remove(Query().project == project)
 
-    project_details = get_project_details(upload_folder_path, project)
+    project_details = get_project_details(upload_folder_path, project, include_hidden=False)
 
     if not project_details:
         return
@@ -233,7 +230,7 @@ def update_version_index_for_project(upload_folder_path: Path, index_db: TinyDB,
     Project = Query()
     project_table.remove(Project.name == project)
 
-    details = get_project_details(upload_folder_path, project)
+    details = get_project_details(upload_folder_path, project, include_hidden=False)
 
     if not details:
         return

docat/tests/test_hide_show.py

@@ -6,7 +6,7 @@
 
 def test_hide(client_with_claimed_project):
     """
-    Tests that the version is no longer returned when getting the details after hiding
+    Tests that the version is marked as hidden when getting the details after hiding
     """
     # create a version
     create_response = client_with_claimed_project.post(
@@ -19,7 +19,7 @@ def test_hide(client_with_claimed_project):
     assert project_details_response.status_code == 200
     assert project_details_response.json() == {
         "name": "some-project",
-        "versions": [{"name": "1.0.0", "tags": []}],
+        "versions": [{"name": "1.0.0", "tags": [], "hidden": False}],
     }
 
     # hide the version
@@ -32,45 +32,10 @@ def test_hide(client_with_claimed_project):
     assert project_details_response.status_code == 200
     assert project_details_response.json() == {
         "name": "some-project",
-        "versions": [],
+        "versions": [{"name": "1.0.0", "tags": [], "hidden": True}],
     }
 
 
-def test_hide_only_version_not_listed_in_projects(client_with_claimed_project):
-    """
-    Test that the project is not listed in the projects endpoint when the only version is hidden
-    """
-    # create a version
-    create_response = client_with_claimed_project.post(
-        "/api/some-project/1.0.0", files={"file": ("index.html", io.BytesIO(b"<h1>Hello World</h1>"), "plain/text")}
-    )
-    assert create_response.status_code == 201
-
-    # check detected before hiding
-    projects_response = client_with_claimed_project.get("/api/projects")
-    assert projects_response.status_code == 200
-    assert projects_response.json() == {
-        "projects": [{"name": "some-project", "logo": False, "versions": 1}],
-    }
-
-    # hide the only version
-    hide_response = client_with_claimed_project.post("/api/some-project/1.0.0/hide", headers={"Docat-Api-Key": "1234"})
-    assert hide_response.status_code == 200
-    assert hide_response.json() == {"message": "Version 1.0.0 is now hidden"}
-
-    # check hidden
-    projects_response = client_with_claimed_project.get("/api/projects")
-    assert projects_response.status_code == 200
-    assert projects_response.json() == {
-        "projects": [],
-    }
-
-    # check versions hidden
-    project_details_response = client_with_claimed_project.get("/api/projects/some-project")
-    assert project_details_response.status_code == 200
-    assert project_details_response.json() == {"name": "some-project", "versions": []}
-
-
 def test_hide_creates_hidden_file(client_with_claimed_project):
     """
     Tests that the hidden file is created when hiding a version
@@ -189,7 +154,7 @@ def test_hide_fails_invalid_token(client_with_claimed_project):
 
 def test_show(client_with_claimed_project):
     """
-    Tests that the version is returned again after requesting show.
+    Tests that the version is no longer marked as hidden after requesting show.
     """
     # create a version
     create_response = client_with_claimed_project.post(
@@ -207,7 +172,7 @@ def test_show(client_with_claimed_project):
     assert project_details_response.status_code == 200
     assert project_details_response.json() == {
         "name": "some-project",
-        "versions": [],
+        "versions": [{"name": "1.0.0", "tags": [], "hidden": True}],
     }
 
     # show the version
@@ -220,7 +185,7 @@ def test_show(client_with_claimed_project):
     assert project_details_response.status_code == 200
     assert project_details_response.json() == {
         "name": "some-project",
-        "versions": [{"name": "1.0.0", "tags": []}],
+        "versions": [{"name": "1.0.0", "tags": [], "hidden": False}],
     }
 
 
@@ -356,7 +321,7 @@ def test_show_fails_invalid_token(client_with_claimed_project):
 
 def test_hide_and_show_with_tag(client_with_claimed_project):
     """
-    Tests that the version is returned again after requesting show on a tag.
+    Tests that the version is no longer marked as hidden after requesting show on a tag.
     """
     # create a version
     create_response = client_with_claimed_project.post(
@@ -379,7 +344,7 @@ def test_hide_and_show_with_tag(client_with_claimed_project):
     assert project_details_response.status_code == 200
     assert project_details_response.json() == {
         "name": "some-project",
-        "versions": [],
+        "versions": [{"name": "1.0.0", "tags": ["latest"], "hidden": True}],
     }
 
     # show the version
@@ -392,5 +357,5 @@ def test_hide_and_show_with_tag(client_with_claimed_project):
     assert project_details_response.status_code == 200
     assert project_details_response.json() == {
         "name": "some-project",
-        "versions": [{"name": "1.0.0", "tags": ["latest"]}],
+        "versions": [{"name": "1.0.0", "tags": ["latest"], "hidden": False}],
     }

docat/tests/test_project.py

@@ -1,10 +1,13 @@
+import io
 from unittest.mock import patch
 
 from fastapi.testclient import TestClient
 
-from docat.app import app
+import docat.app as docat
+from docat.models import ProjectDetail, ProjectVersion
+from docat.utils import get_project_details
 
-client = TestClient(app)
+client = TestClient(docat.app)
 
 
 def test_project_api(temp_project_version):
@@ -15,7 +18,17 @@ def test_project_api(temp_project_version):
         response = client.get("/api/projects")
 
         assert response.ok
-        assert response.json() == {"projects": [{"name": "project", "logo": False, "versions": 1}]}
+        assert response.json() == {
+            "projects": [
+                {
+                    "name": "project",
+                    "logo": False,
+                    "versions": [
+                        {"name": "1.0", "tags": ["latest"], "hidden": False},
+                    ],
+                }
+            ]
+        }
 
 
 def test_project_api_without_any_projects():
@@ -35,11 +48,59 @@ def test_project_details_api(temp_project_version):
         response = client.get(f"/api/projects/{project}")
 
         assert response.ok
-        assert response.json() == {"name": "project", "versions": [{"name": "1.0", "tags": ["latest"]}]}
+        assert response.json() == {"name": "project", "versions": [{"name": "1.0", "tags": ["latest"], "hidden": False}]}
 
 
 def test_project_details_api_with_a_project_that_does_not_exist():
     response = client.get("/api/projects/i-do-not-exist")
 
     assert not response.ok
     assert response.json() == {"message": "Project i-do-not-exist does not exist"}
+
+
+def test_get_project_details_with_hidden_versions(client_with_claimed_project):
+    """
+    Make sure that get_project_details works when include_hidden is set to True.
+    """
+    # create a version
+    create_response = client_with_claimed_project.post(
+        "/api/some-project/1.0.0", files={"file": ("index.html", io.BytesIO(b"<h1>Hello World</h1>"), "plain/text")}
+    )
+    assert create_response.status_code == 201
+
+    # check detected before hiding
+    details = get_project_details(docat.DOCAT_UPLOAD_FOLDER, "some-project", include_hidden=True)
+    assert details == ProjectDetail(name="some-project", versions=[ProjectVersion(name="1.0.0", tags=[], hidden=False)])
+
+    # hide the version
+    hide_response = client_with_claimed_project.post("/api/some-project/1.0.0/hide", headers={"Docat-Api-Key": "1234"})
+    assert hide_response.status_code == 200
+    assert hide_response.json() == {"message": "Version 1.0.0 is now hidden"}
+
+    # check hidden
+    details = get_project_details(docat.DOCAT_UPLOAD_FOLDER, "some-project", include_hidden=True)
+    assert details == ProjectDetail(name="some-project", versions=[ProjectVersion(name="1.0.0", tags=[], hidden=True)])
+
+
+def test_project_details_without_hidden_versions(client_with_claimed_project):
+    """
+    Make sure that project_details works when include_hidden is set to False.
+    """
+    # create a version
+    create_response = client_with_claimed_project.post(
+        "/api/some-project/1.0.0", files={"file": ("index.html", io.BytesIO(b"<h1>Hello World</h1>"), "plain/text")}
+    )
+    assert create_response.status_code == 201
+
+    # check detected before hiding
+    details = get_project_details(docat.DOCAT_UPLOAD_FOLDER, "some-project", include_hidden=False)
+    assert details == ProjectDetail(name="some-project", versions=[ProjectVersion(name="1.0.0", tags=[], hidden=False)])
+
+    # hide the version
+    hide_response = client_with_claimed_project.post("/api/some-project/1.0.0/hide", headers={"Docat-Api-Key": "1234"})
+    assert hide_response.status_code == 200
+    assert hide_response.json() == {"message": "Version 1.0.0 is now hidden"}
+
+    # check hidden
+    details = get_project_details(docat.DOCAT_UPLOAD_FOLDER, "some-project", include_hidden=False)
+    assert details == ProjectDetail(name="some-project", versions=[])

docat/tests/test_upload_icon.py

@@ -160,7 +160,7 @@ def test_get_project_recongizes_icon(client_with_claimed_project):
             {
                 "name": "some-project",
                 "logo": False,
-                "versions": 1,
+                "versions": [{"name": "1.0.0", "tags": [], "hidden": False}],
             }
         ]
     }
@@ -178,7 +178,7 @@ def test_get_project_recongizes_icon(client_with_claimed_project):
             {
                 "name": "some-project",
                 "logo": True,
-                "versions": 1,
+                "versions": [{"name": "1.0.0", "tags": [], "hidden": False}],
             }
         ]
     }

docat/tests/test_utils.py

@@ -1,4 +1,3 @@
-import io
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
@@ -87,25 +86,3 @@ def test_remove_symlink_version(temp_project_version):
     remove_docs(project, "latest", docat.DOCAT_UPLOAD_FOLDER)
 
     assert not symlink_to_latest.exists()
-
-
-def test_get_all_projects_counts_versions_correctly(client_with_claimed_project):
-    """
-    Tests whether get_all_projects returns the correct number of versions.
-    (Don't count symlinks)
-    """
-
-    versions = ["1.0.0", "2.0.0", "3.0.0"]
-    for version in versions:
-        response = client_with_claimed_project.post(
-            f"/api/some-project/{version}", files={"file": ("index.html", io.BytesIO(b"<h1>Hello World</h1>"), "plain/text")}
-        )
-        assert response.status_code == 201
-
-    # tag "3.0.0" as latest
-    response = client_with_claimed_project.put(f"/api/some-project/{versions[-1]}/tags/latest")
-    assert response.status_code == 201
-
-    response = client_with_claimed_project.get("/api/projects")
-    assert response.status_code == 200
-    assert response.json() == {"projects": [{"name": "some-project", "logo": False, "versions": len(versions)}]}

web/src/components/Project.tsx

@@ -50,9 +50,9 @@ export default function Project(props: Props): JSX.Element {
         />
       </div>
       <div className={styles.subhead}>
-        {props.project.versions === 1
-          ? `${props.project.versions} version`
-          : `${props.project.versions} versions`}
+        {props.project.versions.length === 1
+          ? `${props.project.versions.length} version`
+          : `${props.project.versions.length} versions`}
       </div>
     </div>
   )