Drop cache info when a redirection took place

When a redirection takes place, Reffy follows the redirection logic (typically done through scripting) but the cache info it gets from Puppeteer remains for the initial URL. Reffy incorrectly assumed that info also applied to the final page. There's no easy way to retrieve the cache info of the final URL. Since that should only affect a spec that moves, and only until we detect and update the URL of the spec in browser-specs, this update simply drops the cache info to force Reffy to crawl the spec. Fix #1774.
w3c · tidoust · Mar 5, 2025 · Feb 17, 2025 · Feb 17, 2025 · 28d88b9080dd4314e607eccf1dbca0f642667eeb
commit 28d88b9080dd4314e607eccf1dbca0f642667eeb
diff --git a/src/lib/mock-server.js b/src/lib/mock-server.js
@@ -169,13 +169,40 @@ mockAgent
     }
   });
 
+mockAgent
+  .get("https://www.w3.org")
+  .intercept({ method: "GET", path: "/TR/iredirect/" })
+  .reply(200,
+    `<!DOCTYPE html><script>window.location = '/TR/recentlyupdated/';</script>`,
+    {
+      headers: {
+        "Content-Type": "text/html",
+        "Last-Modified": "Fri, 11 Feb 2022 00:00:42 GMT"
+      }
+    }
+  );
+
+mockAgent
+  .get("https://www.w3.org")
+  .intercept({ method: "GET", path: "/TR/recentlyupdated/" })
+  .reply(200,
+    `<html><title>Recently updated</title>
+    <h1>Recently updated</h1>`,
+    {
+      headers: {
+        "Content-Type": "text/html",
+        "Last-Modified": (new Date()).toString()
+      }
+    }
+  );
+
 mockAgent
   .get("https://drafts.csswg.org")
   .intercept({ method: "GET", path: "/server-hiccup/" })
   .reply(200,
     `<html><title>Server hiccup</title>
     <h1> Index of Server Hiccup Module Level 42 </h1>`,
-    { header: { "Content-Type": "text/html" } })
+    { headers: { "Content-Type": "text/html" } })
   .persist();
 
 /*nock.emitter.on('error', function (err) {

diff --git a/src/lib/specs-crawler.js b/src/lib/specs-crawler.js
@@ -138,8 +138,15 @@ async function crawlSpec(spec, crawlOptions) {
         if (result.crawled) {
             spec.crawled = result.crawled;
         }
-        if (result.crawlCacheInfo) {
-          spec.crawlCacheInfo = result.crawlCacheInfo;
+        if (result.crawlCacheInfo &&
+            (result.crawled === spec.url ||
+                result.crawled === spec.nightly?.url)) {
+            // Note: Some redirection took place. That happens when, e.g., a
+            // WICG spec gets moved to another group, until we update the URL
+            // in browser-specs. Redirection is done through scripting. Reffy
+            // follows the redirect but the cache info it receives from
+            // Puppeteer is for the initial URL. We cannot rely on it!
+            spec.crawlCacheInfo = result.crawlCacheInfo;
         }
         crawlOptions.modules.forEach(mod => {
             if (result[mod.property]) {

diff --git a/tests/crawl.js b/tests/crawl.js
@@ -103,7 +103,6 @@ if (global.describe && describe instanceof Function) {
       assert.equal(results.results[0].title, 'A test spec');
     });
 
-
     it("skips processing and reuse fallback data when spec cache info indicates it has not changed", async () => {
       const url = "https://www.w3.org/TR/ididnotchange/";
       const fallback = path.resolve(scriptPath, 'crawl-cache.json');
@@ -116,6 +115,15 @@ if (global.describe && describe instanceof Function) {
       assert.equal(results[0].title, "Change is the only constant");
       assert.ifError(results[0].error);
       assert.equal(results[0].refs, "A useful list of refs");
+    });
+
+    it("does not return cache info when a redirection took place", async () => {
+      const url = "https://www.w3.org/TR/iredirect/";
+      const results = await crawlSpecs(
+        [{ url, nightly: { url } }],
+        { forceLocalFetch: true });
+      assert.equal(results[0].title, "Recently updated");
+      assert.equal(results[0].crawlCacheInfo, undefined);
     })
 
     it("reports HTTP error statuses", async () => {