From fe6c79885d21e510450cc366adad8bb2ac611142 Mon Sep 17 00:00:00 2001 From: Francois Daoust Date: Mon, 3 Jun 2024 15:11:34 +0200 Subject: [PATCH] Make `multipage` flag specifically target release and/or nightly The job still fails to load the ED of the CSS 2.2 spec to extract the list of pages, even with optimized loading (added in last commit). Something's rotten in the State of Network. In practice, build does not *need* to load the ED of the CSS 2.2 spec. It just does it because the spec is identified as multipage due to the /TR version. To avoid unnecessary spec loads, this update converts the `multipage` flag from a boolean to a string with 3 possible values: "all", "release", "nightly". The extraction of the list of pages is now only done on the version targeted by the `multipage` property. The update also serializes the extraction of pages to avoid sending too many requests at once. There are only a handful of specs for which we need to look for pages, serialization should not make build much longer than it already is. --- CONTRIBUTING.md | 7 +++++-- schema/specs.json | 7 ++++++- specs.json | 10 +++++----- src/build-index.js | 29 +++++++++++++++-------------- src/lint.js | 4 ++-- test/lint.js | 6 +++--- 6 files changed, 36 insertions(+), 27 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 17b7bf1f..ed37592c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -120,9 +120,12 @@ automatically is not the expected one. - `forceCurrent`: a boolean flag to tell the code that the spec should be seen as the current spec in the series. The property must only be set when value is `true`. -- `multipage`: a boolean flag to identify the spec as a multipage spec. This +- `multipage`: a flag to identify the spec as a multipage spec. This instructs the code to extract the list of pages from the index page and fill -out the `release.pages` and `nightly.pages` properties in the list. +out the `release.pages` and `nightly.pages` properties in the list. Possible +values for the flag: `"nightly"` to signal that the nightly version is +multipage, `"release"` to signal that the release version if multipage, or +`"all"` to signal that both the nightly and release versions are multipage. - `categories`: an array that is treated as incremental update to adjust the list of [`categories`](README.md#categories) that the spec belongs to. Values may be one of `"reset"` to start from an empty list, `"+browser"` to add diff --git a/schema/specs.json b/schema/specs.json index 882fab6b..3339e13b 100644 --- a/schema/specs.json +++ b/schema/specs.json @@ -30,7 +30,12 @@ "obsoletedBy": { "$ref": "definitions.json#/$defs/obsoletedBy" }, "formerNames": { "$ref": "definitions.json#/$defs/formerNames" }, "forceCurrent": { "type": "boolean" }, - "multipage": { "type": "boolean" } + "multipage": { + "type": "string", + "enum": [ + "all", "release", "nightly" + ] + } }, "required": ["url"], "additionalProperties": false diff --git a/specs.json b/specs.json index dcd00843..1ea6e1f9 100644 --- a/specs.json +++ b/specs.json @@ -154,7 +154,7 @@ "https://fullscreen.spec.whatwg.org/", { "url": "https://html.spec.whatwg.org/multipage/", - "multipage": true, + "multipage": "nightly", "nightly": { "sourcePath": "source" } @@ -303,7 +303,7 @@ "https://svgwg.org/specs/animations/", { "url": "https://tc39.es/ecma262/multipage/", - "multipage": true, + "multipage": "nightly", "shortname": "ecmascript", "shortTitle": "ECMAScript", "tests": { @@ -1065,14 +1065,14 @@ "url": "https://www.w3.org/TR/CSS21/", "shortname": "CSS2", "seriesVersion": "2.1", - "multipage": true, + "multipage": "all", "nightly": { "url": "https://www.w3.org/TR/CSS21/" } }, { "url": "https://www.w3.org/TR/CSS22/", - "multipage": true, + "multipage": "release", "nightly": { "sourcePath": "css2/Overview.bs" } @@ -1495,7 +1495,7 @@ "https://www.w3.org/TR/svg-strokes/", { "url": "https://www.w3.org/TR/SVG11/", - "multipage": true, + "multipage": "all", "nightly": { "url": "https://www.w3.org/TR/SVG11/" } diff --git a/src/build-index.js b/src/build-index.js index bea8a713..8435e46a 100644 --- a/src/build-index.js +++ b/src/build-index.js @@ -138,7 +138,7 @@ async function runSkeleton(specs, { log }) { res.forceCurrent = true; } else if (parts[1] === "multipage") { - res.multipage = true; + res.multipage = "all"; } return res; } @@ -320,22 +320,23 @@ async function runShortTitle(index) { async function runPages(index) { const browser = await puppeteer.launch(); - return Promise.all( - index.map(async spec => { - if (spec.multipage) { - if (spec.release) { - spec.release.pages = await extractPages(spec.release.url, browser); - } - if (spec.nightly) { - spec.nightly.pages = await extractPages(spec.nightly.url, browser); - } + try { + for (const spec of index) { + if (spec.release && (spec.multipage === "all" || spec.multipage === "release")) { + spec.release.pages = await extractPages(spec.release.url, browser); + } + if (spec.nightly && (spec.multipage === "all" || spec.multipage === "nightly")) { + spec.nightly.pages = await extractPages(spec.nightly.url, browser); + } + if (spec.hasOwnProperty("multipage")) { delete spec.multipage; } - return spec; - }) - ).finally(async _ => { + } + } + finally { await browser.close(); - }); + } + return index; } diff --git a/src/lint.js b/src/lint.js index 5359625c..df8ec5fa 100644 --- a/src/lint.js +++ b/src/lint.js @@ -33,7 +33,7 @@ function shortenDefinition(spec) { return `${spec.url} current`; } else if (Object.keys(short).length === 2 && - spec.multipage) { + spec.multipage === "all") { return `${spec.url} multipage`; } else { @@ -56,7 +56,7 @@ function lintStr(specsStr) { url: new URL(spec.split(" ")[0]).toString(), seriesComposition: (spec.split(' ')[1] === "delta") ? "delta" : "full", forceCurrent: (spec.split(' ')[1] === "current"), - multipage: (spec.split(' ')[1] === "multipage"), + multipage: (spec.split(' ')[1] === "multipage") ? "all" : undefined } : Object.assign({}, spec, { url: new URL(spec.url).toString() })) .filter((spec, idx, list) => diff --git a/test/lint.js b/test/lint.js index b75244f0..7b89ff35 100644 --- a/test/lint.js +++ b/test/lint.js @@ -98,7 +98,7 @@ describe("Linter", () => { it("lints an object with only a URL and a multipage flag to a string", () => { const specs = [ - { "url": "https://www.w3.org/TR/spec-1/", "multipage": true } + { "url": "https://www.w3.org/TR/spec-1/", "multipage": "all" } ]; assert.equal(lintStr(toStr(specs)), toStr([ "https://www.w3.org/TR/spec-1/ multipage" @@ -123,9 +123,9 @@ describe("Linter", () => { ])); }); - it("lints an object with a multipage flag set to false", () => { + it("lints an object with a multipage flag set to null", () => { const specs = [ - { "url": "https://www.w3.org/TR/spec/", "multipage": false } + { "url": "https://www.w3.org/TR/spec/", "multipage": null } ]; assert.equal(lintStr(toStr(specs)), toStr([ "https://www.w3.org/TR/spec/"