Skip to content

Commit ad921b2

Browse files
domenicdomenic
committed
Start web platform tests server before tests run
This is an attempt to address recent flaky builds, which we think might be caused because the server is starting in the middle of a JSDOM API test, causing a timeout for the currently-running JSDOM API test. The previous design started booting the server the first time it was needed, and made all tests wait for the server-creation promise before commencing their actual logic. Now, server-creation runs in a before() block. In the process, it removes the fallback to w3c-test.org, as that configuration isn't tested and generally can cause issues.
1 parent 4e7dcca commit ad921b2

File tree

4 files changed

+97
-92
lines changed

4 files changed

+97
-92
lines changed

‎test/web-platform-tests/run-single-wpt.js

Lines changed: 3 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,20 @@
11
"use strict";
2-
/* eslint-disable no-console, no-process-exit, global-require */
2+
/* eslint-disable no-console */
33
const fs = require("fs");
44
const path = require("path");
5-
const dns = require("dns");
6-
const childProcess = require("child_process");
7-
const { EventEmitter } = require("events");
8-
const q = require("q");
95
const { specify } = require("mocha-sugar-free");
106
const { inBrowserContext, nodeResolverPromise } = require("../util.js");
11-
const requestHead = require("request-promise-native").head;
127
const jsdom = require("../../lib/old-api.js");
138

14-
const wptDir = path.resolve(__dirname, "tests");
15-
16-
const configPaths = {
17-
default: path.resolve(__dirname, "wpt-config.json"),
18-
toUpstream: path.resolve(__dirname, "tuwpt-config.json")
19-
};
20-
21-
const configs = {
22-
default: require(configPaths.default),
23-
toUpstream: require(configPaths.toUpstream)
24-
};
25-
269
const globalPool = { maxSockets: 6 };
2710

28-
module.exports = ({ toUpstream = false } = {}) => {
11+
module.exports = urlPrefixFactory => {
2912
if (inBrowserContext()) {
3013
return () => {
3114
// TODO: browser support for running WPT
3215
};
3316
}
3417

35-
const configType = toUpstream ? "toUpstream" : "default";
36-
const configPath = configPaths[configType];
37-
const config = configs[configType];
38-
39-
const server = new EventEmitter();
40-
41-
let serverHasStarted;
42-
server.started = new Promise(resolve => {
43-
serverHasStarted = resolve;
44-
});
45-
server.isStarted = false;
46-
47-
let urlPrefix = `http://${config.host}:${config.ports.http[0]}/`;
48-
49-
dns.lookup("web-platform.test", err => {
50-
if (err) {
51-
console.warn();
52-
console.warn("Host entries not present for web platform tests.");
53-
console.warn("See https://github.com/w3c/web-platform-tests#running-the-tests");
54-
55-
if (!toUpstream) {
56-
console.warn("Falling back to hosted versions at w3c-test.org");
57-
urlPrefix = "http://w3c-test.org/";
58-
}
59-
serverHasStarted();
60-
return;
61-
}
62-
63-
const configArg = path.relative(path.resolve(wptDir), configPath);
64-
const args = ["./wpt.py", "serve", "--config", configArg];
65-
const python = childProcess.spawn("python", args, {
66-
cwd: wptDir,
67-
stdio: "inherit"
68-
});
69-
70-
python.on("error", e => {
71-
console.warn();
72-
console.warn("Error starting python server process:", e.message);
73-
74-
if (toUpstream) {
75-
console.error("Cannot proceed with running the tests.");
76-
process.exit(1);
77-
} else {
78-
console.warn("Falling back to hosted versions at w3ctest.org");
79-
urlPrefix = "http://w3c-test.org/";
80-
serverHasStarted();
81-
}
82-
});
83-
84-
pollForServer(() => urlPrefix).then(serverHasStarted);
85-
86-
process.on("exit", () => {
87-
// Python doesn't register a default handler for SIGTERM and it doesn't run __exit__() methods of context managers
88-
// when it gets that signal. Using SIGINT avoids this problem
89-
python.kill("SIGINT");
90-
});
91-
});
92-
9318
return (testPath, title = testPath) => {
9419
specify({
9520
title,
@@ -99,22 +24,12 @@ module.exports = ({ toUpstream = false } = {}) => {
9924
slow: 10000,
10025
skipIfBrowser: true,
10126
fn() {
102-
return server.started.then(() => createJSDOM(urlPrefix, testPath));
27+
return createJSDOM(urlPrefixFactory(), testPath);
10328
}
10429
});
10530
};
10631
};
10732

108-
function pollForServer(urlGetter) {
109-
console.log("Checking if the web platform tests server is up");
110-
return requestHead(urlGetter())
111-
.then(() => console.log("Server is up!"))
112-
.catch(err => {
113-
console.log(`Server is not up yet (${err.message}); trying again`);
114-
return q.delay(500).then(() => pollForServer(urlGetter));
115-
});
116-
}
117-
11833
function createJSDOM(urlPrefix, testPath) {
11934
const reporterPathname = "/resources/testharnessreport.js";
12035
const unhandledExceptions = [];

‎test/web-platform-tests/run-tuwpts.js

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
"use strict";
22
const path = require("path");
3-
const { describe } = require("mocha-sugar-free");
3+
const { describe, before } = require("mocha-sugar-free");
44
const { spawnSync } = require("child_process");
55
const { readManifest, getPossibleTestFilePaths } = require("./wpt-manifest-utils.js");
6+
const startWPTServer = require("./start-wpt-server.js");
67

78
const wptPath = path.resolve(__dirname, "tests");
89
const testsPath = path.resolve(__dirname, "to-upstream");
910
const manifestFilename = path.resolve(__dirname, "tuwpt-manifest.json");
10-
const runSingleWPT = require("./run-single-wpt.js")({ toUpstream: true });
1111

1212
// We can afford to re-generate the manifest each time; we have few enough files that it's cheap.
1313
const testsRootArg = path.relative(wptPath, testsPath);
@@ -18,6 +18,14 @@ spawnSync("python", args, { cwd: wptPath, stdio: "inherit" });
1818
const manifest = readManifest(manifestFilename);
1919
const possibleTestFilePaths = getPossibleTestFilePaths(manifest);
2020

21+
let wptServerURL;
22+
const runSingleWPT = require("./run-single-wpt.js")(() => wptServerURL);
23+
before({ timeout: 30 * 1000 }, () => {
24+
return startWPTServer({ toUpstream: true }).then(url => {
25+
wptServerURL = url;
26+
});
27+
});
28+
2129
describe("Local tests in web-platform-test format (to-upstream)", () => {
2230
for (const test of possibleTestFilePaths) {
2331
runSingleWPT(test);

‎test/web-platform-tests/run-wpts.js

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ const path = require("path");
33
const fs = require("fs");
44
const jsYAML = require("js-yaml");
55
const { Minimatch } = require("minimatch");
6-
const { describe, specify } = require("mocha-sugar-free");
6+
const { describe, specify, before } = require("mocha-sugar-free");
77
const { readManifest, getPossibleTestFilePaths, stripPrefix } = require("./wpt-manifest-utils.js");
8-
const runSingleWPT = require("./run-single-wpt.js")({ toUpstream: false });
8+
const startWPTServer = require("./start-wpt-server.js");
99

1010
const validReasons = new Set(["fail", "timeout", "needs-await", "needs-node8"]);
1111

@@ -30,6 +30,14 @@ const minimatchers = new Map();
3030

3131
checkToRun();
3232

33+
let wptServerURL;
34+
const runSingleWPT = require("./run-single-wpt.js")(() => wptServerURL);
35+
before({ timeout: 30 * 1000 }, () => {
36+
return startWPTServer({ toUpstream: false }).then(url => {
37+
wptServerURL = url;
38+
});
39+
});
40+
3341
describe("web-platform-tests", () => {
3442
for (const toRunDoc of toRunDocs) {
3543
describe(toRunDoc.DIR, () => {

‎test/web-platform-tests/start-wpt-server.js

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
"use strict";
2+
/* eslint-disable no-console, global-require */
3+
const path = require("path");
4+
const dns = require("dns");
5+
const childProcess = require("child_process");
6+
const q = require("q");
7+
const { inBrowserContext } = require("../util.js");
8+
const requestHead = require("request-promise-native").head;
9+
const dnsLookup = q.denodeify(dns.lookup);
10+
11+
const wptDir = path.resolve(__dirname, "tests");
12+
13+
const configPaths = {
14+
default: path.resolve(__dirname, "wpt-config.json"),
15+
toUpstream: path.resolve(__dirname, "tuwpt-config.json")
16+
};
17+
18+
const configs = {
19+
default: require(configPaths.default),
20+
toUpstream: require(configPaths.toUpstream)
21+
};
22+
23+
module.exports = ({ toUpstream = false } = {}) => {
24+
if (inBrowserContext()) {
25+
return Promise.resolve();
26+
}
27+
28+
const configType = toUpstream ? "toUpstream" : "default";
29+
const configPath = configPaths[configType];
30+
const config = configs[configType];
31+
32+
const urlPrefix = `http://${config.host}:${config.ports.http[0]}/`;
33+
34+
return dnsLookup("web-platform.test").then(
35+
() => {
36+
const configArg = path.relative(path.resolve(wptDir), configPath);
37+
const args = ["./wpt.py", "serve", "--config", configArg];
38+
const python = childProcess.spawn("python", args, {
39+
cwd: wptDir,
40+
stdio: "inherit"
41+
});
42+
43+
return new Promise((resolve, reject) => {
44+
python.on("error", e => {
45+
reject(new Error("Error starting python server process:", e.message));
46+
});
47+
48+
resolve(pollForServer(urlPrefix));
49+
50+
process.on("exit", () => {
51+
// Python doesn't register a default handler for SIGTERM and it doesn't run __exit__() methods of context
52+
// managers when it gets that signal. Using SIGINT avoids this problem.
53+
python.kill("SIGINT");
54+
});
55+
});
56+
},
57+
() => {
58+
throw new Error("Host entries not present for web platform tests. See " +
59+
"https://github.com/w3c/web-platform-tests#running-the-tests");
60+
}
61+
);
62+
};
63+
64+
function pollForServer(url) {
65+
return requestHead(url)
66+
.then(() => {
67+
console.log(`WPT server at ${url} is up!`);
68+
return url;
69+
})
70+
.catch(err => {
71+
console.log(`WPT server at ${url} is not up yet (${err.message}); trying again`);
72+
return q.delay(500).then(() => pollForServer(url));
73+
});
74+
}

0 commit comments

Comments
 (0)