Skip to content

Commit

Permalink
QA-536: move san file handling in its own class (#20898)
Browse files Browse the repository at this point in the history
* move san file handling in its own class, and also deploy it for launched subprocesses

* move this function as well.

* use san file handler

* fix import

* fix name clash

* don't shadow param

* fix binary name

* Update js/client/modules/@arangodb/testutils/instance.js

Co-authored-by: Manuel Pöter <[email protected]>

* Update js/client/modules/@arangodb/testutils/process-utils.js

Co-authored-by: Manuel Pöter <[email protected]>

* fix constructing parameters

* debug

* remove spooky chars

* remove spooky chars

* no debug

* Update js/client/modules/@arangodb/testutils/san-file-handler.js

Co-authored-by: Manuel Pöter <[email protected]>

* preserve any crash

* backwards compatibility

* backwards compatibility

---------

Co-authored-by: Manuel Pöter <[email protected]>
Co-authored-by: Vadim Kondratev <[email protected]>
  • Loading branch information
3 people committed May 15, 2024
1 parent adf9d99 commit 25c89db
Show file tree
Hide file tree
Showing 3 changed files with 145 additions and 80 deletions.
93 changes: 14 additions & 79 deletions js/client/modules/@arangodb/testutils/instance.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const rp = require('@arangodb/testutils/result-processing');
const yaml = require('js-yaml');
const internal = require('internal');
const crashUtils = require('@arangodb/testutils/crash-utils');
const {sanHandler} = require('@arangodb/testutils/san-file-handler');
const crypto = require('@arangodb/crypto');
const ArangoError = require('@arangodb').ArangoError;
const debugGetFailurePoints = require('@arangodb/test-helper').debugGetFailurePoints;
Expand All @@ -41,7 +42,6 @@ const debugGetFailurePoints = require('@arangodb/test-helper').debugGetFailurePo
const {
toArgv,
executeExternal,
executeExternalAndWait,
killExternal,
statusExternal,
statisticsExternal,
Expand Down Expand Up @@ -73,13 +73,6 @@ let tcpdump;

let PORTMANAGER;

var regex = /[^\u0000-\u00ff]/; // Small performance gain from pre-compiling the regex
function containsDoubleByte(str) {
if (!str.length) return false;
if (str.charCodeAt(0) > 255) return true;
return regex.test(str);
}

function getSockStatFile(pid) {
try {
return fs.read("/proc/" + pid + "/net/sockstat");
Expand Down Expand Up @@ -224,9 +217,7 @@ class instance {
}
this.JWT = null;
this.jwtFiles = null;

this.sanOptions = _.clone(this.options.sanOptions);
this.sanitizerLogPaths = {};
this.sanHandler = new sanHandler('arangod', this.options.sanOptions, this.options.isSan, this.options.extremeVerbosity);

this._makeArgsArangod();

Expand Down Expand Up @@ -468,20 +459,7 @@ class instance {
if (this.args.hasOwnProperty('server.jwt-secret')) {
this.JWT = this.args['server.jwt-secret'];
}
if (this.options.isSan) {
let rootDir = this.rootDir;
if (containsDoubleByte(rootDir)) {
rootDir = this.topLevelTmpDir;
}
for (const [key, value] of Object.entries(this.sanOptions)) {
let oneLogFile = fs.join(rootDir, key.toLowerCase().split('_')[0] + '.log');
// we need the log files to contain the exe name, otherwise our code to pick them up won't find them
this.sanOptions[key]['log_exe_name'] = "true";
const origPath = this.sanOptions[key]['log_path'];
this.sanOptions[key]['log_path'] = oneLogFile;
this.sanitizerLogPaths[key] = { upstream: origPath, local: oneLogFile };
}
}
this.sanHandler.detectLogfiles(this.rootDir, this.topLevelTmpDir);
}

// //////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -693,21 +671,9 @@ class instance {
if (this.options.extremeVerbosity) {
print(Date() + ' starting process ' + cmd + ' with arguments: ' + JSON.stringify(argv));
}
let backup = {};
if (this.options.isSan) {
print("Using sanOptions ", this.sanOptions);
for (const [key, value] of Object.entries(this.sanOptions)) {
let oneSet = "";
for (const [keyOne, valueOne] of Object.entries(value)) {
if (oneSet.length > 0) {
oneSet += ":";
}
oneSet += `${keyOne}=${valueOne}`;
}
backup[key] = process.env[key];
process.env[key] = oneSet;
}
}

this.sanHandler.setSanOptions();

if ((this.useableMemory === undefined) && (this.options.memory !== undefined)){
throw new Error(`${this.name} don't have planned memory though its configured!`);
}
Expand All @@ -719,11 +685,8 @@ class instance {
}
process.env['ARANGODB_SERVER_DIR'] = this.rootDir;
let ret = executeExternal(cmd, argv, false, pu.coverageEnvironment());
if (this.options.isSan) {
for (const [key, value] of Object.entries(backup)) {
process.env[key] = value;
}
}

this.sanHandler.resetSanOptions();
if (this.useableMemory !== 0) {
delete process.env['ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY'];
}
Expand Down Expand Up @@ -793,34 +756,6 @@ class instance {
internal.addPidToMonitor(this.pid);
}
};

fetchSanFileAfterExit() {
if (!this.options.isSan) {
return;
}

for (const [key, value] of Object.entries(this.sanitizerLogPaths)) {
print("processing ", value);
const { upstream, local } = value;
let fn = `${local}.arangod.${this.pid}`;
if (this.options.extremeVerbosity) {
print(`checking for ${fn}: ${fs.exists(fn)}`);
}
if (fs.exists(fn)) {
let content = fs.read(fn);
if (upstream) {
print("found file ", fn, " - writing file ", `${upstream}.arangod.${this.pid}`);
fs.write(`${upstream}.arangod.${this.pid}`, content);
}
if (content.length > 10) {
crashUtils.GDB_OUTPUT += `Report of '${this.name}' in ${fn} contains: \n`;
crashUtils.GDB_OUTPUT += content;
this.serverCrashedLocal = true;
}
}
}
}

waitForExitAfterDebugKill() {
// Crashutils debugger kills our instance, but we neet to get
// testing.js sapwned-PID-monitoring adjusted.
Expand All @@ -836,7 +771,7 @@ class instance {
} catch(ex) {
print(ex);
}
this.fetchSanFileAfterExit();
this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
this.pid = null;
print('done');
}
Expand All @@ -847,10 +782,10 @@ class instance {
}
this.exitStatus = statusExternal(this.pid, true);
if (this.exitStatus.status !== 'TERMINATED') {
this.fetchSanFileAfterExit();
this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
throw new Error(this.name + " didn't exit in a regular way: " + JSON.stringify(this.exitStatus));
}
this.fetchSanFileAfterExit();
this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
this.exitStatus = null;
this.pid = null;
}
Expand Down Expand Up @@ -1051,7 +986,7 @@ class instance {
} else if (this.options.useKillExternal) {
let sockStat = this.getSockStat("Shutdown by kill - sockstat before: ");
this.exitStatus = killExternal(this.pid);
this.fetchSanFileAfterExit();
this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
this.pid = null;
print(sockStat);
} else if (this.protocol === 'unix') {
Expand All @@ -1073,7 +1008,7 @@ class instance {
print(Date() + ' Wrong shutdown response: ' + JSON.stringify(reply) + "' " + sockStat + " continuing with hard kill!");
this.shutdownArangod(true);
} else {
this.fetchSanFileAfterExit();
this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
if (!this.options.noStartStopLogs) {
print(sockStat);
}
Expand Down Expand Up @@ -1101,7 +1036,7 @@ class instance {
this.shutdownArangod(true);
}
else {
this.fetchSanFileAfterExit();
this.serverCrashedLocal = this.serverCrashedLocal || this.sanHandler.fetchSanFileAfterExit(this.pid);
if (!this.options.noStartStopLogs) {
print(sockStat);
}
Expand Down
14 changes: 13 additions & 1 deletion js/client/modules/@arangodb/testutils/process-utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const rp = require('@arangodb/testutils/result-processing');
const yaml = require('js-yaml');
const internal = require('internal');
const crashUtils = require('@arangodb/testutils/crash-utils');
const {sanHandler} = require('@arangodb/testutils/san-file-handler');
const crypto = require('@arangodb/crypto');
const ArangoError = require('@arangodb').ArangoError;
const debugGetFailurePoints = require('@arangodb/test-helper').debugGetFailurePoints;
Expand Down Expand Up @@ -395,13 +396,24 @@ function executeAndWait (cmd, args, options, valgrindTest, rootDir, coreCheck =
}

// V8 executeExternalAndWait thinks that timeout is in ms, so *1000

let sh = new sanHandler(cmd.replace(/.*\//, ''), options.sanOptions, options.isSan, options.extremeVerbosity);
sh.detectLogfiles(instanceInfo.rootDir, instanceInfo.rootDir);
sh.setSanOptions();

let res = executeExternalAndWait(cmd, args, false, timeout * 1000, coverageEnvironment());

instanceInfo.pid = res.pid;
instanceInfo.exitStatus = res;
sh.resetSanOptions();
crashUtils.calculateMonitorValues(options, instanceInfo, res.pid, cmd);
const deltaTime = time() - startTime;

let errorMessage = ' - ';
if (sh.fetchSanFileAfterExit(res.pid)) {
serverCrashedLocal = true;
res.status = false;
errorMessage += " Sanitizer indicated issues - ";
}

if (coreCheck &&
instanceInfo.exitStatus.hasOwnProperty('signal') &&
Expand Down
118 changes: 118 additions & 0 deletions js/client/modules/@arangodb/testutils/san-file-handler.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/* jshint strict: false, sub: true */
/* global print, arango */
'use strict';

// //////////////////////////////////////////////////////////////////////////////
// / DISCLAIMER
// /
// / Copyright 2014-2024 ArangoDB GmbH, Cologne, Germany
// / Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
// /
// / Licensed under the Business Source License 1.1 (the "License");
// / you may not use this file except in compliance with the License.
// / You may obtain a copy of the License at
// /
// / https://github.com/arangodb/arangodb/blob/devel/LICENSE
// /
// / Unless required by applicable law or agreed to in writing, software
// / distributed under the License is distributed on an "AS IS" BASIS,
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// / See the License for the specific language governing permissions and
// / limitations under the License.
// /
// / Copyright holder is ArangoDB GmbH, Cologne, Germany
// /
// / @author Wilfried Goesgens
// //////////////////////////////////////////////////////////////////////////////

const _ = require('lodash');
const fs = require('fs');
const crashUtils = require('@arangodb/testutils/crash-utils');

var regex = /[^\u0000-\u00ff]/; // Small performance gain from pre-compiling the regex
function containsDoubleByte(str) {
if (!str.length) return false;
if (str.charCodeAt(0) > 255) return true;
return regex.test(str);
}

class sanHandler {
constructor(binaryName, sanOptions, isSan, extremeVerbosity) {
this.binaryName = binaryName;
this.sanOptions = _.clone(sanOptions);
this.enabled = isSan;
this.extremeVerbosity = extremeVerbosity;
this.sanitizerLogPaths = {};
this.backup = {};
}
detectLogfiles(rootDir, tmpDir) {
if (this.enabled) {
if (containsDoubleByte(rootDir)) {
rootDir = tmpDir;
}
for (const [key, value] of Object.entries(this.sanOptions)) {
let oneLogFile = fs.join(rootDir, key.toLowerCase().split('_')[0] + '.log');
// we need the log files to contain the exe name, otherwise our code to pick them up won't find them
this.sanOptions[key]['log_exe_name'] = "true";
const origPath = this.sanOptions[key]['log_path'];
this.sanOptions[key]['log_path'] = oneLogFile;
this.sanitizerLogPaths[key] = { upstream: origPath, local: oneLogFile };
}
}
}
setSanOptions() {
if (this.enabled) {
print("Using sanOptions ", this.sanOptions);
for (const [key, value] of Object.entries(this.sanOptions)) {
let oneSet = "";
for (const [keyOne, valueOne] of Object.entries(value)) {
if (oneSet.length > 0) {
oneSet += ":";
}
let val = valueOne.replace(/,/g, '_');
oneSet += `${keyOne}=${val}`;
}
this.backup[key] = process.env[key];
process.env[key] = oneSet;
}
}
}
resetSanOptions() {
if (this.enabled) {
for (const [key, value] of Object.entries(this.backup)) {
process.env[key] = value;
}
}
}

fetchSanFileAfterExit(pid) {
if (!this.enabled) {
return false;
}
let ret = false;
for (const [key, value] of Object.entries(this.sanitizerLogPaths)) {
print("processing ", value);
const { upstream, local } = value;
let fn = `${local}.${this.binaryname}.${pid}`;
if (this.extremeVerbosity) {
print(`checking for ${fn}: ${fs.exists(fn)}`);
}
if (fs.exists(fn)) {
let content = fs.read(fn);
if (upstream) {
print("found file ", fn, " - writing file ", `${upstream}.${this.binaryName}.${this.pid}`);
fs.write(`${upstream}.${this.binaryName}.${this.pid}`, content);
}
if (content.length > 10) {
crashUtils.GDB_OUTPUT += `Report of '${this.name}' in ${fn} contains: \n`;
crashUtils.GDB_OUTPUT += content;
ret = true;
}
}
}
return ret;
}

}

exports.sanHandler = sanHandler;

0 comments on commit 25c89db

Please sign in to comment.