Skip to content

Commit 7998c1f

Browse files
watsonwatson
authored
ci: add benchmarks (#1145)
This commit enables benchmarks for the Node.js agent. For every commit to master, Jenkins will the benchmarks located in the `test/benchmarks` directory. Currently, those benchmarks are: - `test/benchmarks/001-transaction-and-span-no-stack-trace.js`, Measures the overhead of starting and ending a transaction and a span without a stack trace - `test/benchmarks/002-transaction-and-span-overhead-realistic-size.js`, Measures the overhead of starting and ending a transaction and a span with a realistic size stack trace - `test/benchmarks/003-transaction-and-span-with-stack-trace.js`, Measures the overhead of starting and ending a transaction and a span with a simple uniform stack trace - `test/benchmarks/004-transaction.js`, Measures the overhead of starting and ending a transaction only - `test/benchmarks/005-transaction-reading-file.js`, Measures the overhead of starting and ending a transaction only while reading a file The benchmarks are using the benchmark.js benchmarking tool. For each of the benchmarks, it would be best to get the relative margin of error down below 1%, but benchmark.js doesn't support running the benchmark until a given relative margin of error threshold. So for now this commit just run each of them for 2x60 seconds (60 seconds for the benchmark it self and 60 seconds for the control), which in most cases gets them below the magic 1%. A PR has been opened to add this feature to benchmark.js: bestiejs/benchmark.js#223 Each of the benchmarks a new document is added to our Elasticsearch benchmarking cluster when running on Jenkins. To add a new benchmark, simply add a new file to the `test/benchmarks` directory. To get more info about how to run the benchmarks locally, run: npm run bench -- --help Closes #293 Closes #306
1 parent 499871a commit 7998c1f

19 files changed

+741
-0
lines changed

‎.ci/schedule-daily.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ pipeline {
4141
job: 'apm-agent-nodejs/apm-agent-nodejs-mbp/master',
4242
parameters: [
4343
booleanParam(name: 'Run_As_Master_Branch', value: true),
44+
booleanParam(name: 'bench_ci', value: false),
4445
booleanParam(name: 'doc_ci', value: true),
4546
booleanParam(name: 'tav_ci', value: true),
4647
booleanParam(name: 'test_edge_ci', value: true)

‎.ci/scripts/prepare-benchmarks-env.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
set -eo pipefail
3+
4+
# This particular configuration is required to be installed in the baremetal
5+
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
6+
export NVM_DIR="$HOME/.nvm"
7+
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
8+
command -v nvm
9+
10+
## If NODE_VERSION env variable exists then use it otherwise use node as default
11+
if [ -z "${NODE_VERSION}" ] ; then
12+
NODE_VERSION="node"
13+
fi
14+
nvm install ${NODE_VERSION}
15+
16+
set +x
17+
npm config list
18+
npm install
19+
20+
node --version
21+
npm --version

‎.ci/scripts/run-benchmarks.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
set -ueo pipefail
3+
4+
SCRIPTPATH=$(dirname "$0")
5+
source ./${SCRIPTPATH}/prepare-benchmarks-env.sh
6+
7+
RESULT_FILE=${1:-apm-agent-benchmark-results.json}
8+
9+
npm run bench:ci ${RESULT_FILE}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ test/types/transpile/index.js
1010
build
1111
coverage
1212
node_modules
13+
test/benchmarks/.tmp

‎Jenkinsfile‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ pipeline {
2828
}
2929
parameters {
3030
booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
31+
booleanParam(name: 'bench_ci', defaultValue: true, description: 'Enable benchmarks.')
3132
booleanParam(name: 'tav_ci', defaultValue: true, description: 'Enable TAV tests.')
3233
booleanParam(name: 'tests_ci', defaultValue: true, description: 'Enable tests.')
3334
booleanParam(name: 'test_edge_ci', defaultValue: true, description: 'Enable tests for edge versions of nodejs.')
@@ -269,6 +270,52 @@ pipeline {
269270
githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
270271
}
271272
}
273+
/**
274+
Run the benchmarks and store the results on ES.
275+
The result JSON files are also archive into Jenkins.
276+
*/
277+
stage('Benchmarks') {
278+
agent { label 'metal' }
279+
options { skipDefaultCheckout() }
280+
environment {
281+
HOME = "${env.WORKSPACE}"
282+
RESULT_FILE = 'apm-agent-benchmark-results.json'
283+
NODE_VERSION = '12'
284+
}
285+
when {
286+
beforeAgent true
287+
allOf {
288+
anyOf {
289+
branch 'master'
290+
tag pattern: 'v\\d+\\.\\d+\\.\\d+.*', comparator: 'REGEXP'
291+
expression { return params.Run_As_Master_Branch }
292+
}
293+
expression { return params.bench_ci }
294+
}
295+
}
296+
steps {
297+
withGithubNotify(context: 'Benchmarks', tab: 'artifacts') {
298+
dir(env.BUILD_NUMBER) {
299+
deleteDir()
300+
unstash 'source'
301+
dir(BASE_DIR){
302+
sh '.ci/scripts/run-benchmarks.sh "${RESULT_FILE}"'
303+
}
304+
}
305+
}
306+
}
307+
post {
308+
always {
309+
catchError(message: 'sendBenchmarks failed', buildResult: 'FAILURE') {
310+
sendBenchmarks(file: "${BUILD_NUMBER}/${BASE_DIR}/${RESULT_FILE}",
311+
index: 'benchmark-nodejs', archive: true)
312+
}
313+
catchError(message: 'deleteDir failed', buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
314+
deleteDir()
315+
}
316+
}
317+
}
318+
}
272319
}
273320
post {
274321
cleanup {

‎TESTING.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,3 +124,9 @@ Clean up Docker containers and volumes:
124124
```
125125
npm run docker:clean
126126
```
127+
128+
Run the benchmarks:
129+
130+
```
131+
npm run bench
132+
```

‎package.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
"test:types": "tsc --project test/types/tsconfig.json && tsc --project test/types/transpile/tsconfig.json && node test/types/transpile/index.js",
2222
"test:babel": "babel test/babel/src.js --out-file test/babel/out.js && node test/babel/out.js",
2323
"test:esm": "node --experimental-modules test/esm",
24+
"bench": "./test/benchmarks/scripts/run-benchmarks.sh",
25+
"bench:ci": "./test/benchmarks/scripts/run-benchmarks-ci.sh",
2426
"local:start": "./test/script/local-deps-start.sh",
2527
"local:stop": "./test/script/local-deps-stop.sh",
2628
"docker:start": "docker-compose -f ./test/docker-compose.yml up -d",
@@ -113,8 +115,10 @@
113115
"@types/node": "^12.0.8",
114116
"apollo-server-express": "^2.6.3",
115117
"aws-sdk": "^2.477.0",
118+
"benchmark": "^2.1.4",
116119
"bluebird": "^3.5.2",
117120
"cassandra-driver": "^4.0.0",
121+
"columnify": "^1.5.4",
118122
"commitlint-config-squash-pr": "^1.0.0",
119123
"connect": "^3.7.0",
120124
"container-info": "^1.0.1",
@@ -148,6 +152,7 @@
148152
"mysql": "^2.16.0",
149153
"mysql2": "^1.6.3",
150154
"ndjson": "^1.5.0",
155+
"numeral": "^2.0.6",
151156
"nyc": "^14.1.1",
152157
"once": "^1.4.0",
153158
"p-finally": "^1.0.0",

‎test/benchmarks/001-transaction-and-span-no-stack-trace.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
'use strict'
2+
3+
/* eslint-disable no-unused-vars, no-undef */
4+
5+
const bench = require('./utils/bench')
6+
7+
bench('transaction-and-span-no-stack-trace', {
8+
agentConf: {
9+
captureSpanStackTraces: false
10+
},
11+
setup () {
12+
var agent = this.benchmark.agent
13+
},
14+
fn (deferred) {
15+
if (agent) agent.startTransaction()
16+
setImmediate(() => {
17+
const span = agent && agent.startSpan()
18+
setImmediate(() => {
19+
if (agent) {
20+
span.end()
21+
agent.endTransaction()
22+
}
23+
setImmediate(() => {
24+
deferred.resolve()
25+
})
26+
})
27+
})
28+
}
29+
})

‎test/benchmarks/002-transaction-and-span-overhead-realistic-size.js‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
'use strict'
2+
3+
/* eslint-disable no-unused-vars, no-undef */
4+
5+
const bench = require('./utils/bench')
6+
7+
bench('transaction-and-span-overhead-realistic-size', {
8+
agentConf: {
9+
captureSpanStackTraces: true
10+
},
11+
setup () {
12+
var agent = this.benchmark.agent
13+
var callstack = this.benchmark.callstack
14+
15+
// To avoid randomness, but still generate what appears to be natural random
16+
// call stacks, number of spans etc, use a pre-defined set of numbers
17+
var numbers = [2, 5, 10, 1, 2, 21, 2, 5, 6, 9, 1, 11, 9, 8, 12]
18+
var numbersSpanIndex = 5
19+
var numbersStackLevelIndex = 0
20+
21+
function addSpan (amount, cb) {
22+
setImmediate(() => {
23+
const span = agent && agent.startSpan()
24+
setImmediate(() => {
25+
if (agent) span.end()
26+
if (--amount === 0) cb()
27+
else addSpan(amount, cb)
28+
})
29+
})
30+
}
31+
},
32+
fn (deferred) {
33+
if (agent) agent.startTransaction()
34+
const amount = numbers[numbersStackLevelIndex++ % numbers.length]
35+
callstack(amount, () => {
36+
const amount = numbers[numbersSpanIndex++ % numbers.length]
37+
addSpan(amount, () => {
38+
if (agent) agent.endTransaction()
39+
deferred.resolve()
40+
})
41+
})
42+
}
43+
})

‎test/benchmarks/003-transaction-and-span-with-stack-trace.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
'use strict'
2+
3+
/* eslint-disable no-unused-vars, no-undef */
4+
5+
const bench = require('./utils/bench')
6+
7+
bench('transaction-and-span-with-stack-trace', {
8+
agentConf: {
9+
captureSpanStackTraces: false
10+
},
11+
setup () {
12+
var agent = this.benchmark.agent
13+
},
14+
fn (deferred) {
15+
if (agent) agent.startTransaction()
16+
setImmediate(() => {
17+
const span = agent && agent.startSpan()
18+
setImmediate(() => {
19+
if (agent) {
20+
span.end()
21+
agent.endTransaction()
22+
}
23+
deferred.resolve()
24+
})
25+
})
26+
}
27+
})

0 commit comments

Comments
 (0)