[v8-dev] performance benchmark of async-design-patterns - recursive-callbacks vs. promises vs. async/await

# kai zhu (a month ago)

i ran the test-script on travis-ci against various v8/nodejs versions (including v8-6.6/node-10) [1] [2].

here are the raw data and visualization [3] of recursive-callback/promise/async-await vs v8/nodejs-versions. not-surprisingly, recursive-callbacks are consistently faster, but only a small margin (7-13%) than promises across historical nodejs versions on travis-ci.

of note:

  • the standard deviation error-bars should be taken with a grain of salt, as the travis-ci rerun indicates higher inconsistency
  • the low numbers from node-0.9 and node-0.10 are meaningless, as the sockets seem to constantly hang and timeout according to travis logs.
  • note the sudden drop for the travis-ci rerun of node-9.11.1 (don’t have explanation, and the travis-logs shows the low numbers are quite consistent within the rerun)
  • node-10.0.0 is slower than node-8.11.1, including the travis rerun
  • async/await is only available on node-7 and higher

[1] travis-ci build logs #20 travis-ci.org/kaizhu256/node-performance-recursiveCallback-vs-promise-vs-asyncAwait/builds/372961240, travis-ci.org/kaizhu256/node-performance-recursiveCallback-vs-promise-vs-asyncAwait/builds/372961240

[2] travis-ci build logs #21 (rerun to verify consistency) travis-ci.org/kaizhu256/node-performance-recursiveCallback-vs-promise-vs-asyncAwait/builds/373134575, travis-ci.org/kaizhu256/node-performance-recursiveCallback-vs-promise-vs-asyncAwait/builds/373134575

[3] visualisation kaizhu256.github.io/node-performance-recursiveCallback-vs-promise-vs-asyncAwait/index.html, kaizhu256.github.io/node-performance-recursiveCallback-vs-promise-vs-asyncAwait/index.html

[
    {
        "version": "v8-3.14.5.8<br>node-0.9.12",
        "clientHttpRequestWithRecursiveCallback": "456 (83 sigma)",
        "recursiveCallbackVsPromiseRatio": null
    },
    {
        "version": "v8-3.14.5.11<br>node-0.10.48",
        "clientHttpRequestWithRecursiveCallback": "49 (1 sigma)",
        "recursiveCallbackVsPromiseRatio": null
    },
    {
        "version": "v8-3.28.73<br>node-0.11.16",
        "clientHttpRequestWithRecursiveCallback": "1997 (41 sigma)",
        "clientHttpRequestWithPromise": "1854 (67 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0771305285868393
    },
    {
        "version": "v8-3.28.71.20<br>node-0.12.18",
        "clientHttpRequestWithRecursiveCallback": "1964 (14 sigma)",
        "clientHttpRequestWithPromise": "1829 (15 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0738108255877528
    },
    {
        "version": "v8-4.5.103.53<br>node-4.9.1",
        "clientHttpRequestWithRecursiveCallback": "2355 (31 sigma)",
        "clientHttpRequestWithPromise": "2170 (22 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0852534562211982
    },
    {
        "version": "v8-4.6.85.32<br>node-5.12.0",
        "clientHttpRequestWithRecursiveCallback": "2471 (69 sigma)",
        "clientHttpRequestWithPromise": "2319 (76 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0655454937473048
    },
    {
        "version": "v8-5.1.281.111<br>node-6.14.1",
        "clientHttpRequestWithRecursiveCallback": "2589 (37 sigma)",
        "clientHttpRequestWithPromise": "2374 (38 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0905644481887111
    },
    {
        "version": "v8-5.5.372.43<br>node-7.10.1",
        "clientHttpRequestWithRecursiveCallback": "3238 (53 sigma)",
        "clientHttpRequestWithPromise": "2846 (35 sigma)",
        "clientHttpRequestWithAsyncAwait": "2860 (31 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.1377371749824314
    },
    {
        "version": "v8-6.2.414.50<br>node-8.11.1",
        "clientHttpRequestWithRecursiveCallback": "4699 (74 sigma)",
        "clientHttpRequestWithPromise": "4305 (88 sigma)",
        "clientHttpRequestWithAsyncAwait": "4294 (52 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.091521486643438
    },
    {
        "version": "v8-6.2.414.46-node.23<br>node-9.11.1",
        "clientHttpRequestWithRecursiveCallback": "4809 (80 sigma)",
        "clientHttpRequestWithPromise": "4423 (94 sigma)",
        "clientHttpRequestWithAsyncAwait": "4404 (81 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.087271082975356
    },
    {
        "version": "v8-6.6.346.24-node.5<br>node-10.0.0",
        "clientHttpRequestWithRecursiveCallback": "4100 (69 sigma)",
        "clientHttpRequestWithPromise": "3836 (98 sigma)",
        "clientHttpRequestWithAsyncAwait": "3837 (46 sigma)",
        "recursiveCallbackVsPromiseRatio": 1.0688216892596454
    },
    {
        "version": "v8-3.14.5.8<br>node-0.9.12",
        "clientHttpRequestWithRecursiveCallback-rerun": "477 (46 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": null
    },
    {
        "version": "v8-3.14.5.11<br>node-0.10.48",
        "clientHttpRequestWithRecursiveCallback-rerun": "48 (2 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": null
    },
    {
        "version": "v8-3.28.73<br>node-0.11.16",
        "clientHttpRequestWithRecursiveCallback-rerun": "1887 (25 sigma)",
        "clientHttpRequestWithPromise-rerun": "1763 (25 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0771305285868393
    },
    {
        "version": "v8-3.28.71.20<br>node-0.12.18",
        "clientHttpRequestWithRecursiveCallback-rerun": "2494 (23 sigma)",
        "clientHttpRequestWithPromise-rerun": "2282 (35 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0738108255877528
    },
    {
        "version": "v8-4.5.103.53<br>node-4.9.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "3246 (99 sigma)",
        "clientHttpRequestWithPromise-rerun": "3008 (91 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0852534562211982
    },
    {
        "version": "v8-4.6.85.32<br>node-5.12.0",
        "clientHttpRequestWithRecursiveCallback-rerun": "2709 (26 sigma)",
        "clientHttpRequestWithPromise-rerun": "2507 (30 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0655454937473048
    },
    {
        "version": "v8-5.1.281.111<br>node-6.14.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "2401 (38 sigma)",
        "clientHttpRequestWithPromise-rerun": "2193 (55 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0905644481887111
    },
    {
        "version": "v8-5.5.372.43<br>node-7.10.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "4065 (83 sigma)",
        "clientHttpRequestWithPromise-rerun": "3565 (91 sigma)",
        "clientHttpRequestWithAsyncAwait-rerun": "3620 (89 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.1377371749824314
    },
    {
        "version": "v8-6.2.414.50<br>node-8.11.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "4915 (56 sigma)",
        "clientHttpRequestWithPromise-rerun": "4537 (49 sigma)",
        "clientHttpRequestWithAsyncAwait-rerun": "4476 (44 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.091521486643438
    },
    {
        "version": "v8-6.2.414.46-node.23<br>node-9.11.1",
        "clientHttpRequestWithRecursiveCallback-rerun": "2141 (67 sigma)",
        "clientHttpRequestWithPromise-rerun": "2011 (67 sigma)",
        "clientHttpRequestWithAsyncAwait-rerun": "2019 (43 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.087271082975356
    },
    {
        "version": "v8-6.6.346.24-node.5<br>node-10.0.0",
        "clientHttpRequestWithRecursiveCallback-rerun": "3609 (84 sigma)",
        "clientHttpRequestWithPromise-rerun": "3350 (47 sigma)",
        "clientHttpRequestWithAsyncAwait-rerun": "3353 (65 sigma)",
        "recursiveCallbackVsPromiseRatio-rerun": 1.0688216892596454
    }
]

kai zhu kaizhu256 at gmail.com