refactor(perf): use webdriver to execute benchmarks

- use performance log of chromedriver / appium to get timeline data for calculating metrics for benchmarks - change all benchmarks to be made of a standalone application and a protractor test that collectes timeline data - fix and simplify benchmarks - add dart2js to build - remove benchpress Closes #330
2014-12-22 17:50:10 -08:00
parent d642c6afb5
commit df4ac0dd33
66 changed files with 1146 additions and 985 deletions
@@ -1,39 +0,0 @@
-var util = require('./util');
-var path = require('path');
-var benchpress = require('angular-benchpress/lib/cli');
-var through2 = require('through2');
-var Q = require('q');
-var path = require('path');
-
-module.exports = function(gulp, plugins, config) {
-  return function() {
-    var benchmarkParentFolders = {};
-    var createBpConfStream = util.streamToPromise(
-      gulp.src(path.join(config.buildDir, config.mainHtmls))
-      .pipe(through2.obj(function(file, enc, done) {
-        file.path = path.join(path.dirname(file.path), config.configFile.name);
-        file.contents = new Buffer(config.configFile.content);
-        this.push(file);
-        benchmarkParentFolders[getParentFolder(file.path)] = true;
-        done();
-      }))
-      .pipe(gulp.dest(config.buildDir)));
-
-    return createBpConfStream.then(function() {
-      return Promise.all(Object.keys(benchmarkParentFolders).map(function(benchmarkParentPath) {
-        var defer = Q.defer();
-        benchpress.build({
-          benchmarksPath: benchmarkParentPath,
-          buildPath: path.join(benchmarkParentPath, path.join('../', config.outputFolderName))
-        }, defer.makeNodeResolver());
-        return defer.promise;
-      }));
-    });
-  };
-};
-
-function getParentFolder(file) {
-  var parts = path.dirname(file).split(path.sep);
-  parts.pop();
-  return parts.join(path.sep);
-}
@@ -2,7 +2,7 @@ module.exports = function(gulp, plugins, config) {
  return function() {
    plugins.connect.server({
      root: [__dirname+'/../../'+config.path],
-      port: 8000,
+      port: config.port,
      livereload: false,
      open: false
    })();
@@ -0,0 +1,46 @@
+var util = require('./util');
+var Q = require('q');
+var spawn = require('child_process').spawn;
+var through2 = require('through2');
+var path = require('path');
+var glob = require('glob');
+
+module.exports = function(gulp, plugins, config) {
+  return function() {
+    var webFolders = [].slice.call(glob.sync(path.join(config.src, '*/web')));
+    return nextFolder();
+
+    function nextFolder() {
+      if (!webFolders.length) {
+        return;
+      }
+      var folder = getParentFolder(webFolders.shift());
+      var destFolder = path.resolve(path.join(config.dest, path.basename(folder)));
+      return util.processToPromise(spawn(config.command, ['build', '-o', destFolder], {
+        stdio: 'inherit',
+        cwd: folder
+      })).then(function() {
+        return replaceDartWithJsScripts(gulp, destFolder);
+      }).then(nextFolder);
+    }
+  };
+};
+
+function getParentFolder(folder) {
+  var parts = folder.split(path.sep);
+  parts.pop();
+  return parts.join(path.sep);
+}
+
+function replaceDartWithJsScripts(gulp, folder) {
+  return util.streamToPromise(gulp.src(path.join(folder, '**/*.html'))
+    .pipe(through2.obj(function(file, enc, done) {
+      var content = file.contents.toString();
+      content = content.replace(/\.dart/, '.dart.js');
+      content = content.replace(/application\/dart/, 'text/javascript');
+      file.contents = new Buffer(content);
+      this.push(file);
+      done();
+    }))
+    .pipe(gulp.dest(folder)));
+}
@@ -0,0 +1,179 @@
+var webdriver = require('protractor/node_modules/selenium-webdriver');
+
+module.exports = {
+  perfLogs: perfLogs,
+  sumTimelineStats: sumTimelineStats,
+  runSimpleBenchmark: runSimpleBenchmark,
+  verifyNoErrors: verifyNoErrors,
+  printObjectAsMarkdown: printObjectAsMarkdown
+};
+
+function perfLogs() {
+  return plainLogs('performance').then(function(entries) {
+    var entriesByMethod = {};
+    entries.forEach(function(entry) {
+      var message = JSON.parse(entry.message).message;
+      var entries = entriesByMethod[message.method];
+      if (!entries) {
+        entries = entriesByMethod[message.method] = [];
+      }
+      entries.push(message.params);
+    });
+    return entriesByMethod;
+  });
+}
+
+// Needed as selenium-webdriver does not forward
+// performance logs in the correct way
+function plainLogs(type) {
+  var webdriver = require('protractor/node_modules/selenium-webdriver');
+  return browser.driver.schedule(
+      new webdriver.Command(webdriver.CommandName.GET_LOG).
+          setParameter('type', type),
+      'WebDriver.manage().logs().get(' + type + ')');
+};
+
+
+function sumTimelineStats(messages) {
+  var recordStats = {
+    script: 0,
+    gc: {
+      time: 0,
+      amount: 0
+    },
+    render: 0
+  };
+  messages.forEach(function(message) {
+    sumTimelineRecordStats(message.record, recordStats);
+  });
+  return recordStats;
+}
+
+function sumTimelineRecordStats(record, result) {
+  var summedChildrenDuration = 0;
+  if (record.children) {
+    record.children.forEach(function(child) {
+      summedChildrenDuration += sumTimelineRecordStats(child, result);
+    });
+  }
+  // in case a script forced a gc or a reflow
+  // we need to substract the gc time / reflow time
+  // from the script time!
+  var recordDuration = (record.endTime ? record.endTime - record.startTime : 0)
+    - summedChildrenDuration;
+
+  var recordSummed = true;
+  if (record.type === 'FunctionCall') {
+    result.script += recordDuration;
+  } else if (record.type === 'GCEvent') {
+    result.gc.time += recordDuration;
+    result.gc.amount += record.data.usedHeapSizeDelta;
+  } else if (record.type === 'RecalculateStyles' ||
+      record.type === 'Layout' ||
+      record.type === 'UpdateLayerTree' ||
+      record.type === 'Paint' ||
+      record.type === 'Rasterize' ||
+      record.type === 'CompositeLayers') {
+    result.render += recordDuration;
+  } else {
+    recordSummed = false;
+  }
+  if (recordSummed) {
+    return recordDuration;
+  } else {
+    return summedChildrenDuration;
+  }
+}
+
+function runSimpleBenchmark(config) {
+  var url = config.url;
+  var buttonSelectors = config.buttons;
+  // TODO: Don't use a fixed number of warmup / measure iterations,
+  // but make this dependent on the variance of the test results!
+  var warmupCount = browser.params.warmupCount;
+  var measureCount = browser.params.measureCount;
+  var name = config.name;
+
+  browser.get(url);
+  // TODO(tbosch): replace this with a proper protractor/ng2.0 integration
+  // and remove this function as well as all method calls.
+  browser.sleep(browser.params.sleepInterval)
+
+  var btns = buttonSelectors.map(function(selector) {
+    return $(selector);
+  });
+
+  multiClick(btns, warmupCount);
+  gc();
+  // empty perflogs queue
+  perfLogs();
+
+  multiClick(btns, measureCount);
+  gc();
+  return perfLogs().then(function(logs) {
+    var stats = sumTimelineStats(logs['Timeline.eventRecorded']);
+    printObjectAsMarkdown(name, stats);
+    return stats;
+  });
+}
+
+function gc() {
+  // TODO(tbosch): this only works on chrome.
+  // For iOS Safari we need an extension to appium...
+  browser.executeScript('window.gc()');
+}
+
+function multiClick(buttons, count) {
+  var actions = browser.actions();
+  for (var i=0; i<count; i++) {
+    buttons.forEach(function(button) {
+      actions.click(button);
+    });
+  }
+  actions.perform();
+}
+
+function verifyNoErrors() {
+  browser.manage().logs().get('browser').then(function(browserLog) {
+    var filteredLog = browserLog.filter(function(logEntry) {
+      return logEntry.level.value > webdriver.logging.Level.WARNING.value;
+    });
+    expect(filteredLog.length).toEqual(0);
+    if (filteredLog.length) {
+      console.log('browser console errors: ' + require('util').inspect(filteredLog));
+    }
+  });
+}
+
+function printObjectAsMarkdown(name, obj) {
+  var props = [['name']];
+  var vals = [name];
+  flattenObj(obj, [], props, vals);
+  // log header
+  var separators = [];
+  var header = props.map(function(propPath) {
+    separators.push('----');
+    return propPath.join('.');
+  }).join(' | ');
+  console.log('\n'+header);
+  console.log(separators.join(' | '));
+  console.log(vals.join(' | '));
+  console.log('\n');
+
+  function flattenObj(obj, propPathPrefix, targetProps, targetVals) {
+    for (var prop in obj) {
+      var val = obj[prop];
+      var currPropPath = propPathPrefix.concat([prop]);
+      if (val && typeof val === 'object') {
+        flattenObj(val, currPropPath, targetProps, targetVals);
+      } else {
+        targetProps.push(currPropPath);
+        var valStr = val;
+        if (typeof val === 'number') {
+          valStr = val.toFixed(2);
+        }
+        targetVals.push(valStr);
+      }
+    }
+  }
+}