From e4b1e6c62280329eda0d30e7ab1caed9a6e61252 Mon Sep 17 00:00:00 2001 From: Joey Perrott Date: Wed, 4 Mar 2020 14:37:21 -0800 Subject: [PATCH] build: Add pullapprove verification tool to dev-infra-private package (#35911) Migrates pullapprove verification tool to be available in the dev-infra-private package PR Close #35911 --- dev-infra/BUILD.bazel | 17 ++- dev-infra/cli.ts | 8 +- dev-infra/package.json | 2 +- dev-infra/pullapprove/BUILD.bazel | 19 +++ dev-infra/pullapprove/verify.ts | 215 ++++++++++++++++++++++++++++++ dev-infra/rollup.config.js | 16 +++ 6 files changed, 274 insertions(+), 3 deletions(-) create mode 100644 dev-infra/pullapprove/BUILD.bazel create mode 100644 dev-infra/pullapprove/verify.ts create mode 100644 dev-infra/rollup.config.js diff --git a/dev-infra/BUILD.bazel b/dev-infra/BUILD.bazel index 0c5c6cb3a6..11e9640a56 100644 --- a/dev-infra/BUILD.bazel +++ b/dev-infra/BUILD.bazel @@ -1,5 +1,6 @@ load("@build_bazel_rules_nodejs//:index.bzl", "pkg_npm") load("@npm_bazel_typescript//:index.bzl", "ts_library") +load("@npm_bazel_rollup//:index.bzl", "rollup_bundle") ts_library( name = "cli", @@ -7,10 +8,24 @@ ts_library( "cli.ts", ], deps = [ + "//dev-infra/pullapprove", "@npm//@types/node", ], ) +rollup_bundle( + name = "bundle", + config_file = "rollup.config.js", + entry_point = ":cli.ts", + format = "umd", + sourcemap = "hidden", + deps = [ + ":cli", + "@npm//rollup-plugin-commonjs", + "@npm//rollup-plugin-node-resolve", + ], +) + pkg_npm( name = "npm_package", srcs = [ @@ -18,6 +33,6 @@ pkg_npm( ], visibility = ["//visibility:public"], deps = [ - ":cli", + ":bundle", ], ) diff --git a/dev-infra/cli.ts b/dev-infra/cli.ts index f03de903f6..2d7086b568 100644 --- a/dev-infra/cli.ts +++ b/dev-infra/cli.ts @@ -5,10 +5,16 @@ * Use of this source code is governed by an MIT-style license that can be * found in the LICENSE file at https://angular.io/license */ +import {verify} from './pullapprove/verify'; + const args = process.argv.slice(2); + // TODO(josephperrott): Set up proper cli flag/command handling -switch(args[0]) { +switch (args[0]) { + case 'pullapprove:verify': + verify(); + break; default: console.info('No commands were matched'); } diff --git a/dev-infra/package.json b/dev-infra/package.json index c289865688..6c14408cc8 100644 --- a/dev-infra/package.json +++ b/dev-infra/package.json @@ -5,6 +5,6 @@ "license": "MIT", "private": true, "bin": { - "ng-dev": "./cli.js" + "ng-dev": "./bundle.js" } } diff --git a/dev-infra/pullapprove/BUILD.bazel b/dev-infra/pullapprove/BUILD.bazel new file mode 100644 index 0000000000..7da7be1484 --- /dev/null +++ b/dev-infra/pullapprove/BUILD.bazel @@ -0,0 +1,19 @@ +load("@npm_bazel_typescript//:index.bzl", "ts_library") + +ts_library( + name = "pullapprove", + srcs = [ + "verify.ts", + ], + visibility = ["//dev-infra:__subpackages__"], + deps = [ + "@npm//@types/minimatch", + "@npm//@types/node", + "@npm//@types/shelljs", + "@npm//@types/yaml", + "@npm//minimatch", + "@npm//shelljs", + "@npm//tslib", + "@npm//yaml", + ], +) diff --git a/dev-infra/pullapprove/verify.ts b/dev-infra/pullapprove/verify.ts new file mode 100644 index 0000000000..e43c4f8287 --- /dev/null +++ b/dev-infra/pullapprove/verify.ts @@ -0,0 +1,215 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {readFileSync} from 'fs'; +import {IMinimatch, Minimatch} from 'minimatch'; +import * as path from 'path'; +import {cd, exec, set} from 'shelljs'; +import {parse as parseYaml} from 'yaml'; + +interface GlobMatcher { + group: string; + glob: string; + matcher: IMinimatch; + matchCount: number; +} + +export function verify() { + // Exit early on shelljs errors + set('-e'); + + // Regex Matcher for contains_any_globs conditions + const CONTAINS_ANY_GLOBS_REGEX = /^'([^']+)',?$/; + // Full path of the angular project directory + const ANGULAR_PROJECT_DIR = process.cwd(); + // Change to the Angular project directory + cd(ANGULAR_PROJECT_DIR); + + // Whether to log verbosely + const VERBOSE_MODE = process.argv.includes('-v'); + // Full path to PullApprove config file + const PULL_APPROVE_YAML_PATH = path.resolve(ANGULAR_PROJECT_DIR, '.pullapprove.yml'); + // All relative path file names in the git repo, this is retrieved using git rather + // that a glob so that we only get files that are checked in, ignoring things like + // node_modules, .bazelrc.user, etc + const ALL_FILES = exec('git ls-tree --full-tree -r --name-only HEAD', {silent: true}) + .trim() + .split('\n') + .filter((_: string) => !!_); + if (!ALL_FILES.length) { + console.error( + `No files were found to be in the git tree, did you run this command from \n` + + `inside the angular repository?`); + process.exit(1); + } + + /** Gets the glob matching information from each group's condition. */ + function getGlobMatchersFromCondition( + groupName: string, condition: string): [GlobMatcher[], string[]] { + const trimmedCondition = condition.trim(); + const globMatchers: GlobMatcher[] = []; + const badConditionLines: string[] = []; + + // If the condition starts with contains_any_globs, evaluate all of the globs + if (trimmedCondition.startsWith('contains_any_globs')) { + trimmedCondition.split('\n') + .slice(1, -1) + .map(glob => { + const trimmedGlob = glob.trim(); + const match = trimmedGlob.match(CONTAINS_ANY_GLOBS_REGEX); + if (!match) { + badConditionLines.push(trimmedGlob); + return ''; + } + return match[1]; + }) + .filter(globString => !!globString) + .forEach(globString => globMatchers.push({ + group: groupName, + glob: globString, + matcher: new Minimatch(globString, {dot: true}), + matchCount: 0, + })); + } + return [globMatchers, badConditionLines]; + } + + /** Create logs for each review group. */ + function logGroups(groups: Map>) { + Array.from(groups.entries()).sort().forEach(([groupName, globs]) => { + console.groupCollapsed(groupName); + Array.from(globs.values()) + .sort((a, b) => b.matchCount - a.matchCount) + .forEach(glob => console.info(`${glob.glob} - ${glob.matchCount}`)); + console.groupEnd(); + }); + } + + /** Logs a header within a text drawn box. */ + function logHeader(...params: string[]) { + const totalWidth = 80; + const fillWidth = totalWidth - 2; + const headerText = params.join(' ').substr(0, fillWidth); + const leftSpace = Math.ceil((fillWidth - headerText.length) / 2); + const rightSpace = fillWidth - leftSpace - headerText.length; + const fill = (count: number, content: string) => content.repeat(count); + + console.info(`┌${fill(fillWidth, '─')}┐`); + console.info(`│${fill(leftSpace, ' ')}${headerText}${fill(rightSpace, ' ')}│`); + console.info(`└${fill(fillWidth, '─')}┘`); + } + + /** Runs the pull approve verification check on provided files. */ + function runVerification(files: string[]) { + // All of the globs created for each group's conditions. + const allGlobs: GlobMatcher[] = []; + // The pull approve config file. + const pullApprove = readFileSync(PULL_APPROVE_YAML_PATH, {encoding: 'utf8'}); + // All of the PullApprove groups, parsed from the PullApprove yaml file. + const parsedPullApproveGroups = + parseYaml(pullApprove).groups as{[key: string]: {conditions: string}}; + // All files which were found to match a condition in PullApprove. + const matchedFiles = new Set(); + // All files which were not found to match a condition in PullApprove. + const unmatchedFiles = new Set(); + // All PullApprove groups which matched at least one file. + const matchedGroups = new Map>(); + // All PullApprove groups which did not match at least one file. + const unmatchedGroups = new Map>(); + // All condition lines which were not able to be correctly parsed, by group. + const badConditionLinesByGroup = new Map(); + // Total number of condition lines which were not able to be correctly parsed. + let badConditionLineCount = 0; + + // Get all of the globs from the PullApprove group conditions. + Object.entries(parsedPullApproveGroups).forEach(([groupName, group]) => { + for (const condition of group.conditions) { + const [matchers, badConditions] = getGlobMatchersFromCondition(groupName, condition); + if (badConditions.length) { + badConditionLinesByGroup.set(groupName, badConditions); + badConditionLineCount += badConditions.length; + } + allGlobs.push(...matchers); + } + }); + + if (badConditionLineCount) { + console.info(`Discovered ${badConditionLineCount} parsing errors in PullApprove conditions`); + console.info(`Attempted parsing using: ${CONTAINS_ANY_GLOBS_REGEX}`); + console.info(); + console.info(`Unable to properly parse the following line(s) by group:`); + badConditionLinesByGroup.forEach((badConditionLines, groupName) => { + console.info(`- ${groupName}:`); + badConditionLines.forEach(line => console.info(` ${line}`)); + }); + console.info(); + console.info( + `Please correct the invalid conditions, before PullApprove verification can be completed`); + process.exit(1); + } + + // Check each file for if it is matched by a PullApprove condition. + for (let file of files) { + const matched = allGlobs.filter(glob => glob.matcher.match(file)); + matched.length ? matchedFiles.add(file) : unmatchedFiles.add(file); + matched.forEach(glob => glob.matchCount++); + } + + // Add each glob for each group to a map either matched or unmatched. + allGlobs.forEach(glob => { + const groups = glob.matchCount ? matchedGroups : unmatchedGroups; + const globs = groups.get(glob.group) || new Map(); + // Set the globs map in the groups map + groups.set(glob.group, globs); + // Set the glob in the globs map + globs.set(glob.glob, glob); + }); + + // PullApprove is considered verified if no files or groups are found to be unsed. + const verificationSucceeded = !(unmatchedFiles.size || unmatchedGroups.size); + + /** + * Overall result + */ + logHeader('Result'); + if (verificationSucceeded) { + console.info('PullApprove verification succeeded!'); + } else { + console.info(`PullApprove verification failed.\n`); + console.info(`Please update '.pullapprove.yml' to ensure that all necessary`); + console.info(`files/directories have owners and all patterns that appear in`); + console.info(`the file correspond to actual files/directories in the repo.`); + } + /** + * File by file Summary + */ + logHeader('PullApprove file match results'); + console.groupCollapsed(`Matched Files (${matchedFiles.size} files)`); + VERBOSE_MODE && matchedFiles.forEach(file => console.info(file)); + console.groupEnd(); + console.groupCollapsed(`Unmatched Files (${unmatchedFiles.size} files)`); + unmatchedFiles.forEach(file => console.info(file)); + console.groupEnd(); + + /** + * Group by group Summary + */ + logHeader('PullApprove group matches'); + console.groupCollapsed(`Matched Groups (${matchedGroups.size} groups)`); + VERBOSE_MODE && logGroups(matchedGroups); + console.groupEnd(); + console.groupCollapsed(`Unmatched Groups (${unmatchedGroups.size} groups)`); + logGroups(unmatchedGroups); + console.groupEnd(); + + // Provide correct exit code based on verification success. + process.exit(verificationSucceeded ? 0 : 1); + } + + + runVerification(ALL_FILES); +} diff --git a/dev-infra/rollup.config.js b/dev-infra/rollup.config.js new file mode 100644 index 0000000000..8c8713475f --- /dev/null +++ b/dev-infra/rollup.config.js @@ -0,0 +1,16 @@ +const node = require('rollup-plugin-node-resolve'); +const commonjs = require('rollup-plugin-commonjs'); + +module.exports = { + external: ['shelljs', 'minimatch', 'yaml'], + preferBuiltins: true, + output: { + banner: "#!/usr/bin/env node", + }, + plugins: [ + node({ + mainFields: ['browser', 'es2015', 'module', 'jsnext:main', 'main'], + }), + commonjs(), + ], +};