refactor(dev-infra): set up new method for checking range of commits (#41341)

Check a range of commits by retrieving the log files to be parsed with the expected format for the parser. This change is in part of a larger set of changes making the process for obtaining and parsing commits for release note creation and message validation consistent. This consistency will make it easier to debug as well as ease the design of tooling which is built on top of these processes. PR Close #41341
2021-03-24 16:17:15 -07:00
parent 18bc9ffb51
commit 381ea9d7d4
14 changed files with 208 additions and 131 deletions
@@ -12,11 +12,13 @@ ts_library(
    deps = [
        "//dev-infra/utils",
        "@npm//@types/conventional-commits-parser",
+        "@npm//@types/git-raw-commits",
        "@npm//@types/inquirer",
        "@npm//@types/node",
        "@npm//@types/shelljs",
        "@npm//@types/yargs",
        "@npm//conventional-commits-parser",
+        "@npm//git-raw-commits",
        "@npm//inquirer",
        "@npm//shelljs",
        "@npm//yargs",
@@ -31,7 +31,7 @@ export function getCommitMessageConfig() {
  return config as Required<typeof config>;
 }

-/** Scope requirement level to be set for each commit type.  */
+/** Scope requirement level to be set for each commit type. */
 export enum ScopeRequirement {
  Required,
  Optional,
@@ -8,8 +8,6 @@

 import {Commit as ParsedCommit, Options, sync as parse} from 'conventional-commits-parser';

-import {exec} from '../utils/shelljs';
-

 /** A parsed commit, containing the information needed to validate the commit. */
 export interface Commit {
@@ -43,6 +41,30 @@ export interface Commit {
  isRevert: boolean;
 }

+/**
+ * A list of tuples expressing the fields to extract from each commit log entry. The tuple contains
+ * two values, the first is the key for the property and the second is the template shortcut for the
+ * git log command.
+ */
+const commitFields = {
+  hash: '%H',
+  shortHash: '%h',
+  author: '%aN',
+};
+/** The additional fields to be included in commit log entries for parsing. */
+export type CommitFields = typeof commitFields;
+/** The commit fields described as git log format entries for parsing. */
+export const commitFieldsAsFormat = (fields: CommitFields) => {
+  return Object.entries(fields).map(([key, value]) => `%n-${key}-%n${value}`).join('');
+};
+/**
+ * The git log format template to create git log entries for parsing.
+ *
+ * The conventional commits parser expects to parse the standard git log raw body (%B) into its
+ * component parts. Additionally it will parse additional fields with keys defined by
+ * `-{key name}-` separated by new lines.
+ * */
+export const gitLogFormatForParsing = `%B${commitFieldsAsFormat(commitFields)}`;
 /** Markers used to denote the start of a note section in a commit. */
 enum NoteSections {
  BREAKING_CHANGE = 'BREAKING CHANGE',
@@ -87,7 +109,9 @@ const parseOptions: Options&{notesPattern: (keywords: string) => RegExp} = {


 /** Parse a full commit message into its composite parts. */
-export function parseCommitMessage(fullText: string): Commit {
+export function parseCommitMessage(fullText: string|Buffer): Commit {
+  // Ensure the fullText symbol is a `string`, even if a Buffer was provided.
+  fullText = fullText.toString();
  /** The commit message text with the fixup and squash markers stripped out. */
  const strippedCommitMsg = fullText.replace(FIXUP_PREFIX_RE, '')
                                .replace(SQUASH_PREFIX_RE, '')
@@ -126,30 +150,3 @@ export function parseCommitMessage(fullText: string): Commit {
    isRevert: REVERT_PREFIX_RE.test(fullText),
  };
 }
-
-/** Retrieve and parse each commit message in a provide range. */
-export function parseCommitMessagesForRange(range: string): Commit[] {
-  /** A random number used as a split point in the git log result. */
-  const randomValueSeparator = `${Math.random()}`;
-  /**
-   * Custom git log format that provides the commit header and body, separated as expected with the
-   * custom separator as the trailing value.
-   */
-  const gitLogFormat = `%s%n%n%b${randomValueSeparator}`;
-
-  // Retrieve the commits in the provided range.
-  const result = exec(`git log --reverse --format=${gitLogFormat} ${range}`);
-  if (result.code) {
-    throw new Error(`Failed to get all commits in the range:\n  ${result.stderr}`);
-  }
-
-  return result
-      // Separate the commits from a single string into individual commits.
-      .split(randomValueSeparator)
-      // Remove extra space before and after each commit message.
-      .map(l => l.trim())
-      // Remove any superfluous lines which remain from the split.
-      .filter(line => !!line)
-      // Parse each commit message.
-      .map(commit => parseCommitMessage(commit));
-}
@@ -50,11 +50,11 @@ async function handler({fileEnvVariable, file, source}: Arguments<RestoreCommitM
  }

  throw new Error(
-      'No file path and commit message source provide.  Provide values via positional command ' +
+      'No file path and commit message source provide. Provide values via positional command ' +
      'arguments, or via the --file-env-variable flag');
 }

-/** yargs command module describing the command.  */
+/** yargs command module describing the command. */
 export const RestoreCommitMessageModule: CommandModule<{}, RestoreCommitMessageOptions> = {
  handler,
  builder,
@@ -0,0 +1,32 @@
+/**
+ * @license
+ * Copyright Google LLC All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import * as gitCommits_ from 'git-raw-commits';
+
+import {Commit, gitLogFormatForParsing, parseCommitMessage} from './parse';
+
+// Set `gitCommits` as this imported value to address "Cannot call a namespace" error.
+const gitCommits = gitCommits_;
+
+
+/**
+ * Find all commits within the given range and return an object describing those.
+ */
+export function getCommitsInRange(from: string, to: string = 'HEAD'): Promise<Commit[]> {
+  return new Promise((resolve, reject) => {
+    /** List of parsed commit objects. */
+    const commits: Commit[] = [];
+    /** Stream of raw git commit strings in the range provided. */
+    const commitStream = gitCommits({from, to, format: gitLogFormatForParsing});
+
+    // Accumulate the parsed commits for each commit from the Readable stream into an array, then
+    // resolve the promise with the array when the Readable stream ends.
+    commitStream.on('data', (commit: Buffer) => commits.push(parseCommitMessage(commit)));
+    commitStream.on('error', (err: Error) => reject(err));
+    commitStream.on('end', () => resolve(commits));
+  });
+}
@@ -53,7 +53,7 @@ async function handler({error, file, fileEnvVariable}: Arguments<ValidateFileOpt
  validateFile(filePath, error);
 }

-/** yargs command module describing the command.  */
+/** yargs command module describing the command. */
 export const ValidateFileModule: CommandModule<{}, ValidateFileOptions> = {
  handler,
  builder,
@@ -14,21 +14,27 @@ import {validateCommitRange} from './validate-range';


 export interface ValidateRangeOptions {
-  range: string;
+  startingRef: string;
+  endingRef: string;
 }

 /** Builds the command. */
 function builder(yargs: Argv) {
-  return yargs.option('range', {
-    description: 'The range of commits to check, e.g. --range abc123..xyz456',
-    demandOption: '  A range must be provided, e.g. --range abc123..xyz456',
-    type: 'string',
-    requiresArg: true,
-  });
+  return yargs
+      .positional('startingRef', {
+        description: 'The first ref in the range to select',
+        type: 'string',
+        demandOption: true,
+      })
+      .positional('endingRef', {
+        description: 'The last ref in the range to select',
+        type: 'string',
+        default: 'HEAD',
+      });
 }

 /** Handles the command. */
-async function handler({range}: Arguments<ValidateRangeOptions>) {
+async function handler({startingRef, endingRef}: Arguments<ValidateRangeOptions>) {
  // If on CI, and no pull request number is provided, assume the branch
  // being run on is an upstream branch.
  if (process.env['CI'] && process.env['CI_PULL_REQUEST'] === 'false') {
@@ -38,13 +44,13 @@ async function handler({range}: Arguments<ValidateRangeOptions>) {
    info(`Skipping check of provided commit range`);
    return;
  }
-  validateCommitRange(range);
+  await validateCommitRange(startingRef, endingRef);
 }

-/** yargs command module describing the command.  */
+/** yargs command module describing the command. */
 export const ValidateRangeModule: CommandModule<{}, ValidateRangeOptions> = {
  handler,
  builder,
-  command: 'validate-range',
+  command: 'validate-range <starting-ref> [ending-ref]',
  describe: 'Validate a range of commit messages',
 };
@@ -5,8 +5,9 @@
 * Use of this source code is governed by an MIT-style license that can be
 * found in the LICENSE file at https://angular.io/license
 */
-import {error, info} from '../../utils/console';
-import {Commit, parseCommitMessagesForRange} from '../parse';
+import {error, green, info, red} from '../../utils/console';
+import {Commit} from '../parse';
+import {getCommitsInRange} from '../utils';
 import {printValidationErrors, validateCommitMessage, ValidateCommitMessageOptions} from '../validate';

 // Whether the provided commit is a fixup commit.
@@ -16,12 +17,13 @@ const isNonFixup = (commit: Commit) => !commit.isFixup;
 const extractCommitHeader = (commit: Commit) => commit.header;

 /** Validate all commits in a provided git commit range. */
-export function validateCommitRange(range: string) {
+export async function validateCommitRange(from: string, to: string) {
  /** A list of tuples of the commit header string and a list of error messages for the commit. */
  const errors: [commitHeader: string, errors: string[]][] = [];
+
  /** A list of parsed commit messages from the range. */
-  const commits = parseCommitMessagesForRange(range);
-  info(`Examining ${commits.length} commit(s) in the provided range: ${range}`);
+  const commits = await getCommitsInRange(from, to);
+  info(`Examining ${commits.length} commit(s) in the provided range: ${from}..${to}`);

  /**
   * Whether all commits in the range are valid, commits are allowed to be fixup commits for other
@@ -32,7 +34,7 @@ export function validateCommitRange(range: string) {
      disallowSquash: true,
      nonFixupCommitHeaders: isNonFixup(commit) ?
          undefined :
-          commits.slice(0, i).filter(isNonFixup).map(extractCommitHeader)
+          commits.slice(i + 1).filter(isNonFixup).map(extractCommitHeader)
    };
    const {valid, errors: localErrors} = validateCommitMessage(commit, options);
    if (localErrors.length) {
@@ -42,9 +44,9 @@ export function validateCommitRange(range: string) {
  });

  if (allCommitsInRangeValid) {
-    info('√  All commit messages in range valid.');
+    info(green('√  All commit messages in range valid.'));
  } else {
-    error('✘  Invalid commit message');
+    error(red('✘  Invalid commit message'));
    errors.forEach(([header, validationErrors]) => {
      error.group(header);
      printValidationErrors(validationErrors);