honeymoose/Angular-Cn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(compiler): define interfaces for each lexer token (#42062)

Browse Source 
These token interfaces will make it easier to reason about tokens in the
parser and in specs.

Previously, it was never clear what items could appear in the `parts`
array of a token given a particular `TokenType`. Now, each token interface
declares a labelled tuple for the parts, which helps to document the token
better.

PR Close #42062
This commit is contained in:
Pete Bacon Darwin
2021-08-02 09:53:13 -07:00
committed by atscott
parent f08516db09
commit 9b3d4f5575
8 changed files with 1099 additions and 951 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/compiler/src/i18n/i18n_parser.ts
+5 -5
View File
@@ -11,7 +11,7 @@ import {Parser as ExpressionParser} from '../expression_parser/parser';
import * as html from '../ml_parser/ast';
import {getHtmlTagDefinition} from '../ml_parser/html_tags';
import {InterpolationConfig} from '../ml_parser/interpolation_config';
import {Token, TokenType} from '../ml_parser/lexer';
import {InterpolatedAttributeToken, InterpolatedTextToken, TokenType} from '../ml_parser/tokens';
import {ParseSourceSpan} from '../parse_util';
import * as i18n from './i18n_ast';
@@ -163,16 +163,16 @@ class _I18nVisitor implements html.Visitor {
}
/**
* Split the, potentially interpolated, text up into text and placeholder pieces.
* Convert, text and interpolated tokens up into text and placeholder pieces.
*
* @param text The potentially interpolated string to be split.
* @param tokens The text and interpolated tokens.
* @param sourceSpan The span of the whole of the `text` string.
* @param context The current context of the visitor, used to compute and store placeholders.
* @param previousI18n Any i18n metadata associated with this `text` from a previous pass.
*/
private _visitTextWithInterpolation(
tokens: Token[], sourceSpan: ParseSourceSpan, context: I18nMessageVisitorContext,
previousI18n: i18n.I18nMeta|undefined): i18n.Node {
tokens: (InterpolatedTextToken|InterpolatedAttributeToken)[], sourceSpan: ParseSourceSpan,
context: I18nMessageVisitorContext, previousI18n: i18n.I18nMeta|undefined): i18n.Node {
// Return a sequence of `Text` and `Placeholder` nodes grouped in a `Container`.
const nodes: i18n.Node[] = [];
for (const token of tokens) {
packages/compiler/src/ml_parser/ast.ts
+4 -3
View File
@@ -9,7 +9,7 @@
import {AstPath} from '../ast_path';
import {I18nMeta} from '../i18n/i18n_ast';
import {ParseSourceSpan} from '../parse_util';
import {Token} from './lexer';
import {InterpolatedAttributeToken, InterpolatedTextToken} from './tokens';
interface BaseNode {
sourceSpan: ParseSourceSpan;
@@ -25,7 +25,8 @@ export abstract class NodeWithI18n implements BaseNode {
export class Text extends NodeWithI18n {
constructor(
public value: string, sourceSpan: ParseSourceSpan, public tokens: Token[], i18n?: I18nMeta) {
public value: string, sourceSpan: ParseSourceSpan, public tokens: InterpolatedTextToken[],
i18n?: I18nMeta) {
super(sourceSpan, i18n);
}
override visit(visitor: Visitor, context: any): any {
@@ -58,7 +59,7 @@ export class Attribute extends NodeWithI18n {
constructor(
public name: string, public value: string, sourceSpan: ParseSourceSpan,
readonly keySpan: ParseSourceSpan|undefined, public valueSpan: ParseSourceSpan|undefined,
public valueTokens: Token[]|undefined, i18n: I18nMeta|undefined) {
public valueTokens: InterpolatedAttributeToken[]|undefined, i18n: I18nMeta|undefined) {
super(sourceSpan, i18n);
}
override visit(visitor: Visitor, context: any): any {
packages/compiler/src/ml_parser/html_whitespaces.ts
+5 -5
View File
@@ -8,8 +8,8 @@
import * as html from './ast';
import {NGSP_UNICODE} from './entities';
import {Token, TokenType} from './lexer';
import {ParseTreeResult} from './parser';
import {TextToken, TokenType} from './tokens';
export const PRESERVE_WS_ATTR_NAME = 'ngPreserveWhitespaces';
@@ -77,8 +77,8 @@ export class WhitespaceVisitor implements html.Visitor {
if (isNotBlank || hasExpansionSibling) {
// Process the whitespace in the tokens of this Text node
const tokens = text.tokens.map(
token => token.type === TokenType.TEXT ? createTextTokenAfterWhitespaceProcessing(token) :
token);
token =>
token.type === TokenType.TEXT ? createWhitespaceProcessedTextToken(token) : token);
// Process the whitespace of the value of this Text node
const value = processWhitespace(text.value);
return new html.Text(value, text.sourceSpan, tokens, text.i18n);
@@ -100,8 +100,8 @@ export class WhitespaceVisitor implements html.Visitor {
}
}
function createTextTokenAfterWhitespaceProcessing(token: Token): Token {
return new Token(token.type, [processWhitespace(token.parts[0])], token.sourceSpan);
function createWhitespaceProcessedTextToken({type, parts, sourceSpan}: TextToken): TextToken {
return {type, parts: [processWhitespace(parts[0])], sourceSpan};
}
function processWhitespace(text: string): string {
packages/compiler/src/ml_parser/lexer.ts
+16 -43
View File
@@ -12,39 +12,7 @@ import {NAMED_ENTITIES} from './entities';
import {DEFAULT_INTERPOLATION_CONFIG, InterpolationConfig} from './interpolation_config';
import {TagContentType, TagDefinition} from './tags';
export enum TokenType {
TAG_OPEN_START,
TAG_OPEN_END,
TAG_OPEN_END_VOID,
TAG_CLOSE,
INCOMPLETE_TAG_OPEN,
TEXT,
ESCAPABLE_RAW_TEXT,
RAW_TEXT,
INTERPOLATION,
ENCODED_ENTITY,
COMMENT_START,
COMMENT_END,
CDATA_START,
CDATA_END,
ATTR_NAME,
ATTR_QUOTE,
ATTR_VALUE_TEXT,
ATTR_VALUE_INTERPOLATION,
DOC_TYPE,
EXPANSION_FORM_START,
EXPANSION_CASE_VALUE,
EXPANSION_CASE_EXP_START,
EXPANSION_CASE_EXP_END,
EXPANSION_FORM_END,
EOF
}
export class Token {
constructor(
public type: TokenType|null, public parts: string[], public sourceSpan: ParseSourceSpan) {}
}
import {IncompleteTagOpenToken, TagOpenStartToken, Token, TokenType} from './tokens';
export class TokenError extends ParseError {
constructor(errorMsg: string, public tokenType: TokenType|null, span: ParseSourceSpan) {
@@ -290,9 +258,12 @@ class _Tokenizer {
'Programming error - attempted to end a token which has no token type', null,
this._cursor.getSpan(this._currentTokenStart));
}
const token = new Token(
this._currentTokenType, parts,
(end ?? this._cursor).getSpan(this._currentTokenStart, this._leadingTriviaCodePoints));
const token = {
type: this._currentTokenType,
parts,
sourceSpan:
(end ?? this._cursor).getSpan(this._currentTokenStart, this._leadingTriviaCodePoints),
} as Token;
this.tokens.push(token);
this._currentTokenStart = null;
this._currentTokenType = null;
@@ -527,7 +498,7 @@ class _Tokenizer {
private _consumeTagOpen(start: CharacterCursor) {
let tagName: string;
let prefix: string;
let openTagToken: Token|undefined;
let openTagToken: TagOpenStartToken|IncompleteTagOpenToken|undefined;
try {
if (!chars.isAsciiLetter(this._cursor.peek())) {
throw this._createError(
@@ -590,10 +561,10 @@ class _Tokenizer {
this._endToken([prefix, tagName]);
}
private _consumeTagOpenStart(start: CharacterCursor) {
private _consumeTagOpenStart(start: CharacterCursor): TagOpenStartToken {
this._beginToken(TokenType.TAG_OPEN_START, start);
const parts = this._consumePrefixAndName();
return this._endToken(parts);
return this._endToken(parts) as TagOpenStartToken;
}
private _consumeAttributeName() {
@@ -764,7 +735,7 @@ class _Tokenizer {
*/
private _consumeInterpolation(
interpolationTokenType: TokenType, interpolationStart: CharacterCursor,
prematureEndPredicate: (() => boolean)|null) {
prematureEndPredicate: (() => boolean)|null): void {
const parts: string[] = [];
this._beginToken(interpolationTokenType, interpolationStart);
parts.push(this._interpolationConfig.start);
@@ -783,7 +754,8 @@ class _Tokenizer {
// (This is actually wrong but here for backward compatibility).
this._cursor = current;
parts.push(this._getProcessedChars(expressionStart, current));
return this._endToken(parts);
this._endToken(parts);
return;
}
if (inQuote === null) {
@@ -791,7 +763,8 @@ class _Tokenizer {
// We are not in a string, and we hit the end interpolation marker
parts.push(this._getProcessedChars(expressionStart, current));
parts.push(this._interpolationConfig.end);
return this._endToken(parts);
this._endToken(parts);
return;
} else if (this._attemptStr('//')) {
// Once we are in a comment we ignore any quotes
inComment = true;
@@ -814,7 +787,7 @@ class _Tokenizer {
// We hit EOF without finding a closing interpolation marker
parts.push(this._getProcessedChars(expressionStart, this._cursor));
return this._endToken(parts);
this._endToken(parts);
}
private _getProcessedChars(start: CharacterCursor, end: CharacterCursor): string {
packages/compiler/src/ml_parser/parser.ts
+82 -77
View File
File diff suppressed because it is too large Load Diff
packages/compiler/src/ml_parser/tokens.ts
+172
View File
@@ -0,0 +1,172 @@
/**
* @license
* Copyright Google LLC All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {ParseSourceSpan} from '../parse_util';
export const enum TokenType {
TAG_OPEN_START,
TAG_OPEN_END,
TAG_OPEN_END_VOID,
TAG_CLOSE,
INCOMPLETE_TAG_OPEN,
TEXT,
ESCAPABLE_RAW_TEXT,
RAW_TEXT,
INTERPOLATION,
ENCODED_ENTITY,
COMMENT_START,
COMMENT_END,
CDATA_START,
CDATA_END,
ATTR_NAME,
ATTR_QUOTE,
ATTR_VALUE_TEXT,
ATTR_VALUE_INTERPOLATION,
DOC_TYPE,
EXPANSION_FORM_START,
EXPANSION_CASE_VALUE,
EXPANSION_CASE_EXP_START,
EXPANSION_CASE_EXP_END,
EXPANSION_FORM_END,
EOF
}
export type Token = TagOpenStartToken|TagOpenEndToken|TagOpenEndVoidToken|TagCloseToken|
IncompleteTagOpenToken|TextToken|InterpolationToken|EncodedEntityToken|CommentStartToken|
CommentEndToken|CdataStartToken|CdataEndToken|AttributeNameToken|AttributeQuoteToken|
AttributeValueTextToken|AttributeValueInterpolationToken|DocTypeToken|ExpansionFormStartToken|
ExpansionCaseValueToken|ExpansionCaseExpressionStartToken|ExpansionCaseExpressionEndToken|
ExpansionFormEndToken|EndOfFileToken;
export type InterpolatedTextToken = TextToken|InterpolationToken|EncodedEntityToken;
export type InterpolatedAttributeToken =
AttributeValueTextToken|AttributeValueInterpolationToken|EncodedEntityToken;
export interface TokenBase {
type: TokenType;
parts: string[];
sourceSpan: ParseSourceSpan;
}
export interface TagOpenStartToken extends TokenBase {
type: TokenType.TAG_OPEN_START;
parts: [prefix: string, name: string];
}
export interface TagOpenEndToken extends TokenBase {
type: TokenType.TAG_OPEN_END;
parts: [];
}
export interface TagOpenEndVoidToken extends TokenBase {
type: TokenType.TAG_OPEN_END_VOID;
parts: [];
}
export interface TagCloseToken extends TokenBase {
type: TokenType.TAG_CLOSE;
parts: [prefix: string, name: string];
}
export interface IncompleteTagOpenToken extends TokenBase {
type: TokenType.INCOMPLETE_TAG_OPEN;
parts: [prefix: string, name: string];
}
export interface TextToken extends TokenBase {
type: TokenType.TEXT|TokenType.ESCAPABLE_RAW_TEXT|TokenType.RAW_TEXT;
parts: [text: string];
}
export interface InterpolationToken extends TokenBase {
type: TokenType.INTERPOLATION;
parts: [startMarker: string, expression: string, endMarker: string]|
[startMarker: string, expression: string];
}
export interface EncodedEntityToken extends TokenBase {
type: TokenType.ENCODED_ENTITY;
parts: [decoded: string, encoded: string];
}
export interface CommentStartToken extends TokenBase {
type: TokenType.COMMENT_START;
parts: [];
}
export interface CommentEndToken extends TokenBase {
type: TokenType.COMMENT_END;
parts: [];
}
export interface CdataStartToken extends TokenBase {
type: TokenType.CDATA_START;
parts: [];
}
export interface CdataEndToken extends TokenBase {
type: TokenType.CDATA_END;
parts: [];
}
export interface AttributeNameToken extends TokenBase {
type: TokenType.ATTR_NAME;
parts: [prefix: string, name: string];
}
export interface AttributeQuoteToken extends TokenBase {
type: TokenType.ATTR_QUOTE;
parts: [quote: '\''|'"'];
}
export interface AttributeValueTextToken extends TokenBase {
type: TokenType.ATTR_VALUE_TEXT;
parts: [value: string];
}
export interface AttributeValueInterpolationToken extends TokenBase {
type: TokenType.ATTR_VALUE_INTERPOLATION;
parts: [startMarker: string, expression: string, endMarker: string]|
[startMarker: string, expression: string];
}
export interface DocTypeToken extends TokenBase {
type: TokenType.DOC_TYPE;
parts: [content: string];
}
export interface ExpansionFormStartToken extends TokenBase {
type: TokenType.EXPANSION_FORM_START;
parts: [];
}
export interface ExpansionCaseValueToken extends TokenBase {
type: TokenType.EXPANSION_CASE_VALUE;
parts: [value: string];
}
export interface ExpansionCaseExpressionStartToken extends TokenBase {
type: TokenType.EXPANSION_CASE_EXP_START;
parts: [];
}
export interface ExpansionCaseExpressionEndToken extends TokenBase {
type: TokenType.EXPANSION_CASE_EXP_END;
parts: [];
}
export interface ExpansionFormEndToken extends TokenBase {
type: TokenType.EXPANSION_FORM_END;
parts: [];
}
export interface EndOfFileToken extends TokenBase {
type: TokenType.EOF;
parts: [];
}
packages/compiler/test/ml_parser/html_parser_spec.ts
+1 -1
View File
@@ -8,7 +8,7 @@
import * as html from '../../src/ml_parser/ast';
import {HtmlParser, ParseTreeResult, TreeError} from '../../src/ml_parser/html_parser';
import {TokenType} from '../../src/ml_parser/lexer';
import {TokenType} from '../../src/ml_parser/tokens';
import {ParseError} from '../../src/parse_util';
import {humanizeDom, humanizeDomSourceSpans, humanizeLineColumn, humanizeNodes} from './ast_spec_utils';
packages/compiler/test/ml_parser/lexer_spec.ts
+814 -817
View File
File diff suppressed because it is too large Load Diff