iSharkFly-Docs/angular-docs-cn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "fix(compiler): include leading whitespace in source-spans of i18n messages (#42062)" (#43033)

Browse Source 
This reverts commit f08516db09.

PR Close #43033
This commit is contained in:
atscott
2021-08-03 14:48:52 -07:00
parent 8b6f7ac36b
commit dd82bbfa27
15 changed files with 98 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/compiler/src/i18n/i18n_parser.ts
+63 -31
View File
@@ -7,11 +7,10 @@
*/
import {Lexer as ExpressionLexer} from '../expression_parser/lexer';
import {Parser as ExpressionParser} from '../expression_parser/parser';
import {InterpolationPiece, Parser as ExpressionParser} from '../expression_parser/parser';
import * as html from '../ml_parser/ast';
import {getHtmlTagDefinition} from '../ml_parser/html_tags';
import {InterpolationConfig} from '../ml_parser/interpolation_config';
import {Token, TokenType} from '../ml_parser/lexer';
import {ParseSourceSpan} from '../parse_util';
import * as i18n from './i18n_ast';
@@ -106,18 +105,13 @@ class _I18nVisitor implements html.Visitor {
}
visitAttribute(attribute: html.Attribute, context: I18nMessageVisitorContext): i18n.Node {
const node = attribute.valueTokens === undefined || attribute.valueTokens.length === 1 ?
new i18n.Text(attribute.value, attribute.valueSpan || attribute.sourceSpan) :
this._visitTextWithInterpolation(
attribute.valueTokens, attribute.valueSpan || attribute.sourceSpan, context,
attribute.i18n);
const node = this._visitTextWithInterpolation(
attribute.value, attribute.valueSpan || attribute.sourceSpan, context, attribute.i18n);
return context.visitNodeFn(attribute, node);
}
visitText(text: html.Text, context: I18nMessageVisitorContext): i18n.Node {
const node = text.tokens.length === 1 ?
new i18n.Text(text.value, text.sourceSpan) :
this._visitTextWithInterpolation(text.tokens, text.sourceSpan, context, text.i18n);
const node = this._visitTextWithInterpolation(text.value, text.sourceSpan, context, text.i18n);
return context.visitNodeFn(text, node);
}
@@ -171,36 +165,66 @@ class _I18nVisitor implements html.Visitor {
* @param previousI18n Any i18n metadata associated with this `text` from a previous pass.
*/
private _visitTextWithInterpolation(
tokens: Token[], sourceSpan: ParseSourceSpan, context: I18nMessageVisitorContext,
text: string, sourceSpan: ParseSourceSpan, context: I18nMessageVisitorContext,
previousI18n: i18n.I18nMeta|undefined): i18n.Node {
const {strings, expressions} = this._expressionParser.splitInterpolation(
text, sourceSpan.start.toString(), this._interpolationConfig);
// No expressions, return a single text.
if (expressions.length === 0) {
return new i18n.Text(text, sourceSpan);
}
// Return a sequence of `Text` and `Placeholder` nodes grouped in a `Container`.
const nodes: i18n.Node[] = [];
for (const token of tokens) {
switch (token.type) {
case TokenType.INTERPOLATION:
case TokenType.ATTR_VALUE_INTERPOLATION:
const expression = token.parts[1];
const baseName = extractPlaceholderName(expression) || 'INTERPOLATION';
const phName = context.placeholderRegistry.getPlaceholderName(baseName, expression);
context.placeholderToContent[phName] = {
text: token.parts.join(''),
sourceSpan: token.sourceSpan
};
nodes.push(new i18n.Placeholder(expression, phName, token.sourceSpan));
break;
default:
if (token.parts[0].length > 0) {
nodes.push(new i18n.Text(token.parts[0], token.sourceSpan));
}
break;
}
for (let i = 0; i < strings.length - 1; i++) {
this._addText(nodes, strings[i], sourceSpan);
this._addPlaceholder(nodes, context, expressions[i], sourceSpan);
}
// The last index contains no expression
this._addText(nodes, strings[strings.length - 1], sourceSpan);
// Whitespace removal may have invalidated the interpolation source-spans.
reusePreviousSourceSpans(nodes, previousI18n);
return new i18n.Container(nodes, sourceSpan);
}
/**
* Create a new `Text` node from the `textPiece` and add it to the `nodes` collection.
*
* @param nodes The nodes to which the created `Text` node should be added.
* @param textPiece The text and relative span information for this `Text` node.
* @param interpolationSpan The span of the whole interpolated text.
*/
private _addText(
nodes: i18n.Node[], textPiece: InterpolationPiece, interpolationSpan: ParseSourceSpan): void {
if (textPiece.text.length > 0) {
// No need to add empty strings
const stringSpan = getOffsetSourceSpan(interpolationSpan, textPiece);
nodes.push(new i18n.Text(textPiece.text, stringSpan));
}
}
/**
* Create a new `Placeholder` node from the `expression` and add it to the `nodes` collection.
*
* @param nodes The nodes to which the created `Text` node should be added.
* @param context The current context of the visitor, used to compute and store placeholders.
* @param expression The expression text and relative span information for this `Placeholder`
* node.
* @param interpolationSpan The span of the whole interpolated text.
*/
private _addPlaceholder(
nodes: i18n.Node[], context: I18nMessageVisitorContext, expression: InterpolationPiece,
interpolationSpan: ParseSourceSpan): void {
const sourceSpan = getOffsetSourceSpan(interpolationSpan, expression);
const baseName = extractPlaceholderName(expression.text) || 'INTERPOLATION';
const phName = context.placeholderRegistry.getPlaceholderName(baseName, expression.text);
const text = this._interpolationConfig.start + expression.text + this._interpolationConfig.end;
context.placeholderToContent[phName] = {text, sourceSpan};
nodes.push(new i18n.Placeholder(expression.text, phName, sourceSpan));
}
}
/**
@@ -223,7 +247,7 @@ function reusePreviousSourceSpans(nodes: i18n.Node[], previousI18n: i18n.I18nMet
if (previousI18n instanceof i18n.Container) {
// The `previousI18n` is a `Container`, which means that this is a second i18n extraction pass
// after whitespace has been removed from the AST nodes.
// after whitespace has been removed from the AST ndoes.
assertEquivalentNodes(previousI18n.children, nodes);
// Reuse the source-spans from the first pass.
@@ -258,6 +282,14 @@ function assertEquivalentNodes(previousNodes: i18n.Node[], nodes: i18n.Node[]):
}
}
/**
* Create a new `ParseSourceSpan` from the `sourceSpan`, offset by the `start` and `end` values.
*/
function getOffsetSourceSpan(
sourceSpan: ParseSourceSpan, {start, end}: InterpolationPiece): ParseSourceSpan {
return new ParseSourceSpan(sourceSpan.fullStart.moveBy(start), sourceSpan.fullStart.moveBy(end));
}
const _CUSTOM_PH_EXP =
/\/\/[\s\S]*i18n[\s\S]*\([\s\S]*ph[\s\S]*=[\s\S]*("|')([\s\S]*?)\1[\s\S]*\)/g;
packages/compiler/src/ml_parser/lexer.ts
+10 -33
View File
@@ -230,11 +230,8 @@ class _Tokenizer {
this._consumeTagOpen(start);
}
} else if (!(this._tokenizeIcu && this._tokenizeExpansionForm())) {
// In (possibly interpolated) text the end of the text is given by `isTextEnd()`, while
// the premature end of an interpolation is given by the start of a new HTML element.
this._consumeWithInterpolation(
TokenType.TEXT, TokenType.INTERPOLATION, () => this._isTextEnd(),
() => this._isTagStart());
TokenType.TEXT, TokenType.INTERPOLATION, () => this._isTextEnd());
}
} catch (e) {
this.handleError(e);
@@ -611,18 +608,14 @@ class _Tokenizer {
if (this._cursor.peek() === chars.$SQ || this._cursor.peek() === chars.$DQ) {
const quoteChar = this._cursor.peek();
this._consumeQuote(quoteChar);
// In an attribute then end of the attribute value and the premature end to an interpolation
// are both triggered by the `quoteChar`.
const endPredicate = () => this._cursor.peek() === quoteChar;
this._consumeWithInterpolation(
TokenType.ATTR_VALUE_TEXT, TokenType.ATTR_VALUE_INTERPOLATION, endPredicate,
endPredicate);
TokenType.ATTR_VALUE_TEXT, TokenType.ATTR_VALUE_INTERPOLATION,
() => this._cursor.peek() === quoteChar);
this._consumeQuote(quoteChar);
} else {
const endPredicate = () => isNameEnd(this._cursor.peek());
this._consumeWithInterpolation(
TokenType.ATTR_VALUE_TEXT, TokenType.ATTR_VALUE_INTERPOLATION, endPredicate,
endPredicate);
TokenType.ATTR_VALUE_TEXT, TokenType.ATTR_VALUE_INTERPOLATION, endPredicate);
}
}
@@ -712,21 +705,15 @@ class _Tokenizer {
/**
* Consume a string that may contain interpolation expressions.
*
* The first token consumed will be of `tokenType` and then there will be alternating
* `interpolationTokenType` and `tokenType` tokens until the `endPredicate()` returns true.
*
* If an interpolation token ends prematurely it will have no end marker in its `parts` array.
*
* @param textTokenType the kind of tokens to interleave around interpolation tokens.
* @param interpolationTokenType the kind of tokens that contain interpolation.
* @param endPredicate a function that should return true when we should stop consuming.
* @param endInterpolation a function that should return true if there is a premature end to an
* interpolation expression - i.e. before we get to the normal interpolation closing marker.
*/
private _consumeWithInterpolation(
textTokenType: TokenType, interpolationTokenType: TokenType, endPredicate: () => boolean,
endInterpolation: () => boolean) {
textTokenType: TokenType, interpolationTokenType: TokenType, endPredicate: () => boolean) {
this._beginToken(textTokenType);
const parts: string[] = [];
@@ -735,7 +722,7 @@ class _Tokenizer {
if (this._interpolationConfig && this._attemptStr(this._interpolationConfig.start)) {
this._endToken([this._processCarriageReturns(parts.join(''))], current);
parts.length = 0;
this._consumeInterpolation(interpolationTokenType, current, endInterpolation);
this._consumeInterpolation(interpolationTokenType, current);
this._beginToken(textTokenType);
} else if (this._cursor.peek() === chars.$AMPERSAND) {
this._endToken([this._processCarriageReturns(parts.join(''))]);
@@ -754,17 +741,8 @@ class _Tokenizer {
this._endToken([this._processCarriageReturns(parts.join(''))]);
}
/**
* Consume a block of text that has been interpreted as an Angular interpolation.
*
* @param interpolationTokenType the type of the interpolation token to generate.
* @param interpolationStart a cursor that points to the start of this interpolation.
* @param prematureEndPredicate a function that should return true if the next characters indicate
* an end to the interpolation before its normal closing marker.
*/
private _consumeInterpolation(
interpolationTokenType: TokenType, interpolationStart: CharacterCursor,
prematureEndPredicate: (() => boolean)|null) {
interpolationTokenType: TokenType, interpolationStart: CharacterCursor) {
const parts: string[] = [];
this._beginToken(interpolationTokenType, interpolationStart);
parts.push(this._interpolationConfig.start);
@@ -773,8 +751,7 @@ class _Tokenizer {
const expressionStart = this._cursor.clone();
let inQuote: number|null = null;
let inComment = false;
while (this._cursor.peek() !== chars.$EOF &&
(prematureEndPredicate === null || !prematureEndPredicate())) {
while (this._cursor.peek() !== chars.$EOF) {
const current = this._cursor.clone();
if (this._isTagStart()) {
@@ -806,7 +783,7 @@ class _Tokenizer {
} else if (char === inQuote) {
// Exiting the current quoted string
inQuote = null;
} else if (!inComment && inQuote === null && chars.isQuote(char)) {
} else if (!inComment && chars.isQuote(char)) {
// Entering a new quoted string
inQuote = char;
}
@@ -818,7 +795,7 @@ class _Tokenizer {
}
private _getProcessedChars(start: CharacterCursor, end: CharacterCursor): string {
return this._processCarriageReturns(end.getChars(start));
return this._processCarriageReturns(end.getChars(start))
}
private _isTextEnd(): boolean {
packages/compiler/src/render3/view/i18n/localize_utils.ts
+2 -5
View File
@@ -35,10 +35,7 @@ class LocalizeSerializerVisitor implements i18n.Visitor {
// Two literal pieces in a row means that there was some comment node in-between.
context[context.length - 1].text += text.value;
} else {
const sourceSpan = new ParseSourceSpan(
text.sourceSpan.fullStart, text.sourceSpan.end, text.sourceSpan.fullStart,
text.sourceSpan.details);
context.push(new o.LiteralPiece(text.value, sourceSpan));
context.push(new o.LiteralPiece(text.value, text.sourceSpan));
}
}
@@ -93,7 +90,7 @@ function getSourceSpan(message: i18n.Message): ParseSourceSpan {
const startNode = message.nodes[0];
const endNode = message.nodes[message.nodes.length - 1];
return new ParseSourceSpan(
startNode.sourceSpan.fullStart, endNode.sourceSpan.end, startNode.sourceSpan.fullStart,
startNode.sourceSpan.start, endNode.sourceSpan.end, startNode.sourceSpan.fullStart,
startNode.sourceSpan.details);
}
packages/compiler/test/ml_parser/lexer_spec.ts
+2 -41
View File
@@ -316,32 +316,6 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u
]);
});
it('should end interpolation on an unescaped matching quote', () => {
expect(tokenizeAndHumanizeParts('<t a="{{ a \\" \' b ">')).toEqual([
[lex.TokenType.TAG_OPEN_START, '', 't'],
[lex.TokenType.ATTR_NAME, '', 'a'],
[lex.TokenType.ATTR_QUOTE, '"'],
[lex.TokenType.ATTR_VALUE_TEXT, ''],
[lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a \\" \' b '],
[lex.TokenType.ATTR_VALUE_TEXT, ''],
[lex.TokenType.ATTR_QUOTE, '"'],
[lex.TokenType.TAG_OPEN_END],
[lex.TokenType.EOF],
]);
expect(tokenizeAndHumanizeParts('<t a=\'{{ a " \\\' b \'>')).toEqual([
[lex.TokenType.TAG_OPEN_START, '', 't'],
[lex.TokenType.ATTR_NAME, '', 'a'],
[lex.TokenType.ATTR_QUOTE, '\''],
[lex.TokenType.ATTR_VALUE_TEXT, ''],
[lex.TokenType.ATTR_VALUE_INTERPOLATION, '{{', ' a " \\\' b '],
[lex.TokenType.ATTR_VALUE_TEXT, ''],
[lex.TokenType.ATTR_QUOTE, '\''],
[lex.TokenType.TAG_OPEN_END],
[lex.TokenType.EOF],
]);
});
it('should parse attributes with prefix', () => {
expect(tokenizeAndHumanizeParts('<t ns1:a>')).toEqual([
[lex.TokenType.TAG_OPEN_START, '', 't'],
@@ -619,15 +593,14 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u
});
it('should parse interpolation', () => {
expect(tokenizeAndHumanizeParts(
'{{ a }}b{{ c // comment }}d{{ e "}} \' " f }}g{{ h // " i }}'))
expect(tokenizeAndHumanizeParts('{{ a }}b{{ c // comment }}d{{ e "}}" f }}g{{ h // " i }}'))
.toEqual([
[lex.TokenType.TEXT, ''],
[lex.TokenType.INTERPOLATION, '{{', ' a ', '}}'],
[lex.TokenType.TEXT, 'b'],
[lex.TokenType.INTERPOLATION, '{{', ' c // comment ', '}}'],
[lex.TokenType.TEXT, 'd'],
[lex.TokenType.INTERPOLATION, '{{', ' e "}} \' " f ', '}}'],
[lex.TokenType.INTERPOLATION, '{{', ' e "}}" f ', '}}'],
[lex.TokenType.TEXT, 'g'],
[lex.TokenType.INTERPOLATION, '{{', ' h // " i ', '}}'],
[lex.TokenType.TEXT, ''],
@@ -762,18 +735,6 @@ import {ParseLocation, ParseSourceFile, ParseSourceSpan} from '../../src/parse_u
]);
});
it('should end interpolation on a valid closing tag', () => {
expect(tokenizeAndHumanizeParts('<p>{{ a </p>')).toEqual([
[lex.TokenType.TAG_OPEN_START, '', 'p'],
[lex.TokenType.TAG_OPEN_END],
[lex.TokenType.TEXT, ''],
[lex.TokenType.INTERPOLATION, '{{', ' a '],
[lex.TokenType.TEXT, ''],
[lex.TokenType.TAG_CLOSE, '', 'p'],
[lex.TokenType.EOF],
]);
});
it('should break out of interpolation in text token on valid CDATA', () => {
expect(tokenizeAndHumanizeParts('{{ a }<![CDATA[]]>}')).toEqual([
[lex.TokenType.TEXT, ''],
packages/compiler/test/render3/view/i18n_spec.ts
+1 -1
View File
@@ -478,7 +478,7 @@ describe('serializeI18nMessageForLocalize', () => {
expect(messageParts[3].text).toEqual('');
expect(messageParts[3].sourceSpan.toString()).toEqual('');
expect(messageParts[4].text).toEqual(' D');
expect(messageParts[4].sourceSpan.toString()).toEqual(' D');
expect(messageParts[4].sourceSpan.toString()).toEqual('D');
expect(placeHolders[0].text).toEqual('START_TAG_SPAN');
expect(placeHolders[0].sourceSpan.toString()).toEqual('<span>');