From e8bbf86e66bbea9641ec9b9ecb846854e3ab16c7 Mon Sep 17 00:00:00 2001 From: Peter Bacon Darwin Date: Tue, 4 Jul 2017 17:59:08 +0100 Subject: [PATCH] build(aio): add terms from heading to the search index --- aio/src/app/search/search-worker.js | 3 +- .../processors/generateKeywords.js | 53 ++++++++----- .../processors/generateKeywords.spec.js | 77 +++++++++++++++++-- 3 files changed, 106 insertions(+), 27 deletions(-) diff --git a/aio/src/app/search/search-worker.js b/aio/src/app/search/search-worker.js index ebda62f471..85d112bbca 100644 --- a/aio/src/app/search/search-worker.js +++ b/aio/src/app/search/search-worker.js @@ -18,7 +18,8 @@ self.onmessage = handleMessage; function createIndex(addFn) { return lunr(/** @this */function() { this.ref('path'); - this.field('titleWords', {boost: 50}); + this.field('titleWords', {boost: 100}); + this.field('headingWords', {boost: 50}); this.field('members', {boost: 40}); this.field('keywords', {boost: 20}); addFn(this); diff --git a/aio/tools/transforms/angular-base-package/processors/generateKeywords.js b/aio/tools/transforms/angular-base-package/processors/generateKeywords.js index 99ee65d8d4..be8e60b952 100644 --- a/aio/tools/transforms/angular-base-package/processors/generateKeywords.js +++ b/aio/tools/transforms/angular-base-package/processors/generateKeywords.js @@ -50,19 +50,14 @@ module.exports = function generateKeywordsProcessor(log, readFilesProcessor) { var ignoreWordsMap = convertToMap(wordsToIgnore); - // If the title contains a name starting with ng, e.g. "ngController", then add the module - // name - // without the ng to the title text, e.g. "controller". - function extractTitleWords(title) { - var match = /ng([A-Z]\w*)/.exec(title); - if (match) { - title = title + ' ' + match[1].toLowerCase(); - } - return title; + // If the heading contains a name starting with ng, e.g. "ngController", then add the + // name without the ng to the text, e.g. "controller". + function preprocessText(text) { + return text.replace(/(^|\s)([nN]g([A-Z]\w*))/g, '$1$2 $3'); } function extractWords(text, words, keywordMap) { - var tokens = text.toLowerCase().split(/[.\s,`'"#]+/mg); + var tokens = preprocessText(text).toLowerCase().split(/[.\s,`'"#]+/mg); tokens.forEach(function(token) { var match = token.match(KEYWORD_REGEX); if (match) { @@ -82,13 +77,15 @@ module.exports = function generateKeywordsProcessor(log, readFilesProcessor) { // Ignore internals and private exports (indicated by the ɵ prefix) .filter(function(doc) { return !doc.internal && !doc.privateExport; }); - filteredDocs.forEach(function(doc) { + filteredDocs.forEach(function(doc) { var words = []; var keywordMap = Object.assign({}, ignoreWordsMap); var members = []; - var membersMap = {}; + var membersMap = Object.assign({}, ignoreWordsMap); + const headingWords = []; + const headingWordMap = Object.assign({}, ignoreWordsMap); // Search each top level property of the document for search terms Object.keys(doc).forEach(function(key) { @@ -98,26 +95,44 @@ module.exports = function generateKeywordsProcessor(log, readFilesProcessor) { extractWords(value, words, keywordMap); } + // Special case properties that contain content relating to "members" + // of a doc that represents, say, a class or interface if (key === 'methods' || key === 'properties' || key === 'events') { value.forEach(function(member) { extractWords(member.name, members, membersMap); }); } }); - doc.searchTitle = doc.searchTitle || doc.title || doc.vFile && doc.vFile.title || doc.name; + // Extract all the keywords from the headings + if (doc.vFile && doc.vFile.headings) { + Object.keys(doc.vFile.headings).forEach(function(headingTag) { + doc.vFile.headings[headingTag].forEach(function(headingText) { + extractWords(headingText, headingWords, headingWordMap); + }); + }); + } + // Extract the title to use in searches + doc.searchTitle = doc.searchTitle || doc.title || doc.vFile && doc.vFile.title || doc.name || ''; + + // Attach all this search data to the document doc.searchTerms = { - titleWords: extractTitleWords(doc.searchTitle), + titleWords: preprocessText(doc.searchTitle), + headingWords: headingWords.sort().join(' '), keywords: words.sort().join(' '), members: members.sort().join(' ') }; }); - var searchData = - filteredDocs.filter(function(page) { return page.searchTerms; }).map(function(page) { - return Object.assign( - {path: page.path, title: page.searchTitle, type: page.docType}, page.searchTerms); - }); + // Now process all the search data and collect it up to be used in creating a new document + var searchData = filteredDocs.map(function(page) { + // Copy the properties from the searchTerms object onto the search data object + return Object.assign({ + path: page.path, + title: page.searchTitle, + type: page.docType + }, page.searchTerms); + }); docs.push({ docType: 'json-doc', diff --git a/aio/tools/transforms/angular-base-package/processors/generateKeywords.spec.js b/aio/tools/transforms/angular-base-package/processors/generateKeywords.spec.js index 318b5b9dcd..999fb9fde2 100644 --- a/aio/tools/transforms/angular-base-package/processors/generateKeywords.spec.js +++ b/aio/tools/transforms/angular-base-package/processors/generateKeywords.spec.js @@ -42,9 +42,9 @@ describe('generateKeywords processor', () => { it('should compute `doc.searchTitle` from the doc properties if not already provided', () => { const processor = processorFactory(mockLogger, mockReadFilesProcessor); const docs = [ - { docType: 'class', name: 'A', searchTitle: 'searchTitle A', title: 'title A', vFile: { title: 'vFile A'} }, - { docType: 'class', name: 'B', title: 'title B', vFile: { title: 'vFile B'} }, - { docType: 'class', name: 'C', vFile: { title: 'vFile C'} }, + { docType: 'class', name: 'A', searchTitle: 'searchTitle A', title: 'title A', vFile: { headings: { h1: ['vFile A'] } } }, + { docType: 'class', name: 'B', title: 'title B', vFile: { headings: { h1: ['vFile B'] } } }, + { docType: 'class', name: 'C', vFile: { title: 'vFile C', headings: { h1: ['vFile C'] } } }, { docType: 'class', name: 'D' }, ]; processor.$process(docs); @@ -62,19 +62,82 @@ describe('generateKeywords processor', () => { { docType: 'class', name: 'PublicExport', searchTitle: 'class PublicExport' }, ]; processor.$process(docs); - expect(docs[docs.length - 1].data).toEqual([ + const keywordsDoc = docs[docs.length - 1]; + expect(keywordsDoc.data).toEqual([ jasmine.objectContaining({ title: 'class PublicExport', type: 'class'}) ]); }); + it('should add title words to the search terms', () => { + const processor = processorFactory(mockLogger, mockReadFilesProcessor); + const docs = [ + { + docType: 'class', + name: 'PublicExport', + searchTitle: 'class PublicExport', + vFile: { headings: { h2: ['heading A', 'heading B'] } } + }, + ]; + processor.$process(docs); + const keywordsDoc = docs[docs.length - 1]; + expect(keywordsDoc.data[0].titleWords).toEqual('class PublicExport'); + }); + + it('should add heading words to the search terms', () => { + const processor = processorFactory(mockLogger, mockReadFilesProcessor); + const docs = [ + { + docType: 'class', + name: 'PublicExport', + searchTitle: 'class PublicExport', + vFile: { headings: { h2: ['Important heading', 'Secondary heading'] } } + }, + ]; + processor.$process(docs); + const keywordsDoc = docs[docs.length - 1]; + expect(keywordsDoc.data[0].headingWords).toEqual('heading important secondary'); + }); + + it('should process terms prefixed with "ng" to include the term stripped of "ng"', () => { + const processor = processorFactory(mockLogger, mockReadFilesProcessor); + const docs = [ + { + docType: 'class', + name: 'PublicExport', + searchTitle: 'ngController', + vFile: { headings: { h2: ['ngModel'] } }, + content: 'Some content with ngClass in it.' + }, + ]; + processor.$process(docs); + const keywordsDoc = docs[docs.length - 1]; + expect(keywordsDoc.data[0].titleWords).toEqual('ngController Controller'); + expect(keywordsDoc.data[0].headingWords).toEqual('model ngmodel'); + expect(keywordsDoc.data[0].keywords).toContain('class'); + expect(keywordsDoc.data[0].keywords).toContain('ngclass'); + }); + it('should generate renderedContent property', () => { const processor = processorFactory(mockLogger, mockReadFilesProcessor); const docs = [ - { docType: 'class', name: 'SomeClass', description: 'The is the documentation for the SomeClass API.' }, + { + docType: 'class', + name: 'SomeClass', + description: 'The is the documentation for the SomeClass API.', + vFile: { headings: { h1: ['SomeClass'], h2: ['Some heading'] } } + }, ]; processor.$process(docs); - expect(docs[docs.length - 1].renderedContent).toEqual( - '[{"title":"SomeClass","type":"class","titleWords":"SomeClass","keywords":"api class documentation for is someclass the","members":""}]' + const keywordsDoc = docs[docs.length - 1]; + expect(JSON.parse(keywordsDoc.renderedContent)).toEqual( + [{ + 'title':'SomeClass', + 'type':'class', + 'titleWords':'SomeClass', + 'headingWords':'heading some someclass', + 'keywords':'api class documentation for is someclass the', + 'members':'' + }] ); }); });