Skip to content

Commit dc536d1

Browse files
vkarpov15vkarpov15
authored
Merge pull request #15379 from hasezoey/docsReduceLength
Reduce print length in `website.js` and dont exit early in `generateSearch`
2 parents b1ac0f5 + 6d27965 commit dc536d1

File tree

2 files changed

+135
-87
lines changed

2 files changed

+135
-87
lines changed

‎scripts/generateSearch.js

Lines changed: 122 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,7 @@
11
'use strict';
22

3-
let config;
4-
try {
5-
config = require('../.config.js');
6-
} finally {
7-
if (!config || !config.uri) {
8-
console.error('No Config or config.URI given, please create a .config.js file with those values in the root of the repository');
9-
process.exit(-1);
10-
}
11-
}
3+
const isMain = require.main === module;
4+
125
const cheerio = require('cheerio');
136
const docsFilemap = require('../docs/source');
147
const fs = require('fs');
@@ -37,102 +30,106 @@ const contentSchema = new mongoose.Schema({
3730
contentSchema.index({ title: 'text', body: 'text' });
3831
const Content = mongoose.model('Content', contentSchema, 'Content');
3932

40-
const contents = [];
41-
42-
for (const [filename, file] of Object.entries(docsFilemap.fileMap)) {
43-
if (file.api) {
44-
for (const prop of file.props) {
45-
const content = new Content({
46-
title: `API: ${prop.name}`,
47-
body: prop.description,
48-
url: `${filename}#${prop.anchorId}`
49-
});
50-
const err = content.validateSync();
51-
if (err != null) {
52-
console.error(content);
53-
throw err;
33+
function generateContents() {
34+
const contents = [];
35+
36+
for (const [filename, file] of Object.entries(docsFilemap.fileMap)) {
37+
if (file.api) {
38+
for (const prop of file.props) {
39+
const content = new Content({
40+
title: `API: ${prop.name}`,
41+
body: prop.description,
42+
url: `${filename}#${prop.anchorId}`
43+
});
44+
const err = content.validateSync();
45+
if (err != null) {
46+
console.error(content);
47+
throw err;
48+
}
49+
contents.push(content);
5450
}
55-
contents.push(content);
56-
}
57-
} else if (file.markdown) {
58-
let text = fs.readFileSync(filename, 'utf8');
59-
text = markdown.parse(text);
60-
61-
const content = new Content({
62-
title: file.title,
63-
body: text,
64-
url: filename.replace('.md', '.html').replace(/^docs/, '')
65-
});
66-
67-
content.validateSync();
68-
69-
const $ = cheerio.load(text);
51+
} else if (file.markdown) {
52+
let text = fs.readFileSync(filename, 'utf8');
53+
text = markdown.parse(text);
7054

71-
contents.push(content);
72-
73-
// Break up individual h3's into separate content for more fine grained search
74-
$('h3').each((index, el) => {
75-
el = $(el);
76-
const title = el.text();
77-
const html = el.nextUntil('h3').html();
7855
const content = new Content({
79-
title: `${file.title}: ${title}`,
80-
body: html,
81-
url: `${filename.replace('.md', '.html').replace(/^docs/, '')}#${el.prop('id')}`
56+
title: file.title,
57+
body: text,
58+
url: filename.replace('.md', '.html').replace(/^docs/, '')
8259
});
8360

8461
content.validateSync();
85-
contents.push(content);
86-
});
87-
} else if (file.guide) {
88-
let text = fs.readFileSync(filename, 'utf8');
89-
text = text.substr(text.indexOf('block content') + 'block content\n'.length);
90-
text = pug.render(`div\n${text}`, { filters: { markdown }, filename });
91-
92-
const content = new Content({
93-
title: file.title,
94-
body: text,
95-
url: filename.replace('.pug', '.html').replace(/^docs/, '')
96-
});
9762

98-
content.validateSync();
63+
const $ = cheerio.load(text);
9964

100-
const $ = cheerio.load(text);
65+
contents.push(content);
10166

102-
contents.push(content);
67+
// Break up individual h3's into separate content for more fine grained search
68+
$('h3').each((index, el) => {
69+
el = $(el);
70+
const title = el.text();
71+
const html = el.nextUntil('h3').html();
72+
const content = new Content({
73+
title: `${file.title}: ${title}`,
74+
body: html,
75+
url: `${filename.replace('.md', '.html').replace(/^docs/, '')}#${el.prop('id')}`
76+
});
77+
78+
content.validateSync();
79+
contents.push(content);
80+
});
81+
} else if (file.guide) {
82+
let text = fs.readFileSync(filename, 'utf8');
83+
text = text.substring(text.indexOf('block content') + 'block content\n'.length);
84+
text = pug.render(`div\n${text}`, { filters: { markdown }, filename });
10385

104-
// Break up individual h3's into separate content for more fine grained search
105-
$('h3').each((index, el) => {
106-
el = $(el);
107-
const title = el.text();
108-
const html = el.nextUntil('h3').html();
10986
const content = new Content({
110-
title: `${file.title}: ${title}`,
111-
body: html,
112-
url: `${filename.replace('.pug', '.html').replace(/^docs/, '')}#${el.prop('id')}`
87+
title: file.title,
88+
body: text,
89+
url: filename.replace('.pug', '.html').replace(/^docs/, '')
11390
});
11491

11592
content.validateSync();
93+
94+
const $ = cheerio.load(text);
95+
11696
contents.push(content);
117-
});
118-
}
119-
}
12097

121-
run().catch(async error => {
122-
console.error(error.stack);
98+
// Break up individual h3's into separate content for more fine grained search
99+
$('h3').each((index, el) => {
100+
el = $(el);
101+
const title = el.text();
102+
const html = el.nextUntil('h3').html();
103+
const content = new Content({
104+
title: `${file.title}: ${title}`,
105+
body: html,
106+
url: `${filename.replace('.pug', '.html').replace(/^docs/, '')}#${el.prop('id')}`
107+
});
108+
109+
content.validateSync();
110+
contents.push(content);
111+
});
112+
}
113+
}
123114

124-
// ensure the script exists in case of error
125-
await mongoose.disconnect();
126-
});
115+
return contents;
116+
}
127117

128-
async function run() {
118+
async function generateSearch(config) {
129119
await mongoose.connect(config.uri, { dbName: 'mongoose' });
130120

131121
// wait for the index to be created
132122
await Content.init();
133123

134124
await Content.deleteMany({ version });
135-
let count = 0;
125+
126+
const contents = generateContents();
127+
128+
const promises = [];
129+
let lastPrint = 0;
130+
131+
let doneCount = 0;
132+
console.log('Search Content to save:', contents.length);
136133
for (const content of contents) {
137134
if (version === '8.x') {
138135
let url = content.url.startsWith('/') ? content.url : `/${content.url}`;
@@ -147,18 +144,58 @@ async function run() {
147144
}
148145
content.url = `/docs/${version}${url}`;
149146
}
150-
console.log(`${++count} / ${contents.length}`);
151-
await content.save();
147+
const promise = content.save().then(() => {
148+
doneCount += 1;
149+
const nowDate = Date.now();
150+
// only print every 2 seconds, or if it is the first or last element
151+
if (nowDate - lastPrint > 2000 || doneCount === contents.length || doneCount === 1) {
152+
lastPrint = nowDate;
153+
console.log(`${doneCount} / ${contents.length}`);
154+
}
155+
});
156+
promises.push(promise);
152157
}
153158

159+
await Promise.allSettled(promises);
160+
154161
const results = await Content.
155162
find({ $text: { $search: 'validate' }, version }, { score: { $meta: 'textScore' } }).
156163
sort({ score: { $meta: 'textScore' } }).
157164
limit(10);
158165

159166
console.log(results.map(res => res.url));
160167

161-
console.log(`Added ${contents.length} Content`);
168+
console.log(`Added ${contents.length} Search Content`);
169+
170+
// this likely should not be done as part of this script, but by the caller,
171+
// but this script is currently the only one that connects in the website generation.
172+
await mongoose.disconnect();
173+
}
174+
175+
function getConfig() {
176+
const config = require('../.config.js');
162177

163-
process.exit(0);
178+
if (!config || !config.uri) {
179+
throw new Error('No Config or config.URI given, please create a .config.js file with those values in the root of the repository');
180+
}
181+
182+
return config;
183+
}
184+
185+
module.exports.generateSearch = generateSearch;
186+
module.exports.getConfig = getConfig;
187+
188+
// only run the following code if this file is the main module / entry file
189+
if (isMain) {
190+
(async function main() {
191+
const config = getConfig();
192+
try {
193+
await generateSearch(config);
194+
} catch (error) {
195+
console.error(error);
196+
process.exit(-1);
197+
} finally {
198+
await mongoose.disconnect();
199+
}
200+
})();
164201
}

‎scripts/website.js

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -506,7 +506,7 @@ async function pugify(filename, options, isReload = false) {
506506
await fs.promises.writeFile(newfile, str).catch((err) => {
507507
console.error('could not write', err.stack);
508508
}).then(() => {
509-
console.log('%s : rendered ', new Date(), newfile);
509+
console.log('%s : rendered %s', (new Date()).toISOString(), newfile);
510510
});
511511
}
512512

@@ -606,14 +606,25 @@ if (isMain) {
606606
(async function main() {
607607
console.log(`Processing ~${files.length} files`);
608608

609-
require('./generateSearch');
609+
const generateSearch = require('./generateSearch');
610+
let generateSearchPromise;
611+
try {
612+
const config = generateSearch.getConfig();
613+
generateSearchPromise = generateSearch.generateSearch(config);
614+
} catch (err) {
615+
console.error("Generating Search failed:", err);
616+
}
610617
await deleteAllHtmlFiles();
611618
await pugifyAllFiles();
612619
await copyAllRequiredFiles();
613620
if (!!process.env.DOCS_DEPLOY && !!versionObj.versionedPath) {
614621
await moveDocsToTemp();
615622
}
616623

624+
if (!!generateSearchPromise) {
625+
await generateSearchPromise;
626+
}
627+
617628
console.log('Done Processing');
618629
})();
619630
}

0 commit comments

Comments
 (0)