[INTERNAL] lib/processors/jsdoc: remove duplicates after processingComplete

Processing of @Augments, @mixin and @borrows tags might produce new
duplicates which are not yet visible during the `parseComplete` event.

Moving the cleanup to `processingComplete` fixes this.

The merge of the additional data that the UI5 AST visitor collected,
is still done in `parseComplete` so that it is available when doclets
are cloned during the processing of @Augments, @mixin and @borrows tags.

Cherry-picked from UI5/openui5@96f6f095d.

Author: codeworrior

lib/processors/jsdoc/lib/ui5/plugin.js

@@ -2915,53 +2915,20 @@ exports.handlers = {
 		currentSource = e.source;
 	},
 
+	/**
+	 * Event `parseComplete` is fired by JSDoc after all files have been parsed,
+	 * but before inheritance, mixins or borrows are processed.
+	 *
+	 * We use this event to merge our additional data into the doclets collected by JSDoc.
+	 * The merge must happen before doclets are cloned during the prcessing of augments or borrows.
+	 */
 	parseComplete : function(e) {
-
 		const doclets = e.doclets;
-		const rAnonymous = /^<anonymous>(~|$)/;
-
-		// remove undocumented symbols, ignored symbols, anonymous functions and their members, scope members
-		let l = doclets.length, i, j;
-		for (i = 0, j = 0; i < l; i++) {
-
+		const l = doclets.length;
+		for (let i = 0; i < l; i++) {
 			const doclet = doclets[i];
-			if ( !doclet.undocumented &&
-				!doclet.ignore &&
-				!(doclet.memberof && rAnonymous.test(doclet.memberof)) &&
-				doclet.longname.indexOf("~") < 0 ) {
-				doclets[j++] = doclet;
-			}
-		}
-		if ( j < l ) {
-			doclets.splice(j, l - j);
-			info(`removed ${l - j} undocumented, ignored or anonymous symbols`);
-			l = j;
-		}
 
-		// sort doclets by name, synthetic, lineno, uid
-		// 'ignore' is a combination of criteria, see function above
-		debug("sorting doclets by name");
-		doclets.sort((a, b) => {
-			if ( a.longname === b.longname ) {
-				if ( a.synthetic === b.synthetic ) {
-					if ( a.meta && b.meta && a.meta.filename == b.meta.filename ) {
-						if ( a.meta.lineno !== b.meta.lineno ) {
-							return a.meta.lineno < b.meta.lineno ? -1 : 1;
-						}
-					}
-					return a.__ui5.id - b.__ui5.id;
-				}
-				return a.synthetic && !b.synthetic ? -1 : 1;
-			}
-			return a.longname < b.longname ? -1 : 1;
-		});
-		debug("sorting doclets by name done.");
-
-		for (i = 0, j = 0; i < l; i++) {
-
-			const doclet = doclets[i];
-
-			// add metadata to symbol
+			// add metadata to class symbols
 			if ( classInfos[doclet.longname] ) {
 				// debug("class data", doclet.longname, "'" + classInfos[doclet.longname].export + "'");
 				if ( doclet.__ui5.export === undefined ) {
@@ -3005,6 +2972,7 @@ exports.handlers = {
 				}
 			}
 
+			// add DataType info to typedef symbols
 			if ( typeInfos[doclet.longname] ) {
 				doclet.__ui5.stereotype = 'datatype';
 				doclet.__ui5.metadata = {
@@ -3014,6 +2982,7 @@ exports.handlers = {
 				};
 			}
 
+			// add enum values to enum keys (for enum symbols)
 			if ( (doclet.kind === 'member' || doclet.kind === 'constant') && doclet.isEnum && Array.isArray(doclet.properties) ) {
 				// determine unique enum identifier from key set
 				let enumID = doclet.properties.map(function(prop) {
@@ -3036,31 +3005,96 @@ exports.handlers = {
 					}
 				}
 			}
+		}
+	},
+
+	/**
+	 * Event `processingComplete` is fired by JSDoc after all files have been parsed,
+	 * and after inheritance, mixins and borrows have been processed.
+	 *
+	 * The `e.doclets` contains the symbols that will be given to templates for publishing.
+	 *
+	 * We use this event to remove symbols that are not of interest:
+	 * - undocumented   When JSDoc finds a class, function, object or member without a
+	 *                  JSDoc comment, it creates a doclet with a truthy `undocumented`
+	 *                  property
+	 * - ignore         A symbol that has been marked with `@ignore` in the source code
+	 * - anonymous      JSDoc could not infer a name for the symbol, neither from source
+	 *                  code nor from JSDoc comments
+	 * - local          Local entities (e.g. local vars) can't be addressed from the outside
+	 *                  and therefore are generally not considered as API in UI5
+	 * - duplicates     This plugin generates doclets for the accessor methods of
+	 *                  managed properties, aggregations, events, associations.
+	 *                  Developers might have created JSDoc comments for the same methods,
+	 *                  either because they have overridden them in code or because they
+	 *                  wanted to detail the method contract. If such duplicate doclets
+	 *                  are detected, the developer created doclets are preferred. If
+	 *                  multiple developer created doclets for the same entity exist in the
+	 *                  same file, the last one wins. If multiple doclets exists across
+	 *                  files, the one created last wins (but usually, this indicates a
+	 *                  copy & paste error)
+	 *
+	 * The cleanup is done in `processingComplete` as the processing of `@augments` or
+	 * `@borrows` tags might have created new non-interesting symbols.
+	 */
+	processingComplete(e) {
+		const doclets = e.doclets;
+
+		// sort doclets by name, synthetic, lineno, uid for easier detection of duplicates
+		debug("sorting doclets by name");
+		doclets.sort((a, b) => {
+			if ( a.longname === b.longname ) {
+				if ( a.synthetic === b.synthetic ) {
+					if ( a.meta && b.meta && a.meta.filename == b.meta.filename ) {
+						if ( a.meta.lineno !== b.meta.lineno ) {
+							return a.meta.lineno < b.meta.lineno ? -1 : 1;
+						}
+					}
+					return a.__ui5.id - b.__ui5.id;
+				}
+				return a.synthetic && !b.synthetic ? -1 : 1;
+			}
+			return a.longname < b.longname ? -1 : 1;
+		});
+		debug("sorting doclets by name done.");
+
+		// cleanup doclets
+		const rAnonymous = /^<anonymous>(~|$)/;
+		const l = doclets.length;
+		let j = 0;
+		for (let i = 0; i < l; i++) {
+			const doclet = doclets[i];
+
+			// skip undocumented, ignored, anonymous entities as well as local entities
+			if (doclet.undocumented
+				|| doclet.ignore
+				|| (doclet.memberof && rAnonymous.test(doclet.memberof))
+				|| doclet.longname.includes("~") ) {
+				continue;
+			}
 
 			// check for duplicates: last one wins
 			if ( j > 0 && doclets[j - 1].longname === doclet.longname ) {
 				if ( !doclets[j - 1].synthetic && !doclet.__ui5.updatedDoclet ) {
-					// replacing synthetic comments or updating comments are trivial case. Just log non-trivial duplicates
+					// replacing synthetic comments or updating comments are trivial cases. Just log non-trivial duplicates
 					debug(`ignoring duplicate doclet for ${doclet.longname}: ${location(doclet)} overrides ${location(doclets[j - 1])}`);
 				}
 				doclets[j - 1] = doclet;
-			} else {
-				doclets[j++] = doclet;
+				continue;
 			}
+
+			doclets[j++] = doclet;
 		}
 
 		if ( j < l ) {
 			doclets.splice(j, l - j);
-			info(`removed ${l - j} duplicate symbols - ${doclets.length} remaining`);
+			info(`processingComplete: removed ${l - j} undocumented, ignored, anonymous, local or duplicate symbols - ${doclets.length} remaining`);
 		}
 
 		if ( pluginConfig.saveSymbols ) {
-
 			fs.mkPath(env.opts.destination);
-			fs.writeFileSync(path.join(env.opts.destination, "symbols-parseComplete.json"), JSON.stringify(e.doclets, null, "\t"), 'utf8');
-
+			fs.writeFileSync(path.join(env.opts.destination, "symbols-processingComplete.json"), JSON.stringify(e.doclets, null, "\t"), 'utf8');
 		}
-
 	}
 };