core.16; Don't rely on _id; compound index on key,lang; remove dupes (#184)

Author: gadicc

README.md

@@ -16,7 +16,7 @@ Current versions of each package (requires manual, explicit updates until
 the stable release):
 
 ```
-meteor add msgfmt:core@2.0.0-preview.15 (2015-12-23)
+meteor add msgfmt:core@2.0.0-preview.16         # 2016-01-11
 meteor add msgfmt:extract@2.0.0-preview.12
 meteor add msgfmt:ui@2.0.0-preview.5
 ```

msgfmt:core/.versions

@@ -31,15 +31,15 @@ http@1.1.1
 id-map@1.0.4
 jag:pince@0.0.6
 jquery@1.11.4
-local-test:msgfmt:core@2.0.0-preview.15
+local-test:msgfmt:core@2.0.0-preview.16
 logging@1.0.8
 meteor@1.1.10
 meteorhacks:inject-initial@1.0.2
 minifiers@1.1.7
 minimongo@1.0.10
 mongo@1.1.3
 mongo-id@1.0.1
-msgfmt:core@2.0.0-preview.15
+msgfmt:core@2.0.0-preview.16
 npm-mongo@1.4.39_1
 observe-sequence@1.0.7
 ordered-dict@1.0.4

msgfmt:core/History.md

@@ -1,5 +1,10 @@
 ## vNEXT
 
+## 2.0.0-preview.16
+
+* Don't rely on `_id` in translation data.  Set compound index in mongo
+  on (key, lang).  Remove possible duplicates in database when upgrading.
+
 ## 2.0.0-preview.15
 
 * Run `syncAll` in it's own Fiber to avoid blocking on load

msgfmt:core/lib/mfPkg/messageformat-server.js

@@ -1,7 +1,7 @@
 var Fiber = Npm.require('fibers');
 
 // server only, but used on the client in msgfmt:ui
-mfPkg.mfRevisions = new Mongo.Collection('mfRevisions');
+msgfmt.mfRevisions = new Mongo.Collection('mfRevisions');
 
 /*
  * New entry point for startup (native) and any language modification.  Check that all
@@ -41,7 +41,8 @@ mfPkg.langUpdate = function(lang, strings, meta, lastSync) {
 	 */
 
 	var str, existing, revisionId, obj, updating, dbInsert, result, query,
-		optional = ['_id', 'file', 'line', 'template', 'func', 'removed', 'fuzzy'];
+    // removed in core.16; _id
+		optional = [/*'_id',*/ 'file', 'line', 'template', 'func', 'removed', 'fuzzy'];
 	for (key in strings) {
 		str = strings[key];
 		existing = this.strings[lang][key];
@@ -81,6 +82,8 @@ mfPkg.langUpdate = function(lang, strings, meta, lastSync) {
 			if (str[optional[i]])
 				obj[optional[i]] = str[optional[i]];
 
+    /*
+     - since core.16 we no longer worry about _id's
 		// insert unfound, or re-insert on wrong _id, otherwise update
 		if (existing) {
 			if (str._id && existing._id === str._id) {
@@ -93,6 +96,7 @@ mfPkg.langUpdate = function(lang, strings, meta, lastSync) {
 		} else {
 			dbInsert = true;
 		}
+    */
 
 		/* meteor upsert does allow _id even for insert upsert
 		if (this.strings[lang][key] && str._id
@@ -108,11 +112,16 @@ mfPkg.langUpdate = function(lang, strings, meta, lastSync) {
 			this.strings[lang][key] = obj;
 		*/
 
+    /*
 		if (dbInsert) {
-			obj._id = this.mfStrings.insert(obj)
+			// obj._id = this.mfStrings.insert(obj)  - v16 deprecates use of _id
+      obj._id = this.mfStrings.insert(obj)
 		} else {
-			this.mfStrings.update(obj._id, obj);
+			// this.mfStrings.update(obj._id, obj);  - v16 deprecates use of _id
+      this.mfStrings.update({ key: obj.key, lang: obj.lang }, obj);
 		}
+    */
+    this.mfStrings.upsert({ key: obj.key, lang: obj.lang }, obj);
 
 		if (updating) {
 			// does this update affect translations?
@@ -377,13 +386,51 @@ WebApp.connectHandlers.use(function(req, res, next) {
 
 msgfmt.strings = {};
 
+/*
+ * As of preview.16, we no longer rely on `_id` and properly set a unique compound
+ * index on `key` and `lang`.  In case the database has previous dups, let's clean
+ * them up.
+ */
+
+var checkForDupes = (msgfmt.meta.all && !msgfmt.meta.all._dupeFree);
+if (checkForDupes) {
+  msgfmt.mfMeta.update('all', { $set: { _dupeFree: 1 }} );
+  msgfmt.meta.all._dupeFree = true;
+  log.warn('Checking for dupes in mfStrings... (once off on upgrade from < core.15)');
+}
+
 // Yes, we absolutely do want this to block, to be fully loaded before mfAll.js etc
+// But in future we could see what else needs this and queue in a Fiber
+// e.g. make sure when requesting lang data that we're fully loaded, etc.
+
 log.trace('Retrieving strings from database...');
 var startTime = Date.now();
 var allStrings = msgfmt.mfStrings.find().fetch();
 log.trace('Finished retrieval in ' + (Date.now() - startTime) + ' ms');
-_.each(allStrings, function(str) {
-  checkLocaleMetaExists(str.lang);
-  msgfmt.strings[str.lang][str.key] = str;
-});
+if (checkForDupes) {
+  _.each(allStrings, function(str) {
+    checkLocaleMetaExists(str.lang);
+    var existing = msgfmt.strings[str.lang][str.key];
+    if (existing) {
+      log.warn('Duplicate "' + str.key + '" (' + str.lang + '), keeping latest.');
+      if (str.mtime > existing.mtime) {
+        msgfmt.strings[str.lang][str.key] = str;  // overwrite with newer
+        msgfmt.mfStrings.remove({ key: existing.key, lang: existing.lang });
+      } else {
+        // current str is older, remove it, don't overwrite in msgfmt.strings
+        msgfmt.mfStrings.remove({ key: str.key, lang: str.lang });
+      }
+    } else
+      msgfmt.strings[str.lang][str.key] = str;
+  });
+} else {
+  _.each(allStrings, function(str) {
+    checkLocaleMetaExists(str.lang);
+    msgfmt.strings[str.lang][str.key] = str;
+  });  
+}
 delete allStrings;
+
+// ensure compound indexe on server only (collection exists in both)
+// move to top of file when we're pretty sure everyone is _dupeFree
+msgfmt.mfStrings._ensureIndex( { key:1, lang:1 }, { unique: true } );

msgfmt:core/package.js

@@ -1,6 +1,6 @@
 Package.describe({
   name:    "msgfmt:core",
-  version: "2.0.0-preview.15",
+  version: "2.0.0-preview.16",
   summary: "MessageFormat support, the Meteor way",
   git:     "https://github.com/gadicc/meteor-messageformat.git",
 });

msgfmt:extract/README.md

@@ -4,3 +4,10 @@
 // for now, like this
 msgfmt.extractLogLevel = 'trace';   // debug, warn, info
 ```
+
+**Force rescan**:
+
+```bash
+meteor mongo
+db.mfExtractFiles.remove({});
+```

website/.meteor/versions

@@ -57,9 +57,9 @@ mobile-status-bar@1.0.3
 momentjs:moment@2.10.6
 mongo@1.1.0
 mrt:sp-marked@0.0.2
-msgfmt:core@2.0.0-preview.14
+msgfmt:core@2.0.0-preview.16
 msgfmt:extract@2.0.0-preview.12
-msgfmt:ui@2.0.0-preview.5
+msgfmt:ui@2.0.0-preview.6
 nicolaslopezj:router-layer@0.0.11
 npm-bcrypt@0.7.8_2
 oauth@1.1.4