Use std libraries for UTF16 -> UTF8 conversion in hunspell spellchecker

Author: maxbrunsfeld

binding.gyp

@@ -41,17 +41,20 @@
         }],
         ['OS=="win"', {
           'sources': [
-             'src/spellchecker_win.cc'
+             'src/spellchecker_win.cc',
+             'src/transcoder_win.cc',
           ],
         }],
         ['OS=="linux"', {
           'sources': [
-             'src/spellchecker_linux.cc'
+             'src/spellchecker_linux.cc',
+             'src/transcoder_posix.cc',
           ],
         }],
         ['OS=="mac"', {
           'sources': [
             'src/spellchecker_mac.mm',
+            'src/transcoder_posix.cc',
           ],
           'link_settings': {
             'libraries': [

src/spellchecker_hunspell.cc

@@ -2,16 +2,20 @@
 #include <cwctype>
 #include <algorithm>
 #include "../vendor/hunspell/src/hunspell/hunspell.hxx"
-#include "../vendor/hunspell/src/hunspell/csutil.hxx"
 #include "spellchecker_hunspell.h"
 
 namespace spellchecker {
 
-HunspellSpellchecker::HunspellSpellchecker() : hunspell(NULL) { }
+HunspellSpellchecker::HunspellSpellchecker() : hunspell(NULL), transcoder(NewTranscoder()) { }
+
 HunspellSpellchecker::~HunspellSpellchecker() {
   if (hunspell) {
     delete hunspell;
   }
+
+  if (transcoder) {
+    FreeTranscoder(transcoder);
+  }
 }
 
 bool HunspellSpellchecker::SetDictionary(const std::string& language, const std::string& dirname) {
@@ -53,27 +57,29 @@ bool HunspellSpellchecker::IsMisspelled(const std::string& word) {
 std::vector<MisspelledRange> HunspellSpellchecker::CheckSpelling(const uint16_t *utf16_text, size_t utf16_length) {
   std::vector<MisspelledRange> result;
 
-  if (!hunspell) {
+  if (!hunspell || !transcoder) {
     return result;
   }
 
   std::vector<char> utf8_buffer(256);
-  char *utf8_word = utf8_buffer.data();
 
   size_t word_start = 0;
   bool within_word = false;
   for (size_t i = 0; i < utf16_length; i++) {
-    bool is_word_character = iswalpha(utf16_text[i]);
+    uint16_t c = utf16_text[i];
+    bool is_word_character = iswalpha(c);
     if (within_word) {
       if (!is_word_character) {
         within_word = false;
-        const w_char *utf16_word = reinterpret_cast<const w_char *>(utf16_text + word_start);
-        u16_u8(utf8_word, utf8_buffer.size(), utf16_word, i - word_start);
-        if (hunspell->spell(utf8_word) == 0) {
-          MisspelledRange range;
-          range.start = word_start;
-          range.end = i;
-          result.push_back(range);
+
+        bool converted = TranscodeUTF16ToUTF8(transcoder, (char *)utf8_buffer.data(), utf8_buffer.size(), utf16_text + word_start, i - word_start);
+        if (converted) {
+          if (hunspell->spell(utf8_buffer.data()) == 0) {
+            MisspelledRange range;
+            range.start = word_start;
+            range.end = i;
+            result.push_back(range);
+          }
         }
       }
     } else if (is_word_character) {

src/spellchecker_hunspell.h

@@ -2,6 +2,7 @@
 #define SRC_SPELLCHECKER_HUNSPELL_H_
 
 #include "spellchecker.h"
+#include "transcoder.h"
 
 class Hunspell;
 
@@ -21,6 +22,7 @@ class HunspellSpellchecker : public SpellcheckerImplementation {
 
 private:
   Hunspell* hunspell;
+  Transcoder *transcoder;
 };
 
 }  // namespace spellchecker

src/transcoder.h

@@ -0,0 +1,17 @@
+#ifndef SRC_TRANSCODER_H_
+#define SRC_TRANSCODER_H_
+
+#include <stdlib.h>
+#include <stdint.h>
+
+namespace spellchecker {
+
+struct Transcoder;
+
+Transcoder *NewTranscoder();
+void FreeTranscoder(Transcoder *);
+bool TranscodeUTF16ToUTF8(const Transcoder *, char *out, size_t out_length, const uint16_t *in, size_t in_length);
+
+}  // namespace spellchecker
+
+#endif  // SRC_TRANSCODER_H_

src/transcoder_posix.cc

@@ -0,0 +1,59 @@
+#include <iconv.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+namespace spellchecker {
+
+struct Transcoder {
+  iconv_t conversion;
+};
+
+static int IsBigEndian(void) {
+  union {
+    uint16_t integer;
+    char bytes[2];
+  } two_byte_value;
+
+  two_byte_value.integer = {0x0102};
+  return two_byte_value.bytes[0] == 1;
+}
+
+Transcoder *NewTranscoder() {
+  const char *to_encoding = "UTF-8";
+  const char *from_encoding = IsBigEndian() ? "UTF-16BE" : "UTF-16LE";
+  iconv_t conversion = iconv_open(to_encoding, from_encoding);
+  if (conversion == (iconv_t)-1) {
+    return NULL;
+  }
+
+  Transcoder *result = new Transcoder();
+  result->conversion = conversion;
+  return result;
+}
+
+void FreeTranscoder(Transcoder *transcoder) {
+  iconv_close(transcoder->conversion);
+  delete transcoder;
+}
+
+bool TranscodeUTF16ToUTF8(const Transcoder *transcoder, char *out, size_t out_bytes, const uint16_t *in, size_t in_length) {
+  char *utf16_word = reinterpret_cast<char *>(const_cast<uint16_t *>(in));
+  size_t utf16_bytes = in_length * (sizeof(uint16_t) / sizeof(char));
+
+  size_t iconv_result = iconv(
+    transcoder->conversion,
+    &utf16_word,
+    &utf16_bytes,
+    &out,
+    &out_bytes
+  );
+
+  if (iconv_result == static_cast<size_t>(-1)) {
+    return false;
+  }
+
+  *out = '\0';
+  return true;
+}
+
+}  // namespace spellchecker

src/transcoder_win.cc

@@ -0,0 +1,19 @@
+#include <windows.h>
+#include <stdlib.h>
+#include "transcoder.h"
+
+namespace spellchecker {
+
+struct Transcoder {};
+
+Transcoder *NewTranscoder() { return NULL; }
+
+void FreeTranscoder(Transcoder *transcoder) {}
+
+bool TranscodeUTF16ToUTF8(const Transcoder *transcoder, char *out, size_t out_length, const uint16_t *in, size_t in_length) {
+  int length = WideCharToMultiByte(CP_UTF8, 0, reinterpret_cast<const wchar_t *>(in), in_length, out, out_length, NULL, NULL);
+  out[length] = '\0';
+  return true;
+}
+
+}  // namespace spellchecker