Merge pull request #24 from rust-mobile/ime-support

Input method (soft keyboard) support

Author: rib

android-activity/Cargo.toml

@@ -25,6 +25,7 @@ native-activity = []
 [dependencies]
 log = "0.4"
 jni-sys = "0.3"
+cesu8 = "1"
 ndk = "0.7"
 ndk-sys = "0.4"
 ndk-context = "0.1"

android-activity/game-activity-csrc/game-activity/native_app_glue/android_native_app_glue.c

@@ -674,6 +674,7 @@ static void onTextInputEvent(GameActivity* activity,
     pthread_mutex_lock(&android_app->mutex);
     if (!android_app->destroyed) {
         android_app->textInputState = 1;
+        notifyInput(android_app);
     }
     pthread_mutex_unlock(&android_app->mutex);
 }

android-activity/src/game_activity/input.rs

@@ -26,6 +26,7 @@ use crate::input::{Class, Source};
 pub enum InputEvent<'a> {
     MotionEvent(MotionEvent<'a>),
     KeyEvent(KeyEvent<'a>),
+    TextEvent(crate::input::TextInputState),
 }
 
 /// A bitfield representing the state of modifier keys during an event.

android-activity/src/game_activity/mod.rs

@@ -32,6 +32,7 @@ use crate::{
 mod ffi;
 
 pub mod input;
+use crate::input::{TextInputState, TextSpan};
 use input::{Axis, InputEvent, KeyEvent, MotionEvent};
 
 // The only time it's safe to update the android_app->savedState pointer is
@@ -360,6 +361,121 @@ impl AndroidAppInner {
         }
     }
 
+    unsafe extern "C" fn map_input_state_to_text_event_callback(
+        context: *mut c_void,
+        state: *const ffi::GameTextInputState,
+    ) {
+        // Java uses a modified UTF-8 format, which is a modified cesu8 format
+        let out_ptr: *mut TextInputState = context.cast();
+        let text_modified_utf8: *const u8 = (*state).text_UTF8.cast();
+        let text_modified_utf8 =
+            std::slice::from_raw_parts(text_modified_utf8, (*state).text_length as usize);
+        match cesu8::from_java_cesu8(&text_modified_utf8) {
+            Ok(str) => {
+                let len = *&str.len();
+                (*out_ptr).text = String::from(str);
+
+                let selection_start = (*state).selection.start.clamp(0, len as i32 + 1);
+                let selection_end = (*state).selection.end.clamp(0, len as i32 + 1);
+                (*out_ptr).selection = TextSpan {
+                    start: selection_start as usize,
+                    end: selection_end as usize,
+                };
+                if (*state).composingRegion.start < 0 || (*state).composingRegion.end < 0 {
+                    (*out_ptr).compose_region = None;
+                } else {
+                    (*out_ptr).compose_region = Some(TextSpan {
+                        start: (*state).composingRegion.start as usize,
+                        end: (*state).composingRegion.end as usize,
+                    });
+                }
+            }
+            Err(err) => {
+                log::error!("Invalid UTF8 text in TextEvent: {}", err);
+            }
+        }
+    }
+
+    // TODO: move into a trait
+    pub fn text_input_state(&self) -> TextInputState {
+        unsafe {
+            let activity = (*self.native_app.as_ptr()).activity;
+            let mut out_state = TextInputState {
+                text: String::new(),
+                selection: TextSpan { start: 0, end: 0 },
+                compose_region: None,
+            };
+            let out_ptr = &mut out_state as *mut TextInputState;
+
+            // NEON WARNING:
+            //
+            // It's not clearly documented but the GameActivity API over the
+            // GameTextInput library directly exposes _modified_ UTF8 text
+            // from Java so we need to be careful to convert text to and
+            // from UTF8
+            //
+            // GameTextInput also uses a pre-allocated, fixed-sized buffer for the current
+            // text state but GameTextInput doesn't actually provide it's own thread
+            // safe API to safely access this state  so we have to cooperate with
+            // the GameActivity code that does locking when reading/writing the state
+            // (I.e. we can't just punch through to the GameTextInput layer from here).
+            //
+            // Overall this is all quite gnarly - and probably a good reminder of why
+            // we want to use Rust instead of C/C++.
+            ffi::GameActivity_getTextInputState(
+                activity,
+                Some(AndroidAppInner::map_input_state_to_text_event_callback),
+                out_ptr.cast(),
+            );
+
+            out_state
+        }
+    }
+
+    // TODO: move into a trait
+    pub fn set_text_input_state(&self, state: TextInputState) {
+        unsafe {
+            let activity = (*self.native_app.as_ptr()).activity;
+            let modified_utf8 = cesu8::to_java_cesu8(&state.text);
+            let text_length = modified_utf8.len() as i32;
+            let modified_utf8_bytes = modified_utf8.as_ptr();
+            let ffi_state = ffi::GameTextInputState {
+                text_UTF8: modified_utf8_bytes.cast(), // NB: may be signed or unsigned depending on target
+                text_length,
+                selection: ffi::GameTextInputSpan {
+                    start: state.selection.start as i32,
+                    end: state.selection.end as i32,
+                },
+                composingRegion: match state.compose_region {
+                    Some(span) => {
+                        // The GameText subclass of InputConnection only has a special case for removing the
+                        // compose region if `start == -1` but the docs for `setComposingRegion` imply that
+                        // the region should effectively be removed if any empty region is given (unlike for the
+                        // selection region, it's not meaningful to maintain an empty compose region)
+                        //
+                        // We aim for more consistent behaviour by normalizing any empty region into `(-1, -1)`
+                        // to remove the compose region.
+                        //
+                        // NB `setComposingRegion` itself is documented to clamp start/end to the text bounds
+                        // so apart from this special-case handling in GameText's implementation of
+                        // `setComposingRegion` then there's nothing special about `(-1, -1)` - it's just an empty
+                        // region that should get clamped to `(0, 0)` and then get removed.
+                        if span.start == span.end {
+                            ffi::GameTextInputSpan { start: -1, end: -1 }
+                        } else {
+                            ffi::GameTextInputSpan {
+                                start: span.start as i32,
+                                end: span.end as i32,
+                            }
+                        }
+                    }
+                    None => ffi::GameTextInputSpan { start: -1, end: -1 },
+                },
+            };
+            ffi::GameActivity_setTextInputState(activity, &ffi_state as *const _);
+        }
+    }
+
     pub fn enable_motion_axis(&mut self, axis: Axis) {
         unsafe { ffi::GameActivityPointerAxes_enableAxis(axis as i32) }
     }
@@ -403,7 +519,7 @@ impl AndroidAppInner {
         }
     }
 
-    pub fn input_events<F>(&self, mut callback: F)
+    fn dispatch_key_and_motion_events<F>(&self, mut callback: F)
     where
         F: FnMut(&InputEvent) -> InputStatus,
     {
@@ -426,6 +542,28 @@ impl AndroidAppInner {
         }
     }
 
+    fn dispatch_text_events<F>(&self, mut callback: F)
+    where
+        F: FnMut(&InputEvent) -> InputStatus,
+    {
+        unsafe {
+            let app_ptr = self.native_app.as_ptr();
+            if (*app_ptr).textInputState != 0 {
+                let state = self.text_input_state();
+                callback(&InputEvent::TextEvent(state));
+                (*app_ptr).textInputState = 0;
+            }
+        }
+    }
+
+    pub fn input_events<F>(&self, mut callback: F)
+    where
+        F: FnMut(&InputEvent) -> InputStatus,
+    {
+        self.dispatch_key_and_motion_events(&mut callback);
+        self.dispatch_text_events(&mut callback);
+    }
+
     pub fn internal_data_path(&self) -> Option<std::path::PathBuf> {
         unsafe {
             let app_ptr = self.native_app.as_ptr();

android-activity/src/input.rs

@@ -81,3 +81,41 @@ impl From<Source> for Class {
         source.into()
     }
 }
+
+/// This struct holds a span within a region of text from `start` to `end`.
+///
+/// The `start` index may be greater than the `end` index (swapping `start` and `end` will represent the same span)
+///
+/// The lower index is inclusive and the higher index is exclusive.
+///
+/// An empty span or cursor position is specified with `start == end`.
+///
+#[derive(Debug, Clone, Copy)]
+pub struct TextSpan {
+    /// The start of the span (inclusive)
+    pub start: usize,
+
+    /// The end of the span (exclusive)
+    pub end: usize,
+}
+
+#[derive(Debug, Clone)]
+pub struct TextInputState {
+    pub text: String,
+
+    /// A selection defined on the text.
+    ///
+    /// To set the cursor position, start and end should have the same value.
+    ///
+    /// Changing the selection has no effect on the compose_region.
+    pub selection: TextSpan,
+
+    /// A composing region defined on the text.
+    ///
+    /// When being set, then if there was a composing region, the region is replaced.
+    ///
+    /// The given indices will be clamped to the `text` bounds
+    ///
+    /// If the resulting region is zero-sized, no region is marked (equivalent to passing `None`)
+    pub compose_region: Option<TextSpan>,
+}

android-activity/src/lib.rs

@@ -618,6 +618,16 @@ impl AndroidApp {
             .hide_soft_input(hide_implicit_only);
     }
 
+    /// Fetch the current input text state, as updated by any active IME.
+    pub fn text_input_state(&self) -> input::TextInputState {
+        self.inner.read().unwrap().text_input_state()
+    }
+
+    /// Forward the given input text `state` to any active IME.
+    pub fn set_text_input_state(&self, state: input::TextInputState) {
+        self.inner.read().unwrap().set_text_input_state(state);
+    }
+
     /// Query and process all out-standing input event
     ///
     /// `callback` should return [`InputStatus::Unhandled`] for any input events that aren't directly

android-activity/src/native_activity/input.rs

@@ -337,4 +337,5 @@ impl<'a> KeyEvent<'a> {
 pub enum InputEvent<'a> {
     MotionEvent(self::MotionEvent<'a>),
     KeyEvent(self::KeyEvent<'a>),
+    TextEvent(crate::input::TextInputState),
 }

android-activity/src/native_activity/mod.rs

@@ -9,6 +9,7 @@ use libc::c_void;
 use log::{error, trace};
 use ndk::{asset::AssetManager, native_window::NativeWindow};
 
+use crate::input::{TextInputState, TextSpan};
 use crate::{
     util, AndroidApp, ConfigurationRef, InputStatus, MainEvent, PollEvent, Rect, WindowManagerFlags,
 };
@@ -341,6 +342,20 @@ impl AndroidAppInner {
         }
     }
 
+    // TODO: move into a trait
+    pub fn text_input_state(&self) -> TextInputState {
+        TextInputState {
+            text: String::new(),
+            selection: TextSpan { start: 0, end: 0 },
+            compose_region: None,
+        }
+    }
+
+    // TODO: move into a trait
+    pub fn set_text_input_state(&self, _state: TextInputState) {
+        // NOP: Unsupported
+    }
+
     pub fn enable_motion_axis(&self, _axis: input::Axis) {
         // NOP - The InputQueue API doesn't let us optimize which axis values are read
     }
@@ -390,6 +405,7 @@ impl AndroidAppInner {
                     input::InputEvent::KeyEvent(e) => {
                         ndk::event::InputEvent::KeyEvent(e.into_ndk_event())
                     }
+                    _ => unreachable!(),
                 };
                 queue.finish_event(ndk_event, matches!(handled, InputStatus::Handled));
             }