Skip to content

Commit a011f0b

Browse files
CaydenPierceCaydenPierce
committed
Merge branch 'dev'
2 parents e6db691 + 19bead8 commit a011f0b

File tree

18 files changed

+402
-5
lines changed

18 files changed

+402
-5
lines changed

‎android_smart_glasses/main/.idea/misc.xml

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎android_smart_glasses/main/app/src/main/java/com/wearableintelligencesystem/androidsmartglasses/ASPClientSocket.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -523,7 +523,7 @@ private void parseData(JSONObject data){
523523
String typeOf = data.getString(MessageTypes.MESSAGE_TYPE_LOCAL);
524524

525525
//the first bit does a bunch of if statement. The new, better way of doing things is our syncing of data across ASP and ASG with MessageTypes shared class. Now, we just pass a JSON string through to the UI if it's a MessageType that it needs to see, and let the UI deal with how to parse/handle it
526-
String [] uiMessages = new String[] {MessageTypes.NATURAL_LANGUAGE_QUERY, MessageTypes.SEARCH_ENGINE_RESULT, MessageTypes.VISUAL_SEARCH_RESULT, MessageTypes.TRANSLATE_TEXT_RESULT, MessageTypes.ACTION_SWITCH_MODES, MessageTypes.REFERENCE_SELECT_REQUEST, MessageTypes.VOICE_COMMAND_STREAM_EVENT};
526+
String [] uiMessages = new String[] {MessageTypes.OBJECT_TRANSLATION_RESULT, MessageTypes.NATURAL_LANGUAGE_QUERY, MessageTypes.SEARCH_ENGINE_RESULT, MessageTypes.VISUAL_SEARCH_RESULT, MessageTypes.TRANSLATE_TEXT_RESULT, MessageTypes.ACTION_SWITCH_MODES, MessageTypes.REFERENCE_SELECT_REQUEST, MessageTypes.VOICE_COMMAND_STREAM_EVENT};
527527
for (String uiMessage : uiMessages){
528528
if (typeOf.equals(uiMessage)){
529529
final Intent intent = new Intent();

‎android_smart_glasses/main/app/src/main/java/com/wearableintelligencesystem/androidsmartglasses/MainActivity.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,10 @@ private void switchMode(String mode) {
298298
navController.navigate(R.id.nav_language_translate);
299299
modeDisplayName = "Lang.Trans.";
300300
break;
301+
case MessageTypes.MODE_OBJECT_TRANSLATE:
302+
navController.navigate(R.id.nav_object_translate);
303+
modeDisplayName = "Obj.Trans.";
304+
break;
301305
case MessageTypes.MODE_TEXT_LIST:
302306
setupTextList();
303307
modeDisplayName = "TextList";

‎android_smart_glasses/main/app/src/main/java/com/wearableintelligencesystem/androidsmartglasses/comms/MessageTypes.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,10 @@ public class MessageTypes {
112112
public final static String TRANSLATE_TEXT_RESULT = "TRANSLATE_TEXT_RESULT";
113113
public final static String TRANSLATE_TEXT_RESULT_DATA = "TRANSLATION_RESULT_DATA";
114114

115+
//object translation
116+
public final static String OBJECT_TRANSLATION_RESULT = "OBJECT_TRANSLATION_RESULT";
117+
public final static String OBJECT_TRANSLATION_RESULT_DATA = "OBJECT_TRANSLATION_RESULT_DATA";
118+
115119
public final static String AFFECTIVE_SUMMARY_RESULT = "AFFECTIVE_SUMMARY_RESULT";
116120
public final static String COMMAND_SWITCH_MODE = "COMMAND_SWITCH_MODE";
117121

@@ -126,6 +130,7 @@ public class MessageTypes {
126130
public final static String MODE_REFERENCE_GRID = "MODE_REFERENCE_GRID";
127131
public final static String MODE_WEARABLE_FACE_RECOGNIZER = "MODE_WEARABLE_FACE_RECOGNIZER";
128132
public final static String MODE_LANGUAGE_TRANSLATE = "MODE_LANGUAGE_TRANSLATE";
133+
public final static String MODE_OBJECT_TRANSLATE = "MODE_OBJECT_TRANSLATE";
129134
public final static String MODE_TEXT_LIST = "MODE_TEXT_LIST";
130135
public final static String MODE_TEXT_BLOCK = "MODE_TEXT_BLOCK";
131136
public final static String MODE_BLANK = "MODE_BLANK";

‎android_smart_glasses/main/app/src/main/java/com/wearableintelligencesystem/androidsmartglasses/ui/ObjectTranslateUi.java

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package com.wearableintelligencesystem.androidsmartglasses.ui;
2+
3+
import android.content.Context;
4+
import android.content.Intent;
5+
import android.content.IntentFilter;
6+
import android.graphics.Bitmap;
7+
import android.graphics.BitmapFactory;
8+
import android.os.Bundle;
9+
import android.os.Handler;
10+
import android.util.Log;
11+
import android.view.KeyEvent;
12+
import android.view.LayoutInflater;
13+
import android.view.View;
14+
import android.view.ViewGroup;
15+
import android.widget.ImageView;
16+
import android.widget.TextView;
17+
import android.widget.Toast;
18+
19+
import androidx.annotation.NonNull;
20+
import androidx.annotation.Nullable;
21+
22+
import com.example.wearableintelligencesystemandroidsmartglasses.R;
23+
import com.wearableintelligencesystem.androidsmartglasses.ASPClientSocket;
24+
import com.wearableintelligencesystem.androidsmartglasses.comms.MessageTypes;
25+
26+
import org.json.JSONArray;
27+
import org.json.JSONException;
28+
import org.json.JSONObject;
29+
import org.w3c.dom.Text;
30+
31+
import java.util.List;
32+
33+
public class ObjectTranslateUi extends ASGFragment {
34+
private final String TAG = "WearableAi_ObjectTranslateUi";
35+
36+
private TextView languageName;
37+
private TextView nameOfObjectinSourceLanguage;
38+
private TextView nameOfObjectInTargetLanguage;
39+
40+
public ObjectTranslateUi() {
41+
fragmentLabel = "Object Translate";
42+
}
43+
44+
@Override
45+
public void onCreate(Bundle savedInstanceState) {
46+
super.onCreate(savedInstanceState);
47+
}
48+
49+
@Override
50+
public View onCreateView(LayoutInflater inflater, ViewGroup container,
51+
Bundle savedInstanceState) {
52+
// Inflate the layout for this fragment
53+
return inflater.inflate(R.layout.object_translate_mode_view, container, false);
54+
}
55+
56+
@Override
57+
public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceState) {
58+
super.onCreate(savedInstanceState);
59+
super.onViewCreated(view, savedInstanceState);
60+
61+
nameOfObjectinSourceLanguage = (TextView) view.findViewById(R.id.nameOfObjectinSourceLanguage);
62+
nameOfObjectInTargetLanguage = (TextView) view.findViewById(R.id.nameOfObjectinTargetLanguage);
63+
languageName = (TextView) view.findViewById(R.id.languageName);
64+
}
65+
66+
67+
@Override
68+
protected void onBroadcastReceive(Context context, Intent intent){
69+
Log.d("hereee", "kjbnkjb");
70+
final String action = intent.getAction();
71+
try {
72+
JSONObject data = new JSONObject(intent.getStringExtra(ASPClientSocket.RAW_MESSAGE_JSON_STRING));
73+
String typeOf = data.getString(MessageTypes.MESSAGE_TYPE_LOCAL);
74+
//preview object translation result data
75+
if (typeOf.equals(MessageTypes.OBJECT_TRANSLATION_RESULT)) {
76+
showObjectTranslationResults((new JSONObject(data.getString(MessageTypes.OBJECT_TRANSLATION_RESULT_DATA))));
77+
}
78+
} catch (JSONException e){
79+
e.printStackTrace();
80+
}
81+
}
82+
83+
@Override
84+
public void onPause(){
85+
super.onPause();
86+
}
87+
88+
@Override
89+
public void onResume(){
90+
super.onResume();
91+
}
92+
93+
private void showObjectTranslationResults(JSONObject objectTranslatedData){
94+
//show the reference
95+
try {
96+
nameOfObjectinSourceLanguage.setText(objectTranslatedData.getString("source"));
97+
nameOfObjectInTargetLanguage.setText(objectTranslatedData.getString("target"));
98+
//languageName.setText(objectTranslatedData.getString("French"));
99+
} catch (JSONException e) {
100+
e.printStackTrace();
101+
}
102+
103+
}
104+
105+
@Override
106+
public IntentFilter makeComputeUpdateIntentFilter() {
107+
final IntentFilter intentFilter = new IntentFilter();
108+
intentFilter.addAction(ASPClientSocket.ACTION_UI_DATA);
109+
110+
return intentFilter;
111+
}
112+
}
113+

‎android_smart_glasses/main/app/src/main/res/drawable/circle_shape.xml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<shape
3+
xmlns:android="http://schemas.android.com/apk/res/android"
4+
android:shape="oval">
5+
6+
<solid
7+
android:color="#50ffffff"/>
8+
9+
<size
10+
android:width="30dp"
11+
android:height="30dp"/>
12+
</shape>

‎android_smart_glasses/main/app/src/main/res/layout/object_translate_mode_view.xml

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
3+
<androidx.coordinatorlayout.widget.CoordinatorLayout xmlns:android="http://schemas.android.com/apk/res/android"
4+
xmlns:tools="http://schemas.android.com/tools"
5+
android:layout_width="match_parent"
6+
android:layout_height="match_parent"
7+
xmlns:app="http://schemas.android.com/apk/res-auto"
8+
android:background="#00000000">
9+
10+
<RelativeLayout
11+
android:layout_width="match_parent"
12+
android:layout_height="match_parent"
13+
android:background="@android:color/black"
14+
android:orientation="vertical">
15+
<TextView
16+
android:id="@+id/languageName"
17+
android:layout_width="wrap_content"
18+
android:layout_height="wrap_content"
19+
android:layout_alignParentTop="true"
20+
android:layout_centerHorizontal="true"
21+
android:layout_marginTop="30dp"
22+
android:text="French"
23+
android:textColor="#FFFFFF"
24+
android:textSize="16sp" />
25+
26+
<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
27+
xmlns:tools="http://schemas.android.com/tools"
28+
android:id="@+id/container"
29+
android:layout_width="match_parent"
30+
android:layout_height="match_parent"
31+
android:alpha="0.0"
32+
android:visibility="visible"
33+
tools:context="org.tensorflow.demo.CameraActivity" />
34+
35+
36+
</RelativeLayout>
37+
38+
<RelativeLayout
39+
android:layout_width="match_parent"
40+
android:layout_height="match_parent"
41+
>
42+
43+
<View
44+
android:layout_width="40dp"
45+
android:layout_height="40dp"
46+
android:elevation="12dp"
47+
android:background="@drawable/circle_shape"
48+
android:layout_centerInParent="true" />
49+
50+
<TextView
51+
android:id="@+id/nameOfObjectinTargetLanguage"
52+
android:layout_width="wrap_content"
53+
android:layout_height="wrap_content"
54+
android:layout_above="@id/nameOfObjectinSourceLanguage"
55+
android:layout_centerHorizontal="true"
56+
android:layout_marginBottom="20dp"
57+
tools:text="Apple"
58+
android:textColor="#FFFFFF"
59+
android:textSize="22sp" />
60+
61+
<TextView
62+
android:id="@+id/nameOfObjectinSourceLanguage"
63+
android:layout_width="wrap_content"
64+
android:layout_height="wrap_content"
65+
android:layout_alignParentBottom="true"
66+
android:layout_centerHorizontal="true"
67+
android:layout_marginBottom="50dp"
68+
tools:text="pomme"
69+
android:textColor="#FFFFFF"
70+
android:textSize="16sp" />
71+
</RelativeLayout>
72+
73+
</androidx.coordinatorlayout.widget.CoordinatorLayout>

‎android_smart_glasses/main/app/src/main/res/navigation/nav_graph.xml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@
4242
android:label="Language Translate" >
4343
</fragment>
4444

45+
<fragment
46+
android:id="@+id/nav_object_translate"
47+
android:name="com.wearableintelligencesystem.androidsmartglasses.ui.ObjectTranslateUi"
48+
android:label="Object Translate"
49+
tools:layout="@layout/object_translate_mode_view"></fragment>
4550
<fragment
4651
android:id="@+id/nav_visual_search"
4752
android:name="com.wearableintelligencesystem.androidsmartglasses.ui.VisualSearchUi"

‎android_smart_phone/main/.idea/misc.xml

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎android_smart_phone/main/app/build.gradle

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ android {
5454

5555
dependencies {
5656
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
57+
implementation 'com.google.mlkit:translate:17.0.0'
5758
//implementation 'androidx.appcompat:appcompat:1.4.0'
5859
//implementation 'com.google.android.material:material:1.4.0'
5960
//implementation 'androidx.constraintlayout:constraintlayout:2.1.2'
@@ -100,6 +101,11 @@ dependencies {
100101
implementation "com.google.android.gms:play-services-tasks:18.0.0"
101102
implementation "com.google.android.gms:play-services-location:18.0.0"
102103
implementation "com.google.code.gson:gson:2.8.6"
104+
105+
//TensorFlow libraries
106+
implementation 'org.tensorflow:tensorflow-lite-task-vision:0.3.0'
107+
implementation 'org.tensorflow:tensorflow-lite-gpu-delegate-plugin:0.3.0'
108+
103109

104110
//ROOM
105111
// Room components
@@ -152,9 +158,9 @@ dependencies {
152158
implementation 'com.google.mlkit:face-detection:16.1.2'
153159

154160
////TensorFlow Lite dependencies
155-
implementation 'org.tensorflow:tensorflow-lite:2.4.0'
156-
implementation 'org.tensorflow:tensorflow-lite-gpu:2.4.0'
157-
implementation 'org.tensorflow:tensorflow-lite-support:0.1.0'
161+
implementation 'org.tensorflow:tensorflow-lite:2.7.0'
162+
implementation 'org.tensorflow:tensorflow-lite-gpu:2.7.0'
163+
implementation 'org.tensorflow:tensorflow-lite-support:0.3.0'
158164

159165
////Kotlin Coroutines
160166
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.1'

‎android_smart_phone/main/app/src/main/assets/coco_objectdetection.tflite

3.99 MB
Binary file not shown.

‎android_smart_phone/main/app/src/main/java/com/wearableintelligencesystem/androidsmartphone/ASGRepresentative.java

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,20 @@ public void speakTextToSpeechResults(String text, String languageCode){
284284
}
285285
}
286286

287+
public void sendObjectTranslationResults(JSONObject results){
288+
try{
289+
//build json object to send command result
290+
JSONObject commandResponseObject = new JSONObject();
291+
commandResponseObject.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.OBJECT_TRANSLATION_RESULT);
292+
commandResponseObject.put(MessageTypes.OBJECT_TRANSLATION_RESULT_DATA, results.toString());
293+
294+
//send the command result to web socket, to send to asg
295+
dataObservable.onNext(commandResponseObject);
296+
} catch (JSONException e){
297+
e.printStackTrace();
298+
}
299+
}
300+
287301
public void startAsgConnection(){
288302
killme = false;
289303

‎android_smart_phone/main/app/src/main/java/com/wearableintelligencesystem/androidsmartphone/GLBOXRepresentative.java

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,10 @@ public void handleDataStream(JSONObject data){
5454
sendTranslateRequest(data);
5555
} else if (type.equals(MessageTypes.REFERENCE_TRANSLATE_SEARCH_QUERY)) {
5656
sendReferenceTranslateQuery(data);
57+
}else if (type.equals(MessageTypes.OBJECT_TRANSLATION_REQUEST)) {
58+
sendObjectTranslateRequest(data);
5759
}
60+
5861
}
5962
catch (JSONException e){
6063
e.printStackTrace();
@@ -170,6 +173,39 @@ public void onFailure() {
170173
e.printStackTrace();
171174
}
172175
}
176+
public void sendObjectTranslateRequest(JSONObject data){
177+
Log.d(TAG, "Running sendObjectTranslateRequest");
178+
try{
179+
JSONObject restMessage = new JSONObject();
180+
restMessage.put("query", data.get(MessageTypes.TRANSCRIPT_TEXT));
181+
restMessage.put("source_language", "en");
182+
restMessage.put("target_language", "fr");
183+
restServerComms.restRequest(RestServerComms.TRANSLATE_TEXT_QUERY_ENDPOINT, restMessage, new VolleyCallback(){
184+
@Override
185+
public void onSuccess(JSONObject result){
186+
Log.d(TAG, "GOT translated TEXT RESULT:");
187+
Log.d(TAG, result.toString());
188+
JSONObject resultToSend = new JSONObject();
189+
try {
190+
resultToSend.put("source",data.get(MessageTypes.TRANSCRIPT_TEXT));
191+
resultToSend.put("target",result.getString("response"));
192+
asgRep.sendObjectTranslationResults(resultToSend);
193+
Log.d(TAG, resultToSend.toString());
194+
} catch (JSONException e) {
195+
asgRep.sendCommandResponse("Response Falied");
196+
e.printStackTrace();
197+
}
198+
}
199+
@Override
200+
public void onFailure() {
201+
202+
}
203+
});
204+
} catch (JSONException e){
205+
e.printStackTrace();
206+
}
207+
}
208+
173209
private void sendReferenceTranslateQuery(JSONObject data){
174210
Log.d(TAG, "Running sendReferenceTranslateQuery");
175211
try{

0 commit comments

Comments
 (0)