Skip to content

Commit 432bbac

Browse files
CaydenPierceCaydenPierce
committed
Merge branch 'dev'
2 parents d159567 + 7beacdc commit 432bbac

File tree

17 files changed

+250
-23
lines changed

17 files changed

+250
-23
lines changed

‎android_smart_phone/main/app/src/main/java/com/wearableintelligencesystem/androidsmartphone/GLBOXRepresentative.java

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,11 @@ public void handleDataStream(JSONObject data){
5858
sendReferenceTranslateQuery(data);
5959
}else if (type.equals(MessageTypes.OBJECT_TRANSLATION_REQUEST)) {
6060
sendObjectTranslateRequest(data);
61+
}else if (type.equals(MessageTypes.CONTEXTUAL_SEARCH_REQUEST)) {
62+
sendContextualSearch(data);
6163
} else if (type.equals(MessageTypes.FINAL_TRANSCRIPT)) {
6264
Log.d(TAG, "GOT FINAL TRANSCRIPT");
63-
sendFinalTranscript(data);
65+
//sendFinalTranscript(data);
6466
}
6567

6668
}
@@ -96,8 +98,8 @@ public void onFailure(){
9698
}
9799
}
98100

99-
public void sendFinalTranscript(JSONObject data){
100-
Log.d(TAG, "Running sendFinalTranscript");
101+
public void sendContextualSearch(JSONObject data){
102+
Log.d(TAG, "Running sendContextualSearch");
101103
try{
102104
JSONObject restMessage = new JSONObject();
103105
restMessage.put("transcript", data.getString(MessageTypes.TRANSCRIPT_TEXT).toLowerCase());
@@ -107,7 +109,7 @@ public void sendFinalTranscript(JSONObject data){
107109
restServerComms.restRequest(RestServerComms.FINAL_TRANSCRIPT_SEND_ENDPOINT, restMessage, new VolleyCallback(){
108110
@Override
109111
public void onSuccess(JSONObject result){
110-
asgRep.sendCommandResponse("Final transcript send success, displaying results.");
112+
asgRep.sendCommandResponse("Final transcript to contextual search send success, displaying results.");
111113
//check if there was a result at all
112114
try {
113115
boolean search_result = result.getBoolean("result");
@@ -120,15 +122,47 @@ public void onSuccess(JSONObject result){
120122
}
121123
@Override
122124
public void onFailure(){
123-
asgRep.sendCommandResponse("Semantic search failed, please try again.");
125+
asgRep.sendCommandResponse("Contextual search failed, please try again.");
124126
}
125-
126127
});
127128
} catch (JSONException e){
128129
e.printStackTrace();
129130
}
130131
}
131132

133+
//
134+
// public void sendFinalTranscript(JSONObject data){
135+
// Log.d(TAG, "Running sendFinalTranscript");
136+
// try{
137+
// JSONObject restMessage = new JSONObject();
138+
// restMessage.put("transcript", data.getString(MessageTypes.TRANSCRIPT_TEXT).toLowerCase());
139+
// restMessage.put("timestamp", data.getString(MessageTypes.TIMESTAMP));
140+
// restMessage.put("id", data.getString(MessageTypes.TRANSCRIPT_ID));
141+
//
142+
// restServerComms.restRequest(RestServerComms.FINAL_TRANSCRIPT_SEND_ENDPOINT, restMessage, new VolleyCallback(){
143+
// @Override
144+
// public void onSuccess(JSONObject result){
145+
// asgRep.sendCommandResponse("Final transcript send success, displaying results.");
146+
// //check if there was a result at all
147+
// try {
148+
// boolean search_result = result.getBoolean("result");
149+
// if (search_result) {
150+
// asgRep.sendSearchEngineResults(result);
151+
// }
152+
// } catch (JSONException e){
153+
// e.printStackTrace();
154+
// }
155+
// }
156+
// @Override
157+
// public void onFailure(){
158+
// asgRep.sendCommandResponse("Semantic search failed, please try again.");
159+
// }
160+
// });
161+
// } catch (JSONException e){
162+
// e.printStackTrace();
163+
// }
164+
// }
165+
132166
private void sendNaturalLanguageQuery(JSONObject data){
133167
Log.d(TAG, "Running sendNaturalLanguageQuery");
134168
try{

‎android_smart_phone/main/app/src/main/java/com/wearableintelligencesystem/androidsmartphone/WearableAiAspService.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
import com.wearableintelligencesystem.androidsmartphone.comms.MessageTypes;
2626
import com.wearableintelligencesystem.androidsmartphone.comms.SmsComms;
27+
import com.wearableintelligencesystem.androidsmartphone.contextualsearch.ContextualSearchSystem;
2728
import com.wearableintelligencesystem.androidsmartphone.database.WearableAiRoomDatabase;
2829
import com.wearableintelligencesystem.androidsmartphone.database.facialemotion.FacialEmotion;
2930
import com.wearableintelligencesystem.androidsmartphone.database.facialemotion.FacialEmotionCreator;
@@ -96,6 +97,9 @@ public class WearableAiAspService extends LifecycleService {
9697
//Object Detection system
9798
private ObjectDetectionSystem objectDetectionSystem;
9899

100+
//contextual search system
101+
private ContextualSearchSystem contextualSearchSystem;
102+
99103
//UI
100104
TextView tvIP, tvPort;
101105
TextView tvMessages;
@@ -200,6 +204,11 @@ public void run() {
200204
//start Objectdetection system
201205
objectDetectionSystem = new ObjectDetectionSystem(getApplicationContext());
202206
objectDetectionSystem.setDataObservable(dataObservable);
207+
208+
//start contextual search system
209+
contextualSearchSystem = new ContextualSearchSystem();
210+
contextualSearchSystem.setDataObservable(dataObservable);
211+
203212
//start voice command server to parse transcript for voice command
204213
voiceCommandServer = new VoiceCommandServer(dataObservable, mVoiceCommandRepository, mMemoryCacheRepository, getApplicationContext());
205214

‎android_smart_phone/main/app/src/main/java/com/wearableintelligencesystem/androidsmartphone/comms/MessageTypes.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,13 @@ public class MessageTypes {
9595
public final static String STOP_FOREIGN_LANGUAGE_ASR = "STOP_FOREIGN_LANGUAGE_ASR";
9696
public final static String START_FOREIGN_LANGUAGE_SOURCE_LANGUAGE_NAME = "START_FOREIGN_LANGUAGE_SOURCE_LANGUAGE_NAME";
9797

98+
//contextual/semantic search
99+
public final static String START_CONTEXTUAL_SEARCH = "START_CONTEXTUAL_SEARCH";
100+
public final static String STOP_CONTEXTUAL_SEARCH = "STOP_CONTEXTUAL_SEARCH";
101+
public final static String CONTEXTUAL_SEARCH_REQUEST = "CONTEXTUAL_SEARCH_REQUEST";
102+
public final static String CONTEXTUAL_SEARCH_RESULT = "CONTEXTUAL_SEARCH_RESULT";
103+
public final static String CONTEXTUAL_SEARCH_RESULT_DATA = "CONTEXTUAL_SEARCH_RESULT_DATA";
104+
98105
//object translation
99106
public final static String START_OBJECT_DETECTION = "START_OBJECT_DETECTION";
100107
public final static String STOP_OBJECT_DETECTION = "STOP_OBJECT_DETECTION";
@@ -128,6 +135,7 @@ public class MessageTypes {
128135
public final static String MODE_LIVE_LIFE_CAPTIONS = "MODE_LIVE_LIFE_CAPTIONS";
129136
public final static String MODE_SOCIAL_MODE = "MODE_SOCIAL_MODE";
130137
public final static String MODE_CONVERSATION_MODE = "MODE_CONVERSATION_MODE";
138+
public final static String MODE_CONTEXTUAL_SEARCH = "MODE_CONTEXTUAL_SEARCH";
131139
public final static String MODE_REFERENCE_GRID = "MODE_REFERENCE_GRID";
132140
public final static String MODE_WEARABLE_FACE_RECOGNIZER = "MODE_WEARABLE_FACE_RECOGNIZER";
133141
public final static String MODE_LANGUAGE_TRANSLATE = "MODE_LANGUAGE_TRANSLATE";

‎android_smart_phone/main/app/src/main/java/com/wearableintelligencesystem/androidsmartphone/comms/RestServerComms.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ public RestServerComms(Context context) {
4848
mContext = context;
4949
mRequestQueue = Volley.newRequestQueue(mContext);
5050
// serverUrl = "https://wis.emexwearables.com/api";
51-
// serverUrl = "https://wisdev.emexwearables.com/api";
52-
serverUrl = "http://192.168.11.188:5000";
51+
serverUrl = "https://wis.emexwearables.com/api";
52+
// serverUrl = "http://192.168.76.188:5000";
5353
}
5454

5555
//handles requesting data, sending data

‎android_smart_phone/main/app/src/main/java/com/wearableintelligencesystem/androidsmartphone/contextualsearch/ContextualSearchSystem.java

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package com.wearableintelligencesystem.androidsmartphone.contextualsearch;
2+
3+
import com.wearableintelligencesystem.androidsmartphone.comms.MessageTypes;
4+
5+
import org.json.JSONException;
6+
import org.json.JSONObject;
7+
import io.reactivex.rxjava3.disposables.Disposable;
8+
import io.reactivex.rxjava3.subjects.PublishSubject;
9+
10+
public class ContextualSearchSystem {
11+
private PublishSubject<JSONObject> dataObservable;
12+
private Disposable dataSubscriber;
13+
private boolean iAmActive = false;
14+
15+
public void setDataObservable(PublishSubject<JSONObject> observable){
16+
dataObservable = observable;
17+
dataSubscriber = dataObservable.subscribe(i -> handleDataStream(i));
18+
}
19+
20+
public void setActive(){
21+
iAmActive = true;
22+
}
23+
24+
public void setInactive(){
25+
iAmActive = false;
26+
}
27+
28+
private void handleDataStream(JSONObject data){
29+
try {
30+
String dataType = data.getString(MessageTypes.MESSAGE_TYPE_LOCAL);
31+
if (dataType.equals(MessageTypes.START_CONTEXTUAL_SEARCH)) {
32+
setActive();
33+
} else if (dataType.equals(MessageTypes.STOP_CONTEXTUAL_SEARCH)) {
34+
setInactive();
35+
}
36+
37+
//then, if we're active, check for things to do
38+
if (iAmActive && dataType.equals(MessageTypes.FINAL_TRANSCRIPT)) {
39+
//build and send a contextual search request to the GLBOX
40+
JSONObject contextualSearchRequest = new JSONObject();
41+
contextualSearchRequest.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.CONTEXTUAL_SEARCH_REQUEST);
42+
contextualSearchRequest.put(MessageTypes.TRANSCRIPT_TEXT, data.getString(MessageTypes.TRANSCRIPT_TEXT).toLowerCase());
43+
contextualSearchRequest.put(MessageTypes.TIMESTAMP, data.getString(MessageTypes.TIMESTAMP));
44+
contextualSearchRequest.put(MessageTypes.TRANSCRIPT_ID, data.getString(MessageTypes.TRANSCRIPT_ID));
45+
dataObservable.onNext(contextualSearchRequest);
46+
}
47+
} catch (JSONException e){
48+
e.printStackTrace();
49+
}
50+
}
51+
52+
}

‎android_smart_phone/main/app/src/main/java/com/wearableintelligencesystem/androidsmartphone/voicecommand/SwitchModesVoiceCommand.java

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ class SwitchModesVoiceCommand extends VoiceCommand {
4646
modesList.add(Pair.create("object translate", MessageTypes.MODE_OBJECT_TRANSLATE));
4747
modesList.add(Pair.create("blank", MessageTypes.MODE_BLANK));
4848
modesList.add(Pair.create("speech translate", MessageTypes.MODE_LANGUAGE_TRANSLATE));
49+
modesList.add(Pair.create("contextual search", MessageTypes.MODE_CONTEXTUAL_SEARCH));
4950

5051
}
5152

@@ -85,7 +86,7 @@ public boolean runCommand(VoiceCommandServer vcServer, String preArgs, String wa
8586

8687
//special actions ASP takes based on the mode we are entering
8788
try {
88-
JSONObject translateMessage = new JSONObject();
89+
JSONObject modeMessage = new JSONObject();
8990
if (newMode.equals(MessageTypes.MODE_LANGUAGE_TRANSLATE)){
9091
//get the language we want to translate from into base language
9192
String naturalLanguageLanguage = this.getFirstArg(postArgs.trim().substring(naturalLanguageMode.length()));
@@ -98,17 +99,22 @@ public boolean runCommand(VoiceCommandServer vcServer, String preArgs, String wa
9899
Log.d(TAG, postArgs);
99100
Log.d(TAG, naturalLanguageMode);
100101
Log.d(TAG, naturalLanguageLanguage);
101-
translateMessage.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.START_FOREIGN_LANGUAGE_ASR);
102-
translateMessage.put(MessageTypes.START_FOREIGN_LANGUAGE_SOURCE_LANGUAGE_NAME, naturalLanguageLanguage);
103-
vcServer.dataObservable.onNext(translateMessage);
102+
modeMessage.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.START_FOREIGN_LANGUAGE_ASR);
103+
modeMessage.put(MessageTypes.START_FOREIGN_LANGUAGE_SOURCE_LANGUAGE_NAME, naturalLanguageLanguage);
104+
vcServer.dataObservable.onNext(modeMessage);
104105
} else if (newMode.equals(MessageTypes.MODE_OBJECT_TRANSLATE)) {
105106
JSONObject objectMessage = new JSONObject();
106107
objectMessage.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.START_OBJECT_DETECTION);
107108
vcServer.dataObservable.onNext(objectMessage);
109+
} else if (newMode.equals(MessageTypes.MODE_CONTEXTUAL_SEARCH)) {
110+
JSONObject objectMessage = new JSONObject();
111+
objectMessage.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.START_CONTEXTUAL_SEARCH);
112+
vcServer.dataObservable.onNext(objectMessage);
108113
}else {
109-
translateMessage.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.STOP_FOREIGN_LANGUAGE_ASR);
110-
translateMessage.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.STOP_OBJECT_DETECTION);
111-
vcServer.dataObservable.onNext(translateMessage);
114+
modeMessage.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.STOP_FOREIGN_LANGUAGE_ASR);
115+
modeMessage.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.STOP_OBJECT_DETECTION);
116+
modeMessage.put(MessageTypes.MESSAGE_TYPE_LOCAL, MessageTypes.STOP_CONTEXTUAL_SEARCH);
117+
vcServer.dataObservable.onNext(modeMessage);
112118
}
113119
} catch (JSONException e){
114120
e.printStackTrace();

‎gnu_linux_box/README.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ curl -X POST -F "query=who's the president of the us" http://127.0.0.1:5000/natu
5353
1. Setup a cloud connected Linux box (tested on Ubuntu 18 LTS AWS EC2)
5454
2. Install nginx
5555
3. Clone the repo at /var/www/html
56-
4. Add the two .confi files at `gnu_linux_box/backend/deploy` to `/etc/nginx/site-available` and activate them with:
56+
4. Add the two .conf files at `gnu_linux_box/backend/deploy` to `/etc/nginx/site-available` and activate them with:
5757
```
5858
sudo rm /etc/nginx/sites-enabled/default
5959
sudo ln /etc/nginx/sites-available/wis_backend.conf /etc/nginx/sites-enabled/
@@ -63,3 +63,5 @@ sudo systemctl restart nginx
6363
5. Setup the backend to run by setting up virtualenv and installing requirements (discussed above)
6464
6. Copy the service service file to `/etc/systemd/system`
6565
7. Enable the service file with `sudo systemctl start wis_gunicorn && sudo systemctl enable wis_gunicorn`
66+
8. Use certbot to setup SSL for your domain, and point the /etc/nginx/site-enabled/*conf files to point to that SSL config file
67+
9. Restart Nginx `sudo systemctl restart nginx`

‎gnu_linux_box/backend/api/SemanticSearchApi.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from flask import render_template, make_response
2+
import time
23
from flask_restful import Resource, reqparse, fields, marshal_with
34
from db import Database
45
from datetime import datetime
@@ -50,6 +51,11 @@ def post(self): # NOTE that this is actually using username and not userId...
5051
if timestamp is None:
5152
timestamp = time.time()
5253

54+
#run word frequency on transcript
55+
low_freq_words = self.tools.find_low_freq_words(transcript)
56+
definitions = [self.tools.define_word(lfw) for lfw in low_freq_words]
57+
print("LOW FREQUENCY WORDS TO DEFINE: {}".format(definitions))
58+
5359
#run semantic search on wikipedia
5460
res = self.tools.run_semantic_wiki(transcript)
5561
resp = dict()

‎gnu_linux_box/backend/deploy/wis_backend.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,6 @@ server {
1313
proxy_buffering off;
1414
proxy_set_header Upgrade $http_upgrade;
1515
proxy_set_header Connection "Upgrade";
16-
proxy_pass http://unix:/var/www/html/wis/gnu_linux_box/backend/wis.sock;
16+
proxy_pass http://unix:/var/www/html/WearableIntelligenceSystem/gnu_linux_box/backend/wis.sock;
1717
}
1818
}

‎gnu_linux_box/backend/deploy/wis_gunicorn.service

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ After=network.target
55
[Service]
66
User=ubuntu
77
Group=www-data
8-
WorkingDirectory=/var/www/html/wis/gnu_linux_box/backend
9-
Environment=/var/www/html/wis/gnu_linux_box/venv/bin
8+
WorkingDirectory=/var/www/html/WearableIntelligenceSystem/gnu_linux_box/backend
9+
Environment=/var/www/html/WearableIntelligenceSystem/gnu_linux_box/venv/bin
1010
Environment=PYTHONUNBUFFERED=1
11-
ExecStart=/var/www/html/wis/gnu_linux_box/venv/bin/gunicorn -w 1 --bind unix:wis.sock gunicorn_deploy_flask:app
11+
ExecStart=/var/www/html/WearableIntelligenceSystem/gnu_linux_box/venv/bin/gunicorn -w 1 --bind unix:wis.sock gunicorn_deploy_flask:app
1212

1313
[Install]
1414
WantedBy=multi-user.target

‎gnu_linux_box/backend/deploy/wis_ssl.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ server {
4141

4242
server {
4343
server_name wis.fake;
44-
root /var/www/html/wis/gnu_linux_box/frontend/build/;
44+
root /var/www/html/WearableIntelligenceSystem/gnu_linux_box/frontend/build/;
4545

4646
location /api/ {
4747
proxy_pass https://127.0.0.1:1337/;

‎gnu_linux_box/backend/gunicorn_deploy_flask.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import os
66

77
logging.basicConfig(stream=sys.stderr)
8-
sys.path.insert(0, '/var/www/html/wis/gnu_linux_box/backend')
8+
sys.path.insert(0, '/var/www/html/WearableIntelligenceSystem/gnu_linux_box/backend')
99

1010
print("****************** exec is {}".format(sys.executable))
1111

‎gnu_linux_box/backend/main_tools.py

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
from txtai.pipeline import Similarity
1717
import pandas as pd
1818

19+
#word frequency + definitions
20+
from nltk.corpus import wordnet
1921

2022
#function structured into their classes and/or modules
2123
from utils.bing_visual_search import bing_visual_search
@@ -59,6 +61,19 @@ def __init__(self):
5961
dir(self.embeddings)
6062
print("Embeddings loaded.")
6163

64+
#setup word frequency + definitions
65+
self.df_word_freq = pd.read_csv("./semantic_search/english_word_freq_list/unigram_freq.csv")
66+
self.idx_dict_word_freq = self.df_word_freq.groupby(by='word').apply(lambda x: x.index.tolist()).to_dict()
67+
self.low_freq_constant = 250000 #relative to dataset used, up for debate, maybe user-settable (example english not you first language? Want to be higher)... in general, frequency can't be the only metric - maybe you say a rare word every day, we shouldn't keep defining it - cayden
68+
69+
#load text indices
70+
print("Loading word frequency index...")
71+
self.df_word_freq.iloc[self.idx_dict_word_freq["golgw"]] # run once to build the index
72+
print("--- Word frequency index loaded.")
73+
74+
print("Loading word definitions index...")
75+
syns = wordnet.synsets("dog") #run once to build index
76+
print("--- Word definitions index loaded.")
6277

6378
def semantic_wiki_filters(self, data):
6479
data_filtered = list()
@@ -96,7 +111,6 @@ def semantic_web_speech(self, text):
96111
#run named entity recognition
97112
nes = self.run_ner(text).ents
98113

99-
# for each thing (name entity) search engine (duckduckgo) to find the top result for that thing
100114
entities_results= list()
101115
for ne in nes:
102116
print("Entity found: " + ne.text)
@@ -376,10 +390,37 @@ def translate_text_simple(self, text, source_language="fr", target_language="en"
376390
return response.translations[0].translated_text
377391

378392
def translate_reference(self, text, source_language="en", target_language="fr"):
393+
translated_text = self.translate_text_simple(text, source_language=source_language, target_language=target_language)
379394
translated_text = self.translate_text_simple(text, source_language=source_language, target_language=target_language)
380395
print("Translated text is: {}".format(translated_text))
381396
res = self.get_wikipedia_data_from_page_title(translated_text, language=target_language)
382397
if res is not None:
383398
res["body"] = " ".join(res["body"].split(" ")[:self.summary_limit]) + "..."
384399
res["language"] = target_language
385400
return res
401+
402+
#we use this indexing for fast string search, thanks to: https://stackoverflow.com/questions/44058097/optimize-a-string-query-with-pandas-large-data
403+
def find_low_freq_words(self, text):
404+
low_freq_words = list()
405+
for word in text.split(' '):
406+
print(word)
407+
try:
408+
word_freq = self.df_word_freq.iloc[self.idx_dict_word_freq[word]]
409+
except KeyError as e:
410+
print(e) #word doesn't exist in frequency list
411+
continue
412+
if len(word_freq) > 0 and word_freq["count"].iloc[0] < self.low_freq_constant: #we use iloc[0] because our dataset should only have one of each word
413+
word = word_freq["word"].iloc[0]
414+
low_freq_words.append(word)
415+
return low_freq_words
416+
417+
def define_word(self, word):
418+
# lookup the word
419+
syns = wordnet.synsets(word)
420+
try:
421+
definition = syns[0].definition()
422+
except IndexError as e:
423+
print("Definition unknown for: {}".format(word))
424+
return {word : None}
425+
return {word : definition}
426+

0 commit comments

Comments
 (0)