Skip to content

Commit 938d952

Browse files
previprevi
committed
audio hear
1 parent 73570b5 commit 938d952

File tree

11 files changed

+123
-32
lines changed

11 files changed

+123
-32
lines changed

‎audio.py

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
import pyaudio
2828
import wave
29+
import audioop
2930
import logging
3031

3132
try:
@@ -51,6 +52,21 @@ def get_instance(cls):
5152
cls._instance = Audio()
5253
return cls._instance
5354

55+
def __init__(self):
56+
self.pyaudio = pyaudio.PyAudio()
57+
try:
58+
self.stream_in = self.pyaudio.open(format=FORMAT, channels=1, input_device_index=0, rate=RATE,
59+
input=True,
60+
frames_per_buffer=CHUNK_SIZE)
61+
self.stream_in.start_stream()
62+
except:
63+
logging.info("Audio: input stream not available")
64+
65+
def exit(self):
66+
# cleanup stuff.
67+
self.stream_in.close()
68+
self.pyaudio.terminate()
69+
5470
def say(self, what, locale='en'):
5571
if what and "$" in what:
5672
os.system ('omxplayer sounds/' + what[1:])
@@ -68,11 +84,6 @@ def normalize(self, snd_data):
6884
return r
6985

7086
def record(self, elapse):
71-
p = pyaudio.PyAudio()
72-
stream = p.open(format=FORMAT, channels=1, input_device_index=0, rate=RATE,
73-
input=True,
74-
frames_per_buffer=CHUNK_SIZE)
75-
7687
num_silent = 0
7788
snd_started = False
7889
c = 0
@@ -82,15 +93,12 @@ def record(self, elapse):
8293
while (c * 2.0 * 8192 / RATE) < elapse:
8394
c += 1
8495
# little endian, signed short
85-
snd_data = array('h', stream.read(CHUNK_SIZE))
96+
snd_data = array('h', self.stream_in.read(CHUNK_SIZE))
8697
if byteorder == 'big':
8798
snd_data.byteswap()
8899
r.extend(snd_data)
89100

90101
sample_width = p.get_sample_size(FORMAT)
91-
stream.stop_stream()
92-
stream.close()
93-
p.terminate()
94102

95103
r = self.normalize(r)
96104

@@ -111,11 +119,8 @@ def play(self, filename):
111119
# open the file for reading.
112120
wf = wave.open(SOUNDDIR + filename, 'rb')
113121

114-
# create an audio object
115-
p = pyaudio.PyAudio()
116-
117122
# open stream based on the wave object which has been input.
118-
stream = p.open(format =
123+
stream = self.pyaudio.open(format =
119124
p.get_format_from_width(wf.getsampwidth()),
120125
channels = wf.getnchannels(),
121126
rate = wf.getframerate(),
@@ -132,7 +137,32 @@ def play(self, filename):
132137

133138
# cleanup stuff.
134139
stream.close()
135-
p.terminate()
140+
141+
def hear(self, level, elapse=1.0):
142+
sig_hear = False
143+
ts_total = time.time()
144+
ts_signal = None
145+
146+
while time.time() - ts_total < elapse:
147+
try:
148+
snd_data = self.stream_in.read(CHUNK_SIZE)
149+
snd_rms = audioop.rms(snd_data, 2)
150+
logging.info("snd.rms: " + str(snd_rms))
151+
if snd_rms > level:
152+
sig_hear = True
153+
break
154+
155+
except IOError as ex:
156+
if ex[1] != pyaudio.paInputOverflowed:
157+
raise
158+
buf = '\x00' * CHUNK_SIZE #white noise
159+
logging.info("white noise")
160+
except AttributeError:
161+
pass
162+
163+
164+
return sig_hear
165+
136166

137167
def speech_recog(self, model):
138168

@@ -147,20 +177,13 @@ def speech_recog(self, model):
147177
config.set_string('-dict', MODELDIR + model + '.dict')
148178
decoder = Decoder(config)
149179

150-
p = pyaudio.PyAudio()
151-
logging.info("device info: " + str(p.get_device_info_by_index(0)))
152-
#stream = p.open(format=pyaudio.paInt16, channels=1, input_device_index=0, rate=16000, input=True, frames_per_buffer=1024)
153-
stream = p.open(format=FORMAT, channels=1, input_device_index=0, rate=RATE,
154-
input=True,
155-
frames_per_buffer=CHUNK_SIZE*256)
156-
stream.start_stream()
157180
decoder.start_utt()
158181
tstamp = time.time()
159182
recog_text = ''
160183

161184
while len(recog_text) < 1:
162185
try:
163-
buf = stream.read(CHUNK_SIZE)
186+
buf = self.stream_in.read(CHUNK_SIZE)
164187
logging.info("actual voice")
165188
decoder.process_raw(buf, False, False)
166189
if decoder.hyp().hypstr != '':
@@ -176,8 +199,6 @@ def speech_recog(self, model):
176199
pass
177200

178201
decoder.end_utt()
179-
stream.close()
180-
p.terminate()
181202

182203
logging.info("recog text: " + recog_text)
183204
return recog_text

‎coderbot.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"move_tr_speed": "80", "move_fw_elapse": "1", "camera_path_object_size_min": "4000", "load_at_start": "", "move_tr_elapse": "0.5", "sound_stop": "$shutdown.mp3", "camera_color_object_size_min": "4000", "prog_level": "adv", "prog_scrollbars": "true", "move_fw_speed": "100", "camera_color_object_size_max": "160000", "sound_shutter": "$shutter.mp3", "show_page_prefs": "true", "cv_image_factor": "2", "ctrl_hud_image": "", "button_func": "none", "ctrl_fw_elapse": "-1", "ctrl_tr_elapse": "-1", "move_power_angle_2": "20", "move_power_angle_3": "20", "move_power_angle_1": "15", "move_motor_trim": "0.9", "show_page_program": "true", "sound_start": "$startup.mp3", "camera_exposure_mode": "auto", "ctrl_tr_speed": "80", "ctrl_fw_speed": "100", "camera_refresh_timeout": "0.1", "camera_jpeg_quality": "20", "prog_maxblocks": "-1", "move_motor_mode": "dc", "camera_path_object_size_max": "160000", "show_page_control": "true"}
1+
{"move_tr_speed": "80", "move_fw_elapse": "1", "camera_path_object_size_min": "4000", "load_at_start": "", "move_tr_elapse": "0.5", "sound_stop": "$shutdown.mp3", "camera_color_object_size_min": "4000", "prog_level": "adv", "prog_scrollbars": "true", "move_fw_speed": "100", "camera_color_object_size_max": "160000", "sound_shutter": "$shutter.mp3", "show_page_prefs": "true", "cv_image_factor": "1", "ctrl_hud_image": "", "button_func": "none", "ctrl_fw_elapse": "-1", "ctrl_tr_elapse": "-1", "move_power_angle_2": "20", "move_power_angle_3": "20", "move_power_angle_1": "15", "move_motor_trim": "0.9", "show_page_program": "true", "sound_start": "$startup.mp3", "camera_exposure_mode": "auto", "ctrl_tr_speed": "80", "ctrl_fw_speed": "100", "camera_refresh_timeout": "0.1", "camera_jpeg_quality": "20", "prog_maxblocks": "-1", "move_motor_mode": "dc", "camera_path_object_size_max": "160000", "show_page_control": "true"}

‎data/program_find_text.data

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"dom_code": "<xml xmlns=\"http://www.w3.org/1999/xhtml\"><block type=\"controls_whileUntil\" x=\"7\" y=\"60\"><field name=\"MODE\">WHILE</field><value name=\"BOOL\"><block type=\"logic_boolean\"><field name=\"BOOL\">TRUE</field></block></value><statement name=\"DO\"><block type=\"text_print\"><value name=\"TEXT\"><block type=\"coderbot_adv_findText\"><field name=\"ACCEPT\">alpha</field><value name=\"COLOR\"><block type=\"text\"><field name=\"TEXT\">#00f05d</field></block></value></block></value></block></statement></block></xml>", "code": "while True:\n get_prog_eng().check_end()\n get_cam().set_text(get_cam().find_text(accept=\"alpha\", back_color='#00f05d'))\n", "name": "find_text"}
1+
{"dom_code": "<xml xmlns=\"http://www.w3.org/1999/xhtml\"><block type=\"controls_whileUntil\" x=\"7\" y=\"60\"><field name=\"MODE\">WHILE</field><value name=\"BOOL\"><block type=\"logic_boolean\"><field name=\"BOOL\">TRUE</field></block></value><statement name=\"DO\"><block type=\"text_print\"><value name=\"TEXT\"><block type=\"coderbot_adv_findText\"><field name=\"ACCEPT\">alpha</field><value name=\"COLOR\"><block type=\"text\"><field name=\"TEXT\">#20554b</field></block></value></block></value></block></statement></block></xml>", "code": "while True:\n get_prog_eng().check_end()\n get_cam().set_text(get_cam().find_text(accept=\"alpha\", back_color='#20554b'))\n", "name": "find_text"}

‎data/program_hear_test.data

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"dom_code": "<xml xmlns=\"http://www.w3.org/1999/xhtml\"><block type=\"controls_whileUntil\" x=\"-68\" y=\"108\"><field name=\"MODE\">WHILE</field><value name=\"BOOL\"><block type=\"logic_boolean\"><field name=\"BOOL\">TRUE</field></block></value><statement name=\"DO\"><block type=\"text_print\"><value name=\"TEXT\"><block type=\"coderbot_audio_hear\"><value name=\"LEVEL\"><block type=\"math_number\"><field name=\"NUM\">100</field></block></value><value name=\"ELAPSE\"><block type=\"math_number\"><field name=\"NUM\">1.0</field></block></value></block></value></block></statement></block></xml>", "code": "while True:\n get_prog_eng().check_end()\n get_cam().set_text(get_audio().hear(level=100, elapse=1))\n", "name": "hear_test"}

‎main.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,12 @@
4949
app = Flask(__name__,static_url_path="")
5050
#app.config.from_pyfile('coderbot.cfg')
5151
babel = Babel(app)
52-
app.debug = True
52+
app.debug = False
5353
#sockets = Sockets(app)
5454

5555
app.prog_engine = ProgramEngine.get_instance()
5656
app.prog = None
57+
app.shutdown_requested = False
5758

5859
@babel.localeselector
5960
def get_locale():
@@ -139,7 +140,7 @@ def handle_bot_status():
139140

140141
def video_stream(cam):
141142
refresh_timeout = float(app.bot_config.get("camera_refresh_timeout", "0.1"))
142-
while True:
143+
while not app.shutdown_requested:
143144
last_refresh_time = time.time()
144145
frame = cam.get_image_jpeg()
145146
yield ("--BOUNDARYSTRING\r\n" +
@@ -289,3 +290,4 @@ def run_server():
289290
cam.exit()
290291
if bot:
291292
bot.exit()
293+
app.shutdown_requested = True

‎static/js/blockly/blocks.js

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -704,6 +704,34 @@ Blockly.Python['coderbot_audio_play'] = function(block) {
704704
return 'get_audio().play(' + filename + ')\n';
705705
};
706706

707+
Blockly.Blocks['coderbot_audio_hear'] = {
708+
/**
709+
* Block for audio hear function.
710+
* @this Blockly.Block
711+
*/
712+
init: function() {
713+
this.setHelpUrl(Blockly.Msg.LOGIC_BOOLEAN_HELPURL);
714+
this.setColour(290);
715+
this.appendValueInput('LEVEL')
716+
.setCheck(["Number"])
717+
.appendField(Blockly.Msg.CODERBOT_AUDIO_HEAR + Blockly.Msg.CODERBOT_AUDIO_HEAR_LEVEL);
718+
this.appendValueInput('ELAPSE')
719+
.setCheck(["Number"])
720+
.appendField(Blockly.Msg.CODERBOT_AUDIO_HEAR_ELAPSE);
721+
this.setInputsInline(true);
722+
this.setOutput(true, ['Number']);
723+
this.setTooltip(Blockly.Msg.LOGIC_BOOLEAN_TOOLTIP);
724+
}
725+
};
726+
727+
Blockly.Python['coderbot_audio_hear'] = function(block) {
728+
// Boolean values true and false.
729+
var level = Blockly.Python.valueToCode(block, 'LEVEL', Blockly.Python.ORDER_NONE) || '\'\'';
730+
var elapse = Blockly.Python.valueToCode(block, 'ELAPSE', Blockly.Python.ORDER_NONE) || '\'\'';
731+
var code = 'get_audio().hear(level=' + level + ', elapse=' + elapse + ')';
732+
return [code, Blockly.Python.ORDER_ATOMIC];
733+
};
734+
707735
Blockly.Blocks['coderbot_audio_listen'] = {
708736
/**
709737
* Block for findText function.

‎static/js/blockly/bot_en.js

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@ Blockly.Msg.CODERBOT_SENSOR_FINDCODE = "read Bar-QRCode";
5555
Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_NAME = "record as file";
5656
Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_ELAPSE = " of seconds";
5757
Blockly.Msg.CODERBOT_AUDIO_PLAY_FILE = "play file";
58+
Blockly.Msg.CODERBOT_AUDIO_HEAR = "hear sound";
59+
Blockly.Msg.CODERBOT_AUDIO_HEAR_LEVEL = " of level";
60+
Blockly.Msg.CODERBOT_AUDIO_HEAR_ELAPSE = "for up to seconds";
5861
Blockly.Msg.CODERBOT_AUDIO_LISTEN = "listen";
5962
Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_SIMPLE = "simple commands";
6063
Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_MEDIUM = "medium commands";
@@ -63,4 +66,3 @@ Blockly.Msg.CODERBOT_SONAR_GET_DISTANCE = "get distance with";
6366
Blockly.Msg.CODERBOT_SONAR_SENSOR_1 = "sonar 1";
6467
Blockly.Msg.CODERBOT_SONAR_SENSOR_2 = "sonar 2";
6568
Blockly.Msg.CODERBOT_SONAR_SENSOR_3 = "sonar 3";
66-

‎static/js/blockly/bot_fr.js

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,24 @@ Blockly.Msg.CODERBOT_SENSOR_FINDFACE_Y = "ordonnée";
4545
Blockly.Msg.CODERBOT_SENSOR_FINDFACE_SIZE = "taille";
4646
Blockly.Msg.CODERBOT_SENSOR_FINDFACE_ALL = "x, y, taille (sous forme de liste)";
4747
Blockly.Msg.CODERBOT_SENSOR_FINDLOGO = "trouve le logo";
48+
Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_FIND = "find text of kind";
49+
Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_ACCEPT_ALPHA = "Alpha (A..Z)";
50+
Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_ACCEPT_NUM = "Numeric (0..9)";
51+
Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_ACCEPT_ALPHANUM = "Alphanumeric (A..Z;0..9)";
52+
Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_ACCEPT_UNSPEC = "Any";
53+
Blockly.Msg.CODERBOT_SENSOR_FINDTEXT_COLOR = "background color";
4854
Blockly.Msg.CODERBOT_SENSOR_FINDCODE = "trouve le Bar/QRCode";
55+
Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_NAME = "record as file";
56+
Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_ELAPSE = " of seconds";
57+
Blockly.Msg.CODERBOT_AUDIO_PLAY_FILE = "play file";
58+
Blockly.Msg.CODERBOT_AUDIO_HEAR = "hear sound";
59+
Blockly.Msg.CODERBOT_AUDIO_HEAR_LEVEL = " of level";
60+
Blockly.Msg.CODERBOT_AUDIO_HEAR_ELAPSE = "for up to seconds";
61+
Blockly.Msg.CODERBOT_AUDIO_LISTEN = "listen";
62+
Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_SIMPLE = "simple commands";
63+
Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_MEDIUM = "medium commands";
64+
Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_ADV = "advance commands";
65+
Blockly.Msg.CODERBOT_SONAR_GET_DISTANCE = "get distance with";
66+
Blockly.Msg.CODERBOT_SONAR_SENSOR_1 = "sonar 1";
67+
Blockly.Msg.CODERBOT_SONAR_SENSOR_2 = "sonar 2";
68+
Blockly.Msg.CODERBOT_SONAR_SENSOR_3 = "sonar 3";

‎static/js/blockly/bot_it.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@ Blockly.Msg.CODERBOT_SENSOR_FINDCODE = "leggi Bar/QRCode";
5555
Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_NAME = "registra audio su file";
5656
Blockly.Msg.CODERBOT_AUDIO_RECORD_FILE_ELAPSE = " per secondi";
5757
Blockly.Msg.CODERBOT_AUDIO_PLAY_FILE = "riproduci file";
58+
Blockly.Msg.CODERBOT_AUDIO_HEAR = "senti suono";
59+
Blockly.Msg.CODERBOT_AUDIO_HEAR_LEVEL = " di volume";
60+
Blockly.Msg.CODERBOT_AUDIO_HEAR_ELAPSE = "per secondi";
5861
Blockly.Msg.CODERBOT_AUDIO_LISTEN = "ascolta";
5962
Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_SIMPLE = "commandi semplici";
6063
Blockly.Msg.CODERBOT_AUDIO_LISTEN_MODEL_MEDIUM = "commandi medi";

‎templates/blocks_adv.xml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,18 @@
255255
<block type="coderbot_audio_say"></block>
256256
<block type="coderbot_audio_record"></block>
257257
<block type="coderbot_audio_play"></block>
258+
<block type="coderbot_audio_hear">
259+
<value name="LEVEL">
260+
<block type="math_number">
261+
<field name="NUM">100</field>
262+
</block>
263+
</value>
264+
<value name="ELAPSE">
265+
<block type="math_number">
266+
<field name="NUM">1.0</field>
267+
</block>
268+
</value>
269+
</block>
258270
<block type="coderbot_audio_listen"></block>
259271
</category>
260272
</xml>

0 commit comments

Comments
 (0)