Control Microcontrollers with Voice

[404background]

Also introduced in my article below:
In English: https://404background.com/en/programming/voice-control/
In Japanese: https://404background.com/program/voice-operate/

I tried to control a small robot using a microcontroller with voice.
I used python-venv nodes for voice input and transcription.
In addition, using the ollama node, the local LLM is judging decisions.

Here is the movie. Click and start playing.

Here is the flow.

[{"id":"90e8f4d49463a350","type":"ollama-chat","z":"3bccaf6e46d3dbf5","name":"Chat","x":250,"y":760,"wires":[["e7ac6c0c24a72f71","bbfaf3102b39c3a4"]]},{"id":"e7ac6c0c24a72f71","type":"debug","z":"3bccaf6e46d3dbf5","name":"debug 63","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto","x":420,"y":760,"wires":[]},{"id":"bbfaf3102b39c3a4","type":"change","z":"3bccaf6e46d3dbf5","name":"","rules":[{"t":"set","p":"payload","pt":"msg","to":"payload.message.content","tot":"msg"}],"action":"","property":"","from":"","to":"","reg":false,"x":420,"y":760,"wires":[["0c528e893c93fbe9","e5c06a2515200630"]]},{"id":"0c528e893c93fbe9","type":"debug","z":"3bccaf6e46d3dbf5","name":"debug 64","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":600,"y":760,"wires":[]},{"id":"426d021a3762f146","type":"venv","z":"3bccaf6e46d3dbf5","venvconfig":"5bb32f2738f10b5a","name":"","code":"import pyaudio\nimport wave\n\ndef record_audio(filename=\"recorded_audio.wav\", duration=3):\n    chunk = 1024  # Number of bytes per chunk\n    sample_format = pyaudio.paInt16  # 16-bit audio\n    channels = 1  # Mono\n    rate = 44100  # Sampling rate\n\n    p = pyaudio.PyAudio()  # Initialize the interface\n\n    # Open the stream and start recording immediately\n    stream = p.open(format=sample_format,\n                    channels=channels,\n                    rate=rate,\n                    frames_per_buffer=chunk,\n                    input=True)\n\n    frames = []  # List to store audio data\n\n    # Record for the specified duration\n    for _ in range(0, int(rate / chunk * duration)):\n        data = stream.read(chunk)\n        frames.append(data)\n\n    # Stop and close the stream\n    stream.stop_stream()\n    stream.close()\n    p.terminate()\n\n    # Save the recorded data to a file\n    with wave.open(filename, 'wb') as wf:\n        wf.setnchannels(channels)\n        wf.setsampwidth(p.get_sample_size(sample_format))\n        wf.setframerate(rate)\n        wf.writeframes(b''.join(frames))\n\n# Immediately start recording and save the audio\nrecord_audio()\n","continuous":false,"x":390,"y":420,"wires":[["9624e4572541a91c"]]},{"id":"7b4c42b178603253","type":"inject","z":"3bccaf6e46d3dbf5","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":240,"y":420,"wires":[["426d021a3762f146"]]},{"id":"a8c32402f13f0724","type":"pip","z":"3bccaf6e46d3dbf5","venvconfig":"5bb32f2738f10b5a","name":"","arg":"pyaudio","action":"list","tail":false,"x":310,"y":220,"wires":[["bcd377496bd62266","bb753ef73b34bf57"]]},{"id":"1a4a24308399d4f9","type":"inject","z":"3bccaf6e46d3dbf5","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":160,"y":220,"wires":[["a8c32402f13f0724"]]},{"id":"6db4c6573d2abe81","type":"debug","z":"3bccaf6e46d3dbf5","name":"debug 71","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":480,"y":260,"wires":[]},{"id":"9208f3a0304d90e5","type":"template","z":"3bccaf6e46d3dbf5","name":"","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"{\n    \"model\": \"gemma2:2b-instruct-fp16\",\n    \"messages\": [\n        {\n            \"role\": \"user\",\n            \"content\": \"{{payload}}\"\n        }\n    ]\n}","output":"json","x":600,"y":620,"wires":[["90e8f4d49463a350"]]},{"id":"04f7c3e871bf4209","type":"inject","z":"3bccaf6e46d3dbf5","name":"","props":[{"p":"payload"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"前に進め！","payloadType":"str","x":240,"y":580,"wires":[["fe5db020697da1bc"]]},{"id":"fe5db020697da1bc","type":"template","z":"3bccaf6e46d3dbf5","name":"","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"あなたはロボットのオペレーターです。\n音声をテキストに変換して短い命令を送っているので、同音異義語が混じっている可能性があります。\n漢字ではなく、ひらがなで判断してください。\n\n前に進むという文脈の場合はGを返してください。\n後ろに進むという文脈の場合はBを返してください。\n停止するという文脈の場合はSを返してください。\n終了するという文脈の場合もSを返してください。\n右に回転するという文脈の場合はRを返してください。\n左に回転するという文脈の場合はLを返してください。\nそれ以外の文脈ではNを返してください。\n\n停止または終了の意味合いが含まれる場合は、Sだけ返すことを優先してください。\n命令：{{payload}}","output":"str","x":440,"y":620,"wires":[["9208f3a0304d90e5"]]},{"id":"4929b442771d9bd3","type":"mqtt out","z":"3bccaf6e46d3dbf5","name":"","topic":"servo/control","qos":"","retain":"","respTopic":"","contentType":"","userProps":"","correl":"","expiry":"","broker":"f414466379b9f58d","x":450,"y":860,"wires":[]},{"id":"1cc1a4d6f98f8b95","type":"inject","z":"3bccaf6e46d3dbf5","name":"","props":[{"p":"payload"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"G","payloadType":"str","x":270,"y":860,"wires":[["4929b442771d9bd3"]]},{"id":"842b15fa75b3cbee","type":"inject","z":"3bccaf6e46d3dbf5","name":"","props":[{"p":"payload"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"S","payloadType":"str","x":270,"y":900,"wires":[["4929b442771d9bd3"]]},{"id":"da214776fa4ba1f4","type":"inject","z":"3bccaf6e46d3dbf5","name":"","props":[{"p":"payload"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"後退せよ!","payloadType":"str","x":240,"y":620,"wires":[["fe5db020697da1bc"]]},{"id":"eab8b0caade795fe","type":"inject","z":"3bccaf6e46d3dbf5","name":"","props":[{"p":"payload"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"停止せよ","payloadType":"str","x":240,"y":660,"wires":[["fe5db020697da1bc"]]},{"id":"9624e4572541a91c","type":"venv","z":"3bccaf6e46d3dbf5","venvconfig":"5bb32f2738f10b5a","name":"","code":"from faster_whisper import WhisperModel\n\nmodel_size = \"tiny\"\n\n# Run on GPU with FP16\n# model = WhisperModel(model_size, device=\"cpu\", compute_type=\"float16\")\n# or run on GPU with INT8\n# model = WhisperModel(model_size, device=\"cuda\", compute_type=\"int8_float16\")\n\n# or run on CPU with INT8\nmodel = WhisperModel(model_size, device=\"cpu\", compute_type=\"int8\")\n\nsegments, info = model.transcribe(\"recorded_audio.wav\", beam_size=5, language=\"ja\")\n\n# print(\"Detected language '%s' with probability %f\" % (info.language, info.language_probability))\n\nfor segment in segments:\n    # print(\"[%.2fs -> %.2fs] %s\" % (segment.start, segment.end, segment.text))\n    path = \"faster-whisper.txt\"\n    with open(path, mode='w', encoding='UTF-8') as f:\n        f.write(segment.text)\n    # print(segment.text)","continuous":false,"x":610,"y":420,"wires":[["8f6e80629720a2e3","cdc828a977a99f29"]]},{"id":"bcd377496bd62266","type":"pip","z":"3bccaf6e46d3dbf5","venvconfig":"5bb32f2738f10b5a","name":"","arg":"faster-whisper","action":"install","tail":false,"x":450,"y":220,"wires":[["6db4c6573d2abe81"]]},{"id":"bb753ef73b34bf57","type":"debug","z":"3bccaf6e46d3dbf5","name":"debug 80","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":300,"y":260,"wires":[]},{"id":"8f6e80629720a2e3","type":"debug","z":"3bccaf6e46d3dbf5","name":"debug 81","active":false,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":760,"y":420,"wires":[]},{"id":"cdc828a977a99f29","type":"file in","z":"3bccaf6e46d3dbf5","name":"","filename":"faster-whisper.txt","filenameType":"str","format":"utf8","chunk":false,"sendError":false,"encoding":"none","allProps":false,"x":430,"y":520,"wires":[["fe5db020697da1bc","df872dce2ebc6ac4"]]},{"id":"df872dce2ebc6ac4","type":"debug","z":"3bccaf6e46d3dbf5","name":"debug 82","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":640,"y":520,"wires":[]},{"id":"4058856e36873d5a","type":"inject","z":"3bccaf6e46d3dbf5","name":"","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"","payload":"","payloadType":"date","x":240,"y":520,"wires":[["cdc828a977a99f29"]]},{"id":"6c096f85b7750d85","type":"debug","z":"3bccaf6e46d3dbf5","name":"debug 84","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"false","statusVal":"","statusType":"auto","x":440,"y":820,"wires":[]},{"id":"e5c06a2515200630","type":"function","z":"3bccaf6e46d3dbf5","name":"function 1","func":"msg.payload = msg.payload.replace(/\\r?\\n/g,\"\")\nmsg.payload = msg.payload.replace(\" \",\"\")\n\nreturn msg;","outputs":1,"timeout":0,"noerr":0,"initialize":"","finalize":"","libs":[],"x":260,"y":820,"wires":[["4929b442771d9bd3","6c096f85b7750d85"]]},{"id":"a0a8aad03900ad58","type":"comment","z":"3bccaf6e46d3dbf5","name":"録音：pyaudio","info":"","x":420,"y":380,"wires":[]},{"id":"9f1eb0d7108010c6","type":"comment","z":"3bccaf6e46d3dbf5","name":"文字起こし：Faster Whisper","info":"","x":680,"y":380,"wires":[]},{"id":"b1b2e9e68627be1b","type":"comment","z":"3bccaf6e46d3dbf5","name":"命令文の作成","info":"","x":230,"y":480,"wires":[]},{"id":"372f8e309ea8f287","type":"comment","z":"3bccaf6e46d3dbf5","name":"動作命令の送信：Ollama","info":"","x":270,"y":720,"wires":[]},{"id":"56b7b27d48211a8c","type":"comment","z":"3bccaf6e46d3dbf5","name":"開始","info":"","x":210,"y":380,"wires":[]},{"id":"5bb32f2738f10b5a","type":"venv-config","venvname":"audio","version":"3.8"},{"id":"f414466379b9f58d","type":"mqtt-broker","name":"","broker":"localhost","port":"1883","clientid":"","autoConnect":true,"usetls":false,"protocolVersion":"4","keepalive":"60","cleansession":true,"autoUnsubscribe":true,"birthTopic":"","birthQos":"0","birthRetain":"false","birthPayload":"","birthMsg":{},"closeTopic":"","closeQos":"0","closeRetain":"false","closePayload":"","closeMsg":{},"willTopic":"","willQos":"0","willRetain":"false","willPayload":"","willMsg":{},"userProps":"","sessionExpiry":""}]