What is the recommended way to send I2S microphone data to a remote HTTP endpoint using POST?

[that-one-arab]

Hey @pschatzmann! Amazing library!

I'm a embedded system newbie and I'm trying to make my ESP32 send and receive audio data over HTTP

Currently I am saving the data to an SD card then reading it back when sending over HTTP

I am having trouble making this work because I believe the ESP32's RAM is hitting it's limit when buffering and sending the data at once (recordings are WAV and they are about 300 - 450 KB)

Not to mention reading from SD card then sending back is inefficient

I wanted to ask if there is a recommended way to do this? I don't mind resorting to other protocols such as UDP.

My sketch is as follows:

#include "FS.h"
#include "SD.h"
#include "SPI.h"
#include "AudioTools.h"

#define SD_CS GPIO_NUM_5
#define MicPIN_WS GPIO_NUM_21
#define MicPIN_SCK GPIO_NUM_12
#define MicPIN_SD GPIO_NUM_32
#define BUTTON_PIN GPIO_NUM_4

//#define SAMPLE_RATE 22050
#define SAMPLE_RATE 16000
String RECORDING_FILE_PATH = "/input.wav";

bool recording = false;
unsigned long lastDebounceTime = 0;
unsigned long debounceDelay = 50;
static bool buttonPressed = false;
int buttonState = HIGH;


I2SStream i2s;
File audioFile;
EncodedAudioStream out(&audioFile, new WAVEncoder());
StreamCopy copier(out, i2s);

void setup(void);
void loop();
void buttonISR();
void toggleRecording();
void startRecording();
void stopRecording();

void setup(void) {
  Serial.begin(115200);

  if (!SD.begin(SD_CS)) {
    Serial.println("Card Mount Failed");
    while (1);
  }

  AudioLogger::instance().begin(Serial, AudioLogger::Info);

  auto cfg = i2s.defaultConfig(RX_MODE);
  cfg.channels = 2;
  cfg.sample_rate = SAMPLE_RATE;
  cfg.pin_ws = MicPIN_WS;
  cfg.pin_bck = MicPIN_SCK;
  cfg.pin_data = MicPIN_SD;
  i2s.begin(cfg);

  auto cfg_out = out.defaultConfig();
  cfg_out.channels = 2;
  cfg_out.sample_rate = SAMPLE_RATE;
  out.begin(cfg_out);
  copier.setCheckAvailableForWrite(false);
  pinMode(BUTTON_PIN, INPUT);

  attachInterrupt(digitalPinToInterrupt(BUTTON_PIN), buttonISR, CHANGE);
}

void loop() {
  if (buttonPressed) {
    toggleRecording();
    delay(debounceDelay);
  } else if (recording) {
    copier.copy();
  }
}

void buttonISR() {
  int reading = digitalRead(BUTTON_PIN);

  if ((millis() - lastDebounceTime) > debounceDelay) {
    if (reading != buttonState) {
      buttonState = reading;
      if (buttonState == LOW) {
        buttonPressed = !buttonPressed;
      }
    }
  }
  lastDebounceTime = millis();
}

void toggleRecording() {
  if (!recording) {
    startRecording();
  } else {
    stopRecording();
  }
  buttonPressed = false;
}

void startRecording() {
  if (recording) return;

  if (SD.remove(RECORDING_FILE_PATH)) {
    Serial.println("File deleted");
  }

  audioFile.close();
  audioFile = SD.open(RECORDING_FILE_PATH, FILE_WRITE);

  out.end();
  auto cfg_out = out.defaultConfig();
  cfg_out.channels = 2;
  cfg_out.sample_rate = 16000;
  out.begin(cfg_out);

  copier.begin(out, i2s);
  Serial.println("RECORDING");
  recording = true;
}

void stopRecording() {
  copier.end();
  audioFile.close();
  Serial.println("FILE CLOSED");
  recording = false;
}

[pschatzmann]

You don't need to store the data at all: just have a look at the examples-webserver.
If you are interested in communcation the examples are in examples-communication

[that-one-arab]

@pschatzmann Thank you for your quick answer!
I examined the examples in webserver and communication directory before but could not find one that matched my usecase and I wasn't capable enough to introduce modifications to make them work for my case.

I am currently looking at streams-i2s-webserver_wav.ino but I am assuming this webserver is server by ESP, which does not match my case since my server is a python server hosted remotely and my server only accepts HTTP connections

Looking at communication-ip-send this one looks like the closest one to what I need, but I couldn't get this example to run properly, as I get errors when I run the sketch..

I did modify the sketch, the modifed version is below

#include "AudioTools.h"
#include <WiFi.h>

#define SAMPLE_RATE 16000
#define MicPIN_WS GPIO_NUM_21
#define MicPIN_SCK GPIO_NUM_12
#define MicPIN_SD GPIO_NUM_32

AudioInfo info(16000, 1, 16);
I2SStream i2s;
WiFiClient client;
MeasuringStream clientTimed(client);
StreamCopy copier(clientTimed, i2s);

const char *ssid = "S20";
const char *password = "39891212";
const char *client_address = "192.168.1.35:8000/talk";

void connectWifi() {
  auto cfg = i2s.defaultConfig(RX_MODE);
  cfg.channels = 2;
  cfg.sample_rate = SAMPLE_RATE;
  cfg.pin_ws = MicPIN_WS;
  cfg.pin_bck = MicPIN_SCK;
  cfg.pin_data = MicPIN_SD;
  i2s.begin(cfg);

  // connect to WIFI
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(2000);
    Serial.print(".");
  }
  Serial.println();
  Serial.println(WiFi. localIP());

  // Performance Hack
  client.setNoDelay(true);
  esp_wifi_set_ps(WIFI_PS_NONE);
}

void connectIP() {
  if (!client.connected()) {
    while (!client.connect(client_address)) {
      Serial.println("trying to connect...");
      delay(5000);
    }
  }
}

void setup() {
  Serial.begin(115200);
  AudioLogger::instance().begin(Serial, AudioLogger::Info);

  connectWifi();
  Serial.println("started...");
}

void loop() {
  connectIP();  // e.g if client is shut down we try to reconnect
  copier.copy();
}

And these are the errors:

C:\Users\Muhammed\AppData\Local\Arduino15\packages\esp32\hardware\esp32\2.0.8\libraries\WiFi\src/WiFiClient.h:55:9: note: candidate: 'virtual int WiFiClient::connect(const char*, uint16_t, int32_t)'
     int connect(const char *host, uint16_t port, int32_t timeout_ms);
         ^~~~~~~
C:\Users\Muhammed\AppData\Local\Arduino15\packages\esp32\hardware\esp32\2.0.8\libraries\WiFi\src/WiFiClient.h:55:9: note:   candidate expects 3 arguments, 1 provided
Multiple libraries were found for "WiFi.h"
 Used: C:\Users\Muhammed\AppData\Local\Arduino15\packages\esp32\hardware\esp32\2.0.8\libraries\WiFi
 Not used: C:\Program Files (x86)\Arduino\libraries\WiFi
exit status 1
no matching function for call to 'WiFiClient::connect(const char*&)'

I know the error is a result of me omitting the port variable, but if I leave it there my connection hangs infinitely which makes me think that the connection string is wrong.

Thank you for your help

[pschatzmann]

Your comment does not make any sense to me: if you want to use anther server you can use it just as proxy to forward the request and you don't even need to change the webserver sketch!

[that-one-arab]

@pschatzmann Pardon me for not explaining myself properly, this is my first venture into embedded systems programming.

Basically, I have one python server that contains a single HTTP post route, this route accepts takes an audio file as a request and returns another audio file as a response.

In my use case, this means that my ESP32 device would be a client.

I tried setting it up using the HTTP client library, but I had issues sending the audio file from my ESP32 to my python server using HTTP.Post. issue mainly stemmed from due to the file being large for the ESP32 to send at once, I believe.

I would prefer if I could:

• Send audio input (microphone) to my remote server when an event happens (button click) as an HTTP request
• Wait until the HTTP request that was sent returns a response, containing a new audio file and play it.

I hope this explained what I'm trying to do better. Thank you again.

[pschatzmann]

If you stick with this design it seems that you need to learn how to do Http Post. Just google the topic.
In theory my URLStream should support this, but I never tested this properly...

[that-one-arab]

You were right! While HTTPClient library's POST method was giving me issues, but the sendRequest handled streaming the data automatically, all I needed was to pass a Stream object as a second parameter, in my case it was the SD card File object:

int httpCode = http.sendRequest("POST", &file, file.size());

Now I have a working project, but ideally I would like to get rid of the SD card dependency completely.

I tried the below code:

#include "AudioTools.h"
#include <WiFi.h>
#include <HTTPClient.h>

#define MicPIN_WS GPIO_NUM_21
#define MicPIN_SCK GPIO_NUM_12
#define MicPIN_SD GPIO_NUM_32

const char* ssid = "my_ssid";
const char* password = "my_password";
const char* url = "http://my_ip/stream";

#define SAMPLE_RATE 16000
#define STREAM_DURATION 10000 // 10 seconds in milliseconds

I2SStream i2s;
HTTPClient http;

void setup(void) {
  Serial.begin(115200);

  AudioLogger::instance().begin(Serial, AudioLogger::Info);

  // Setup Wi-Fi
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(1500);
    Serial.println("Connecting to WiFi...");
  }
  Serial.println("Connected to WiFi");

  auto cfg = i2s.defaultConfig(RX_MODE);
  cfg.channels = 2;
  cfg.sample_rate = SAMPLE_RATE;
  cfg.pin_ws = MicPIN_WS;
  cfg.pin_bck = MicPIN_SCK;
  cfg.pin_data = MicPIN_SD;
  i2s.begin(cfg);

  http.begin(url);
  http.addHeader("Content-Type", "application/octet-stream");

  unsigned long startTime = millis();
  while (millis() - startTime < STREAM_DURATION) {
    int httpCode = http.sendRequest("POST", &i2s);

    if (httpCode > 0) {
      // Get the response
      String payload = http.getString();
      Serial.println("HTTP Response code: " + String(httpCode));
      Serial.println(payload);
    } else {
      Serial.println("Error on sending POST: " + String(httpCode));
    }
  }

  http.end();
}

void loop() {
}

The i2s object is a Stream object, so I expected it to work but it didn't in my case. Am I missing something?

Thank you!

[pschatzmann]

I don't see any reason why this should not work.
I noticed that you indicated the size in the working example, but you did not in the I2S case

[that-one-arab]

@pschatzmann That is correct, I omitted the size because I figured that since we are getting live microphone input we would not know the size beforehand. Did I miss or misunderstand something?

[pschatzmann]

Not sure, I think it depends on your python server. I suggest just to calculate the theoritical size and check what happens, or change your file based test by removing the size and check if it is still working

[that-one-arab]

After doing some debugging, it appears there is an issue with not providing the 3rd file size argument to the sendRequest function
I reached that conclusion by following your advice

I opened an issue FYI.