[esp32] Add WHIP example

examples/esp32s3/.gitignore

@@ -0,0 +1 @@
+../esp32/.gitignore

examples/esp32s3/CMakeLists.txt

@@ -0,0 +1,7 @@
+# The following four lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(esp32-peer)
+

examples/esp32s3/components/peer/CMakeLists.txt

@@ -0,0 +1,25 @@
+
+set(COREMQTT_CMAKE_FILE ${CMAKE_SOURCE_DIR}/../../third_party/coreMQTT/mqttFilePaths.cmake)
+set(COREHTTP_CMAKE_FILE ${CMAKE_SOURCE_DIR}/../../third_party/coreHTTP/httpFilePaths.cmake)
+set(PEER_PROJECT_PATH "../../../..")
+
+if (EXISTS ${COREHTTP_CMAKE_FILE})
+  include(${COREHTTP_CMAKE_FILE})
+endif()
+
+if (EXISTS ${COREMQTT_CMAKE_FILE})
+  include(${COREMQTT_CMAKE_FILE})
+endif()
+
+file(GLOB CODES "${PEER_PROJECT_PATH}/src/*.c")
+file(GLOB TRASNPORTS "${PEER_PROJECT_PATH}/src/transports/*.c")
+
+message({HTTP_INCLUDE_PUBLIC_DIRS} ${HTTP_INCLUDE_PUBLIC_DIRS})
+idf_component_register(
+  SRCS ${CODES} ${TRASNPORTS} ${HTTP_SOURCES} ${MQTT_SOURCES} ${MQTT_SERIALIZER_SOURCES}
+  INCLUDE_DIRS "${PEER_PROJECT_PATH}/src" "${PEER_PROJECT_PATH}/src/transports" ${HTTP_INCLUDE_PUBLIC_DIRS} ${MQTT_INCLUDE_PUBLIC_DIRS} 
+  REQUIRES mbedtls srtp json mdns
+)
+
+add_definitions("-DESP32 -DHTTP_DO_NOT_USE_CUSTOM_CONFIG -DMQTT_DO_NOT_USE_CUSTOM_CONFIG")
+

examples/esp32s3/main/CMakeLists.txt

@@ -0,0 +1,6 @@
+idf_component_register(SRCS
+  "app_main.c" "audio.c" "video.c" "wifi.c"
+  INCLUDE_DIRS "."
+)
+
+target_compile_options(${COMPONENT_LIB} PRIVATE "-Wno-format")

examples/esp32s3/main/Kconfig.projbuild

@@ -0,0 +1 @@
+../../esp32/main/Kconfig.projbuild

examples/esp32s3/main/app_main.c

@@ -0,0 +1,102 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/param.h>
+#include "esp_system.h"
+#include "esp_partition.h"
+#include "nvs_flash.h"
+#include "esp_event.h"
+#include "esp_netif.h"
+#include "esp_mac.h"
+#include "mdns.h"
+#include "esp_log.h"
+#include "esp_tls.h"
+#include "esp_ota_ops.h"
+#include "freertos/FreeRTOS.h"
+
+#include "peer.h"
+
+static const char *TAG = "webrtc";
+
+static TaskHandle_t xPcTaskHandle = NULL;
+static TaskHandle_t xAudioTaskHandle = NULL;
+static TaskHandle_t xVideoTaskHandle = NULL;
+
+extern esp_err_t audio_init();
+extern esp_err_t video_init();
+extern void audio_task(void *pvParameters);
+extern void video_task(void *pvParameters);
+extern void wifi_init_sta();
+
+PeerConnection *g_pc;
+PeerConnectionState eState = PEER_CONNECTION_CLOSED;
+
+int64_t get_timestamp() {
+
+  struct timeval tv;
+  gettimeofday(&tv, NULL);
+  return (tv.tv_sec * 1000LL + (tv.tv_usec / 1000LL));
+}
+
+static void oniceconnectionstatechange(PeerConnectionState state, void *user_data) {
+
+  ESP_LOGI(TAG, "PeerConnectionState: %d", state);
+  eState = state;
+}
+
+void peer_connection_task(void *arg) {
+
+  ESP_LOGI(TAG, "peer_connection_task started");
+
+  for(;;) {
+
+    peer_connection_loop(g_pc);
+
+    vTaskDelay(pdMS_TO_TICKS(1));
+  }
+}
+
+void app_main(void) {
+
+  PeerConfiguration config = {
+   .ice_servers = {
+    { .urls = "stun:stun.l.google.com:19302" }
+   },
+   .audio_codec = CODEC_OPUS,
+   .video_codec = CODEC_H264
+  };
+
+  ESP_LOGI(TAG, "[APP] Startup..");
+  ESP_LOGI(TAG, "[APP] Free memory: %d bytes", esp_get_free_heap_size());
+  ESP_LOGI(TAG, "[APP] IDF version: %s", esp_get_idf_version());
+
+  esp_log_level_set("*", ESP_LOG_INFO);
+  esp_log_level_set("esp-tls", ESP_LOG_VERBOSE);
+
+  ESP_ERROR_CHECK(nvs_flash_init());
+  ESP_ERROR_CHECK(esp_netif_init());
+  ESP_ERROR_CHECK(esp_event_loop_create_default());
+  ESP_ERROR_CHECK(mdns_init());
+
+  wifi_init_sta();
+
+  audio_init();
+
+  video_init();
+
+  peer_init();
+
+  g_pc = peer_connection_create(&config);
+  peer_connection_oniceconnectionstatechange(g_pc, oniceconnectionstatechange);
+  peer_signaling_join_channel(NULL, g_pc, NULL);
+
+  xTaskCreatePinnedToCore(audio_task, "audio", 20480, NULL, 5, &xAudioTaskHandle, 0);
+
+  xTaskCreatePinnedToCore(video_task, "video", 10240, NULL, 6, &xVideoTaskHandle, 0);
+
+  xTaskCreatePinnedToCore(peer_connection_task, "peer_connection", 8192, NULL, 10, &xPcTaskHandle, 1);
+
+  ESP_LOGI(TAG, "[APP] Free memory: %d bytes", esp_get_free_heap_size());
+}

examples/esp32s3/main/audio.c

@@ -0,0 +1,155 @@
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "esp_log.h"
+#include "driver/i2s_pdm.h"
+
+#include "esp_audio_enc.h"
+#include "esp_audio_enc_def.h"
+#include "esp_audio_def.h"
+#include "esp_opus_enc.h"
+
+#include "peer_connection.h"
+
+#define I2S_CLK_GPIO 42
+#define I2S_DATA_GPIO 41
+
+static const char *TAG = "AUDIO";
+
+extern PeerConnection *g_pc;
+extern PeerConnectionState eState;
+extern int get_timestamp();
+
+i2s_chan_handle_t rx_handle = NULL;
+
+esp_audio_enc_handle_t enc_handle = NULL;
+esp_audio_enc_in_frame_t aenc_in_frame = { 0 };
+esp_audio_enc_out_frame_t aenc_out_frame = { 0 };
+
+esp_err_t audio_codec_init() {
+
+  uint8_t *inbuf = NULL;
+  uint8_t *outbuf = NULL;
+  int aenc_in_frame_size = 0;
+  int aenc_out_frame_size = 0;
+
+  esp_audio_err_t ret = ESP_AUDIO_ERR_OK;
+
+  esp_opus_enc_config_t config = ESP_OPUS_ENC_CONFIG_DEFAULT();
+  config.sample_rate = ESP_AUDIO_SAMPLE_RATE_8K;
+  config.channel = ESP_AUDIO_MONO;
+  config.bitrate = 18000;
+  config.frame_duration = ESP_OPUS_ENC_FRAME_DURATION_40_MS;
+
+  ret = esp_opus_enc_open(&config, sizeof(esp_opus_enc_config_t), &enc_handle);
+  if (ret != ESP_AUDIO_ERR_OK) {
+    ESP_LOGE(TAG, "audio encoder open failed");
+    return ESP_FAIL;
+  }
+
+  ret = esp_opus_enc_get_frame_size(enc_handle, &aenc_in_frame_size, &aenc_out_frame_size);
+  if (ret != ESP_AUDIO_ERR_OK) {
+    ESP_LOGE(TAG, "audio encoder get frame size failed");
+    return ESP_FAIL;
+  }
+
+  inbuf = calloc(1, aenc_in_frame_size*2);
+  if (!inbuf) {
+    ESP_LOGE(TAG, "inbuf malloc failed");
+    return ESP_FAIL;
+  }
+
+  outbuf = calloc(1, aenc_out_frame_size);
+  if (!outbuf) {
+    ESP_LOGE(TAG, "outbuf malloc failed");
+    return ESP_FAIL;
+  }
+
+  aenc_in_frame.buffer = inbuf;
+  aenc_in_frame.len = aenc_in_frame_size;
+  aenc_out_frame.buffer = outbuf;
+  aenc_out_frame.len = aenc_out_frame_size;
+  ESP_LOGI(TAG, "audio codec init done. in_frame_size=%d, out_frame_size=%d", aenc_in_frame_size, aenc_out_frame_size);
+  return 0;
+}
+
+esp_err_t audio_init(void) {
+
+  i2s_chan_config_t chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(I2S_NUM_AUTO, I2S_ROLE_MASTER);
+  ESP_ERROR_CHECK(i2s_new_channel(&chan_cfg, NULL, &rx_handle));
+
+  i2s_pdm_rx_config_t pdm_rx_cfg = {
+   .clk_cfg = I2S_PDM_RX_CLK_DEFAULT_CONFIG(8000),
+   .slot_cfg = I2S_PDM_RX_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO),
+   .gpio_cfg = {
+    .clk = I2S_CLK_GPIO,
+    .din = I2S_DATA_GPIO,
+    .invert_flags = {
+     .clk_inv = false,
+    },
+   },
+  };
+
+  ESP_ERROR_CHECK(i2s_channel_init_pdm_rx_mode(rx_handle, &pdm_rx_cfg));
+  ESP_ERROR_CHECK(i2s_channel_enable(rx_handle));
+
+  return audio_codec_init();
+}
+
+void audio_deinit(void) {
+
+  ESP_ERROR_CHECK(i2s_channel_disable(rx_handle));
+  ESP_ERROR_CHECK(i2s_del_channel(rx_handle));
+}
+
+int32_t audio_get_samples(uint8_t *buf, size_t size) {
+
+  size_t bytes_read;
+
+  if (i2s_channel_read(rx_handle, (char *)buf, size, &bytes_read, 1000) != ESP_OK) {
+    ESP_LOGE(TAG, "i2s read error");
+  }
+
+  return bytes_read;
+}
+
+void audio_task(void *arg) {
+
+  int ret;
+  static int64_t last_time;
+  int64_t curr_time;
+  float bytes = 0;
+
+  last_time = get_timestamp();
+  ESP_LOGI(TAG, "audio task started");
+
+  for (;;) {
+
+    if (eState == PEER_CONNECTION_COMPLETED) {
+
+      ret = audio_get_samples(aenc_in_frame.buffer, aenc_in_frame.len);
+
+      if (ret == aenc_in_frame.len) {
+
+        if (esp_opus_enc_process(enc_handle, &aenc_in_frame, &aenc_out_frame) == ESP_AUDIO_ERR_OK) {
+
+          peer_connection_send_audio(g_pc, aenc_out_frame.buffer, aenc_out_frame.encoded_bytes);
+
+          bytes += aenc_out_frame.encoded_bytes;
+          if (bytes > 50000) {
+
+            curr_time = get_timestamp();
+            ESP_LOGI(TAG, "audio bitrate: %.1f bps", 1000.0 * (bytes * 8.0 / (float)(curr_time - last_time)));
+            last_time = curr_time;
+            bytes = 0;
+          }
+        }
+      }
+      vTaskDelay(pdMS_TO_TICKS(5));
+
+    } else {
+
+      vTaskDelay(pdMS_TO_TICKS(100));
+    }
+  }
+}
+

examples/esp32s3/main/idf_component.yml

@@ -0,0 +1,20 @@
+## IDF Component Manager Manifest File
+dependencies:
+  espressif/esp_audio_codec: "^1.0.1"
+  espressif/esp_h264: "^0.1.1"
+  espressif/mdns: "*"
+  espressif/esp32-camera: "^2.0.4"
+  ## Required IDF version
+  idf:
+    version: ">=4.1.0"
+  # # Put list of dependencies here
+  # # For components maintained by Espressif:
+  # component: "~1.0.0"
+  # # For 3rd party components:
+  # username/component: ">=1.0.0,<2.0.0"
+  # username2/component2:
+  #   version: "~1.0.0"
+  #   # For transient dependencies `public` flag can be set.
+  #   # `public` flag doesn't have an effect dependencies of the `main` component.
+  #   # All dependencies of `main` are public by default.
+  #   public: true