Is it possible to output LOGS or debug messages to a SPIFFS file that’s inside the ESP32?

[GeorgeFlorian]

Hardware:

Board: ESP32 DEVKIT1
Core Installation version: 1.0.1-git this one
IDE name: PlatformIO IDE
Flash Frequency: 80Mhz
PSRAM enabled: no
Upload Speed: 921600
Computer OS: Linux Mint 19.1 Mate

Let's say that the ESP is installed at a location and after some time in use it crashes. I would like to access some logs to see what happened and to be able to debug it.

I am trying to output debug messages to a SPIFFS file so that I can then access it via a HTML page or by just following the address: http://myIP/events

This is my sketch:

#include <Arduino.h>
#include <WiFi.h>
#include <HTTPClient.h>
#include <ESPAsyncWebServer.h>
#include <stdarg.h>
#include <stdio.h>
#include <SPIFFS.h>

const char* ssid = "SSID";
const char* password =  "password";

AsyncWebServer server(80);

static char log_print_buffer[512];

int vprintf_into_spiffs(const char* szFormat, va_list args) {
	//write evaluated format string into buffer
	int ret = vsnprintf (log_print_buffer, sizeof(log_print_buffer), szFormat, args);

	//output is now in buffer. write to file.
	if(ret >= 0) {
    if(!SPIFFS.exists("/LOGS.txt")) {
      File writeLog = SPIFFS.open("/LOGS.txt", FILE_WRITE);
      if(!writeLog) Serial.println("Couldn't open spiffs_log.txt"); 
      delay(50);
      writeLog.close();
    }
   
		File spiffsLogFile = SPIFFS.open("/LOGS.txt", FILE_APPEND);
		//debug output
		//printf("[Writing to SPIFFS] %.*s", ret, log_print_buffer);
		spiffsLogFile.write((uint8_t*) log_print_buffer, (size_t) ret);
		//to be safe in case of crashes: flush the output
		spiffsLogFile.flush();
		spiffsLogFile.close();
	}
	return ret;
}

void setup() {
 
  Serial.begin(115200);
  delay(1000);
  if (!SPIFFS.begin(true)) {
    Serial.println("An Error has occurred while mounting SPIFFS");
    return;
  }
  esp_log_set_vprintf(&vprintf_into_spiffs);
  //install new logging function
  //trigger some text
  esp_log_level_set("TAG", ESP_LOG_DEBUG);
  //write into log
  esp_log_write(ESP_LOG_DEBUG, "TAG", "text!\n");

  delay(1000);
  WiFi.begin(ssid, password);
  delay(1000);
 
  while (WiFi.status() != WL_CONNECTED) {
    delay(1000);
    Serial.println("Connecting to WiFi..");
  }
 
  Serial.println("Connected to the WiFi network");
  Serial.println(WiFi.localIP());

  server.on("/events", HTTP_GET, [](AsyncWebServerRequest *request){
    if(SPIFFS.exists("/LOGS.txt")) {
      request->send(SPIFFS, "/LOGS.txt", "text/plain");
    } else {
      request->send(200, "text/plain", "LOGS not found ! Restarting in 5 seconds..");
      delay(5000);
      ESP.restart();
    }
    
    });

  server.begin();

  File file2 = SPIFFS.open("/LOGS.txt");

  if(!file2){
      Serial.println("Failed to open file for reading");
      return;
  }

  Serial.println("File Content:");

  while(file2.available()){

      Serial.write(file2.read());
  }

  file2.close();

  Serial.println("End of file content");
}
 
void loop() {
  if ((WiFi.status() == WL_CONNECTED)) { //Check the current connection status
 
    HTTPClient http;
 
    http.begin("http://jsonplaceholder.typicode.com/comments?id=10"); //Specify the URL
    int httpCode = http.GET();                                        //Make the request
 
    if (httpCode > 0) { //Check for the returning code
        Serial.println();
        Serial.printf("[HTTP] GET... code: %d\n", httpCode);
        if(httpCode == HTTP_CODE_OK) {
            String payload = http.getString();
            Serial.println(payload);
            Serial.println();
          }
      } else {
          Serial.printf("[HTTP] GET... failed, error: %s\n", http.errorToString(httpCode).c_str());
        }
 
    http.end(); //Free the resources
  }
 
  delay(10000);
 
}

And it outputs:

Connected to the WiFi network
*myIP*
File Content:
text!
text!
End of file content

So it outputs text! .

From what I've been told:

Which goes to the base problem: The Arduino-ESP32 implementation does not use the esp_log_write() functions, but it’s own logging system where the log output function cannot be altered!

Looking to esp32-hal-log.h

int log_printf(const char *fmt, ...);

#define ARDUHAL_SHORT_LOG_FORMAT(letter, format)  ARDUHAL_LOG_COLOR_ ## letter format ARDUHAL_LOG_RESET_COLOR "\r\n"
#define ARDUHAL_LOG_FORMAT(letter, format)  ARDUHAL_LOG_COLOR_ ## letter "[" #letter "][%s:%u] %s(): " format ARDUHAL_LOG_RESET_COLOR "\r\n", pathToFileName(__FILE__), __LINE__, __FUNCTION__

#if ARDUHAL_LOG_LEVEL >= ARDUHAL_LOG_LEVEL_VERBOSE
#define log_v(format, ...) log_printf(ARDUHAL_LOG_FORMAT(V, format), ##__VA_ARGS__)
#define isr_log_v(format, ...) ets_printf(ARDUHAL_LOG_FORMAT(V, format), ##__VA_ARGS__)
#else
#define log_v(format, ...)
#define isr_log_v(format, ...)
#endif

So depending on the log level, the framework will set log_v (and related functions) to log_printf() for normal executions and ets_printf() for when inside an ISR. And log_printf() is implemented statically in esp32-hal-uart.c

int log_printf(const char *format, ...)
{
    if(s_uart_debug_nr < 0){
        return 0;
    }
    static char loc_buf[64];
    char * temp = loc_buf;
    int len;
    va_list arg;
    va_list copy;
    va_start(arg, format);
    va_copy(copy, arg);
    len = vsnprintf(NULL, 0, format, arg);
    va_end(copy);
    if(len >= sizeof(loc_buf)){
        temp = (char*)malloc(len+1);
        if(temp == NULL) {
            return 0;
        }
    }
    vsnprintf(temp, len+1, format, arg);
#if !CONFIG_DISABLE_HAL_LOCKS
    if(_uart_bus_array[s_uart_debug_nr].lock){
        xSemaphoreTake(_uart_bus_array[s_uart_debug_nr].lock, portMAX_DELAY);
        ets_printf("%s", temp);
        xSemaphoreGive(_uart_bus_array[s_uart_debug_nr].lock);
    } else {
        ets_printf("%s", temp);
    }
#else
    ets_printf("%s", temp);
#endif
    va_end(arg);
    if(len >= sizeof(loc_buf)){
        free(temp);
    }
    return len;
}

Since this uses a static ets_printf() from the library we can’t change that. So, we can’t redirect the the debug output like [D][WiFiGeneric.cpp:342] _eventCallback(): Event: 0 - WIFI_READY but only our own calls with esp_log_write().

So is there any fix ? Or it just won't happen ?

And if this method doesn't work, how can I output debug messages or logs so that I can access them without a Serial Monitor ?

Thank you !

[asetyde]

i quote, also I need to redirect to my net serial logs

[chinswain]

i quote, also I need to redirect to my net serial logs

Same for me

[stale]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

[DennisMoschina]

Were there any updates about this topic in the last two years? I really want to make my esp32 completely wireless and having the opportunity to just post the logs to a Webserver would be fantastic

[GeorgeFlorian]

Were there any updates about this topic in the last two years? I really want to make my esp32 completely wireless and having the opportunity to just post the logs to a Webserver would be fantastic

I couldn't find any SYSTEM logs that I could output. The verbose debug messages will be outputted on the terminal like always, but I've made myself some custom messages throughout the code which I output like PLACEHOLDERS in a web-page using template processing from me-no-dev's ESPAsyncWebServer.

First I made a global variable in which I can store multiple messages:

String strlog;

Then you use a circular buffer to constantly add messages to strlog:

//------------------------- struct circular_buffer
struct ring_buffer
{
  ring_buffer(size_t cap) : buffer(cap) {}

  bool empty() const { return sz == 0; }
  bool full() const { return sz == buffer.size(); }

  void push(String str)
  {
    if (last >= buffer.size())
      last = 0;
    buffer[last] = str;
    ++last;
    if (full())
      first = (first + 1) % buffer.size();
    else
      ++sz;
  }
  void print() const
  {
    strlog = "";
    if (first < last)
      for (size_t i = first; i < last; ++i)
      {
        strlog += (buffer[i] + "<br>");
      }
    else
    {
      for (size_t i = first; i < buffer.size(); ++i)
      {
        strlog += (buffer[i] + "<br>");
      }
      for (size_t i = 0; i < last; ++i)
      {
        strlog += (buffer[i] + "<br>");
      }
    }
  }

private:
  std::vector<String> buffer;
  size_t first = 0;
  size_t last = 0;
  size_t sz = 0;
};

After that you make a function that pushes the message in the buffer and also outputs it in the terminal:

//------------------------- logOutput(String)
void logOutput(String string1)
{
  circle.push(string1);
  Serial.println(string1);
}

Further, you tell the template processor to replace the placeholder with your strlog:

//------------------------- template processor()
String processor(const String &var)
{
  circle.print(); // loads strlog with messages from circular buffer
  if (var == "PH_Version")
    return String("v2.2");
  else if (var == "PLACEHOLDER_LOGS")
    return String(strlog);
  return String();

Next, you make WebServer handlers with template processor:

  server.on("/events_placeholder.html", HTTP_GET, [](AsyncWebServerRequest *request) {
    request->send(SPIFFS, "/events_placeholder.html", "text/html", false, processor);
  });
  server.on("/logs", HTTP_GET, [](AsyncWebServerRequest *request) {
    request->send(SPIFFS, "/events.html", "text/html", false, processor);
  });

In your web-page, events.html in this example, create the placeholder that will be replaced:

    <div class="top_box">
        <div class = "box_head"> <div class = "title"> <h1>Logs</h1> </div> </div>
        <div class = "text_box">%PLACEHOLDER_LOGS%</div>
    </div>

Next, create a JQuery script that periodically replaces the logs:

var text_box = document.querySelector('.text_box');

if (text_box != null) {
    function refresh() {
        setTimeout(function () {
            if ($('.text_box').length) $('.text_box').load('events_placeholder.html');
            refresh();
        }, 1000);
    }

    $(document).ready(function () {
        if ($('.text_box').length) $('.text_box').load('events_placeholder.html');
        refresh();
    });
}

The trick is that events_placeholder.html is a HTML file that only contains %PLACEHOLDER_LOGS%. This is used in an Ajax call to periodically replace the contents from events.html. This can be added to any number of HTML pages.
Just keep in mind that the more web-pages and the more things happening on those pages that you have, the more the web server will slow down. I have around 7 web-pages and 20 placeholders thrown around those pages and the web-server is moderately fast.

Looking at it I am sure it can be vastly improved and made faster, but I am too lazy to change something that works.

[tbertels]

See #4346