Merge branch 'filesystem' into merge-fs2

platformio.ini

@@ -178,6 +178,7 @@ lib_deps =
     #U8g2@~2.27.2
   #For Dallas sensor uncomment following 2 lines
     #OneWire@~2.3.5
+    #milesburton/DallasTemperature@^3.9.0
   #For BME280 sensor uncomment following
     #BME280@~3.0.0
 lib_ignore =
@@ -227,6 +228,7 @@ platform = ${common.platform_wled_default}
 upload_speed = 921600
 board_build.ldscript = ${common.ldscript_4m1m}
 build_flags = ${common.build_flags_esp8266} 
+monitor_filters = esp8266_exception_decoder
 
 [env:heltec_wifi_kit_8]
 board = d1_mini
@@ -373,5 +375,5 @@ build_flags = ${common.build_flags_esp8266} ${common.debug_flags} ${common.build
 
 [env:travis_esp32]
 extends = env:esp32dev
-build_type = debug
+; build_type = debug
 build_flags = ${common.build_flags_esp32} ${common.debug_flags} ${common.build_flags_all_features}

wled00/const.h

@@ -104,6 +104,15 @@
 #define SEG_OPTION_FREEZE         5            //Segment contents will not be refreshed
 #define SEG_OPTION_TRANSITIONAL   7
 
+// WLED Error modes
+#define ERR_NONE         0  // All good :)
+#define ERR_EEP_COMMIT   2  // Could not commit to EEPROM (wrong flash layout?)
+#define ERR_JSON         9  // JSON parsing failed (input too large?)
+#define ERR_FS_BEGIN    10  // Could not init filesystem (no partition?)
+#define ERR_FS_QUOTA    11  // The FS is full or the maximum file size is reached
+#define ERR_FS_PLOAD    12  // It was attempted to load a preset that does not exist
+#define ERR_FS_GENERAL  19  // A general unspecified filesystem error occured
+
 //Timer mode types
 #define NL_MODE_SET               0            //After nightlight time elapsed, set to target brightness
 #define NL_MODE_FADE              1            //Fade to target brightness gradually
@@ -126,6 +135,7 @@
 
 #define ABL_MILLIAMPS_DEFAULT 850; // auto lower brightness to stay close to milliampere limit
 
+
 #define TOUCH_THRESHOLD 32 // limit to recognize a touch, higher value means more sensitive
 
 // Size of buffer for API JSON object (increase for more segments)

wled00/data/index.htm

@@ -1432,7 +1432,8 @@
 function updateTrail(e, slidercol)
 {
 	if (e==null) return;
-	var progress = e.value *100 /255;
+	var max = e.hasAttribute('max') ? e.attributes['max'].value : 255;
+	var progress = e.value * 100 / max;
 	progress = parseInt(progress);
 	var scol;
 	switch (slidercol) {

wled00/fcn_declare.h

@@ -47,6 +47,10 @@ void handleE131Packet(e131_packet_t* p, IPAddress clientIP, bool isArtnet);
 
 //file.cpp
 bool handleFileRead(AsyncWebServerRequest*, String path);
+bool writeObjectToFileUsingId(const char* file, uint16_t id, JsonDocument* content);
+bool writeObjectToFile(const char* file, const char* key, JsonDocument* content);
+bool readObjectFromFileUsingId(const char* file, uint16_t id, JsonDocument* dest);
+bool readObjectFromFile(const char* file, const char* key, JsonDocument* dest);
 
 //hue.cpp
 void handleHue();
@@ -84,8 +88,8 @@ void handleIR();
 
 void deserializeSegment(JsonObject elem, byte it);
 bool deserializeState(JsonObject root);
-void serializeSegment(JsonObject& root, WS2812FX::Segment& seg, byte id);
-void serializeState(JsonObject root);
+void serializeSegment(JsonObject& root, WS2812FX::Segment& seg, byte id, bool forPreset = false);
+void serializeState(JsonObject root, bool forPreset = false);
 void serializeInfo(JsonObject root);
 void serveJson(AsyncWebServerRequest* request);
 bool serveLiveLeds(AsyncWebServerRequest* request, uint32_t wsClient = 0);
@@ -191,7 +195,7 @@ void saveSettingsToEEPROM();
 void loadSettingsFromEEPROM(bool first);
 void savedToPresets();
 bool applyPreset(byte index, bool loadBri = true);
-void savePreset(byte index, bool persist = true);
+void savePreset(byte index, bool persist = true, const char* pname = nullptr, byte prio = 50, JsonObject saveobj = JsonObject());
 void loadMacro(byte index, char* m);
 void applyMacro(byte index);
 void saveMacro(byte index, const String& mc, bool persist = true); //only commit on single save, not in settings

wled00/file.cpp

@@ -4,15 +4,233 @@
  * Utility for SPIFFS filesystem
  */
 
-//filesystem
 #ifndef WLED_DISABLE_FILESYSTEM
-#include <FS.h>
-#ifdef ARDUINO_ARCH_ESP32
-#include "SPIFFS.h"
-#endif
-#include "SPIFFSEditor.h"
-#endif
 
+//find() that reads and buffers data from file stream in 256-byte blocks.
+//Significantly faster, f.find(key) can take SECONDS for multi-kB files
+bool bufferedFind(const char *target, File f) {
+  #ifdef WLED_DEBUG_FS
+    DEBUGFS_PRINT("Find ");
+    DEBUGFS_PRINTLN(target);
+    uint32_t s = millis();
+  #endif
+
+  if (!f || !f.size()) return false;
+  size_t targetLen = strlen(target);
+
+  size_t index = 0;
+  byte c;
+  uint16_t bufsize = 0, count = 0;
+  byte buf[256];
+  f.seek(0);
+
+  while (f.position() < f.size() -1) {
+    bufsize = f.read(buf, 256);
+    count = 0;
+    while (count < bufsize) {
+      if(buf[count] != target[index])
+      index = 0; // reset index if any char does not match
+
+      if(buf[count] == target[index]) {
+        if(++index >= targetLen) { // return true if all chars in the target match
+          f.seek((f.position() - bufsize) + count +1);
+          DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
+          return true;
+        }
+      }
+      count++;
+    }
+  }
+  DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
+  return false;
+}
+
+//find empty spots in file stream in 256-byte blocks.
+bool bufferedFindSpace(uint16_t targetLen, File f) {
+  #ifdef WLED_DEBUG_FS
+    DEBUGFS_PRINTF("Find %d spaces\n", targetLen);
+    uint32_t s = millis();
+  #endif
+
+  if (!f || !f.size()) return false;
+
+  uint16_t index = 0;
+  uint16_t bufsize = 0, count = 0;
+  byte buf[256];
+  f.seek(0);
+
+  while (f.position() < f.size() -1) {
+    bufsize = f.read(buf, 256);
+    count = 0;
+
+    while (count < bufsize) {
+      if(buf[count] != ' ')
+      index = 0; // reset index if not space
+
+      if(buf[count] == ' ') {
+        if(++index >= targetLen) { // return true if space long enough
+          f.seek((f.position() - bufsize) + count +1 - targetLen);
+          DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
+          return true;
+        }
+      }
+      count++;
+    }
+  }
+  DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
+  return false;
+}
+
+bool appendObjectToFile(File f, const char* key, JsonDocument* content, uint32_t s)
+{
+  #ifdef WLED_DEBUG_FS
+    DEBUGFS_PRINTLN("Append");
+    uint32_t s1 = millis();
+  #endif
+  uint32_t pos = 0;
+  if (!f) return false;
+  if (f.size() < 3) f.print("{}");
+
+  //if there is enough empty space in file, insert there instead of appending
+  uint32_t contentLen = measureJson(*content);
+  DEBUGFS_PRINTF("CLen %d\n", contentLen);
+  if (bufferedFindSpace(contentLen + strlen(key) + 1, f)) {
+    if (f.position() > 2) f.write(','); //add comma if not first object
+    f.print(key);
+    serializeJson(*content, f);
+    DEBUGFS_PRINTF("Inserted, took %d ms (total %d)", millis() - s1, millis() - s);
+    return true;
+  }
+
+  //check if last character in file is '}' (typical)
+  f.seek(1, SeekEnd);
+  if (f.read() == '}') pos = f.size() -1;
+
+  if (pos == 0) //not found
+  {
+    DEBUGFS_PRINTLN("not }");
+    while (bufferedFind("}",f)) //find last closing bracket in JSON if not last char
+    {
+      pos = f.position();
+    }
+  }
+  DEBUGFS_PRINT("pos "); DEBUGFS_PRINTLN(pos);
+  if (pos > 2)
+  {
+    f.seek(pos, SeekSet);
+    f.write(',');
+  } else { //file content is not valid JSON object
+    f.seek(0, SeekSet);
+    f.write('{'); //start JSON
+  }
+
+  f.print(key);
+
+  //Append object
+  serializeJson(*content, f);
+  f.write('}');
+
+  f.close();
+  DEBUGFS_PRINTF("Appended, took %d ms (total %d)", millis() - s1, millis() - s);
+}
+
+bool writeObjectToFileUsingId(const char* file, uint16_t id, JsonDocument* content)
+{
+  char objKey[10];
+  sprintf(objKey, "\"%ld\":", id);
+  writeObjectToFile(file, objKey, content);
+}
+
+bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
+{
+  uint32_t s = 0; //timing
+  #ifdef WLED_DEBUG_FS
+    DEBUGFS_PRINTF("Write to %s with key %s >>>\n", file, key);
+    serializeJson(*content, Serial); DEBUGFS_PRINTLN();
+    s = millis();
+  #endif
+
+  uint32_t pos = 0;
+  File    f = WLED_FS.open(file, "r+");
+  if (!f && !WLED_FS.exists(file)) f = WLED_FS.open(file, "w+");
+  if (!f) {
+    DEBUGFS_PRINTLN("Failed to open!");
+    return false;
+  }
+
+  if (!bufferedFind(key, f)) //key does not exist in file
+  {
+    return appendObjectToFile(f, key, content, s);
+  } 
+
+  //exists
+  pos = f.position();
+  //measure out end of old object
+  StaticJsonDocument<1024> doc;
+  deserializeJson(doc, f);
+  uint32_t pos2 = f.position();
+
+  uint32_t oldLen = pos2 - pos;
+  #ifdef WLED_DEBUG_FS
+    DEBUGFS_PRINTF("Old obj len %d >>> ", oldLen);
+    serializeJson(doc, Serial);
+    DEBUGFS_PRINTLN();
+  #endif
+
+  if (!content->isNull() && measureJson(*content) <= oldLen)  //replace
+  {
+    DEBUGFS_PRINTLN("replace");
+    f.seek(pos);
+    serializeJson(*content, f);
+    //pad rest
+    for (uint32_t i = f.position(); i < pos2; i++) {
+      f.write(' ');
+    }
+  } else { //delete
+    DEBUGFS_PRINTLN("delete");
+    pos -= strlen(key);
+    if (pos > 3) pos--; //also delete leading comma if not first object
+    f.seek(pos);
+    for (uint32_t i = pos; i < pos2; i++) {
+      f.write(' ');
+    }
+    if (!content->isNull()) return appendObjectToFile(f, key, content, s);
+  }
+  f.close();
+  DEBUGFS_PRINTF("Deleted, took %d ms\n", millis() - s);
+  return true;
+}
+
+bool readObjectFromFileUsingId(const char* file, uint16_t id, JsonDocument* dest)
+{
+  char objKey[10];
+  sprintf(objKey, "\"%ld\":", id);
+  readObjectFromFile(file, objKey, dest);
+}
+
+bool readObjectFromFile(const char* file, const char* key, JsonDocument* dest)
+{
+  #ifdef WLED_DEBUG_FS
+    DEBUGFS_PRINTF("Read from %s with key %s >>>\n", file, key);
+    uint32_t s = millis();
+  #endif
+  File f = WLED_FS.open(file, "r");
+  if (!f) return false;
+
+  if (!bufferedFind(key, f)) //key does not exist in file
+  {
+    f.close();
+    DEBUGFS_PRINTLN("Obj not found.");
+    return false;
+  }
+
+  deserializeJson(*dest, f);
+
+  f.close();
+  DEBUGFS_PRINTF("Read, took %d ms\n", millis() - s);
+  return true;
+}
+#endif
 
 #if !defined WLED_DISABLE_FILESYSTEM && defined WLED_ENABLE_FS_SERVING
 //Un-comment any file types you need
@@ -38,13 +256,13 @@ bool handleFileRead(AsyncWebServerRequest* request, String path){
   DEBUG_PRINTLN("FileRead: " + path);
   if(path.endsWith("/")) path += "index.htm";
   String contentType = getContentType(request, path);
-  String pathWithGz = path + ".gz";
-  if(SPIFFS.exists(pathWithGz)){
-    request->send(SPIFFS, pathWithGz, contentType);
+  /*String pathWithGz = path + ".gz";
+  if(WLED_FS.exists(pathWithGz)){
+    request->send(WLED_FS, pathWithGz, contentType);
     return true;
-  }
-  if(SPIFFS.exists(path)) {
-    request->send(SPIFFS, path, contentType);
+  }*/
+  if(WLED_FS.exists(path)) {
+    request->send(WLED_FS, path, contentType);
     return true;
   }
   return false;