Adding support for cached-playlist

README.md

@@ -10,9 +10,9 @@
 * Partition-layout for ESP32 is changed along with this branch. This step was necessary in order to resize (enlarge) the memory-region where especially the assignments for the RFID-tags are saved. As all permanent settings (e.g. WiFi-settings) are saved there too, it's necessary to re-enter WiFi-credentials after update. But the most important thing is to recover the assignments for the RFID-tags. Please consult my [migration-document](https://forum.espuino.de/t/wechsel-zum-refactoring-branch-was-ist-zu-beachten/510).
 ## Changelog
 Last three events:
+* 30.06.2021: Added directive `CACHED_PLAYLIST_ENABLE` for faster playlist-generation
 * 22.06.2021: Changed ESP32' partition-layout in order to provider bigger NVS-storage.
 * 15.06.2021: Added interrupt-handling to PCA9555
-* 08.06.2021: Added global support for PA/HP-enable
 ## Known bugs
 * Some webstreams don't run. Guess it's a combination of saturated connection-pool and lack of heap-memory. Works probably better if ESP32-WROVER (e.g. Lolin D32 pro) is used, as this chip has PSRAM. Advice: Don't enable modules (e.g. MQTT) if you don't need them as this could save memory (and trouble).
 ## ESPuino - what's that?

changelog.md

@@ -3,7 +3,8 @@
 * xx.05.2021: Fixing/stabilizing code
 * 08.06.2021: Added global support for PA/HP-enable
 * 15.06.2021: Added interrupt-handling to PCA9555
-* 22.06.2021: Changed ESP32's partition-layout in order to provider bigger NVS-storage.
+* 22.06.2021: Changed ESP32's partition-layout in order to provider bigger NVS-storage
+* 30.06.2021: Added directive `CACHED_PLAYLIST_ENABLE` for faster playlist-generation
 ## Old (monolithic main.cpp)
 * 11.07.2020: Added support for reversed Neopixel addressing.
 * 09.10.2020: mqttUser / mqttPassword can now be configured via webgui.

src/AudioPlayer.cpp

@@ -718,7 +718,7 @@ void AudioPlayer_TrackQueueDispatcher(const char *_itemToPlay, const uint32_t _l
     #endif
 
     if (_playMode != WEBSTREAM) {
-        musicFiles = SdCard_ReturnPlaylist(filename);
+        musicFiles = SdCard_ReturnPlaylist(filename, _playMode);
     } else {
         musicFiles = AudioPlayer_ReturnPlaylistFromWebstream(filename);
     }

src/LogMessages_DE.cpp

@@ -189,5 +189,8 @@
     const char portExpanderNotFound[] PROGMEM = "Port-expander nicht gefunden";
     const char portExpanderInterruptEnabled[] PROGMEM = "Interrupt für Port-Expander aktiviert";
     const char warningRefactoring[] PROGMEM = "!!!!WICHTIG!!!! Beachte bitte https://forum.espuino.de/t/wechsel-zum-refactoring-branch-was-ist-zu-beachten/510 !!!!WICHTIG!!!!";
+    const char playlistGenModeUncached[] PROGMEM = "Playlist-Generierung: uncached";
+    const char playlistGenModeCached[] PROGMEM = "Playlist-Generierung: cached";
+    const char playlistCacheFoundBut0[] PROGMEM = "Playlist-Cache-File gefunden, jedoch 0 Bytes groß";
 
 #endif

src/LogMessages_EN.cpp

@@ -189,6 +189,8 @@
     const char portExpanderNotFound[] PROGMEM = "Unable to detect port-expander";
     const char portExpanderInterruptEnabled[] PROGMEM = "Enabled interrupt-handling for port-expander";
     const char warningRefactoring[] PROGMEM = "!!!!IMPORTANT!!!! Please review https://forum.espuino.de/t/wechsel-zum-refactoring-branch-was-ist-zu-beachten/510 !!!!IMPORTANT!!!!";
-
+    const char playlistGenModeUncached[] PROGMEM = "Playlist-generation: uncached";
+    const char playlistGenModeCached[] PROGMEM = "Playlist-generation: cached";
+    const char playlistCacheFoundBut0[] PROGMEM = "Playlist-cache-file found but 0 bytes";
 
 #endif

src/SdCard.cpp

@@ -82,11 +82,62 @@ bool fileValid(const char *_fileItem) {
 
 /* Puts SD-file(s) or directory into a playlist
     First element of array always contains the number of payload-items. */
-char **SdCard_ReturnPlaylist(const char *fileName) {
+char **SdCard_ReturnPlaylist(const char *fileName, const uint32_t _playMode) {
     static char **files;
+    char *serializedPlaylist;
     char fileNameBuf[255];
+    char cacheFileNameBuf[275];
+    bool readFromCacheFile = false;
 
+    // Look if file/folder requested really exists. If not => break.
     File fileOrDirectory = gFSystem.open(fileName);
+    if (!fileOrDirectory) {
+        Log_Println((char *) FPSTR(dirOrFileDoesNotExist), LOGLEVEL_ERROR);
+        return NULL;
+    }
+
+    #ifdef CACHED_PLAYLIST_ENABLE
+        strncpy(cacheFileNameBuf, fileName, sizeof(cacheFileNameBuf));
+        strcat(cacheFileNameBuf, "/");
+        strcat(cacheFileNameBuf, (const char*) FPSTR(playlistCacheFile));       // Build absolute path of cacheFile
+
+        // Decide if to use cacheFile. It needs to exist first...
+        if (gFSystem.exists(cacheFileNameBuf)) {     // Check if cacheFile (already) exists
+            readFromCacheFile = true;
+        }
+
+        // ...and playmode has to be != random/single (as random along with caching doesn't make sense at all)
+        if (_playMode == ALL_TRACKS_OF_DIR_RANDOM ||
+            _playMode == ALL_TRACKS_OF_DIR_RANDOM_LOOP ||
+            _playMode == SINGLE_TRACK ||
+            _playMode == SINGLE_TRACK_LOOP) {
+                readFromCacheFile = false;
+        }
+
+        // Read linear playlist (csv with #-delimiter) from cachefile (faster!)
+        if (readFromCacheFile) {
+            File cacheFile = gFSystem.open(cacheFileNameBuf);
+            if (cacheFile) {
+                uint32_t cacheFileSize = cacheFile.size();
+
+                if (!(cacheFileSize >= 1)) {        // Make sure it's greater than 0 bytes
+                    Log_Println((char *) FPSTR(playlistCacheFoundBut0), LOGLEVEL_ERROR);
+                    readFromCacheFile = false;
+                } else {
+                    Log_Println((char *) FPSTR(playlistGenModeCached), LOGLEVEL_NOTICE);
+                    serializedPlaylist = (char *) x_calloc(cacheFileSize+10, sizeof(char));
+
+                    char buf;
+                    uint32_t fPos = 0;
+                    while (cacheFile.available() > 0) {
+                        buf = cacheFile.read();
+                        serializedPlaylist[fPos++] = buf;
+                    }
+                }
+                cacheFile.close();
+            }
+        }
+    #endif
 
     snprintf(Log_Buffer, Log_BufferLength, "%s: %u", (char *) FPSTR(freeMemory), ESP.getFreeHeap());
     Log_Println(Log_Buffer, LOGLEVEL_DEBUG);
@@ -99,68 +150,77 @@ char **SdCard_ReturnPlaylist(const char *fileName) {
         Log_Println(Log_Buffer, LOGLEVEL_DEBUG);
     }
 
-    if (!fileOrDirectory) {
-        Log_Println((char *) FPSTR(dirOrFileDoesNotExist), LOGLEVEL_ERROR);
-        return NULL;
-    }
-
-    // File-mode
-    if (!fileOrDirectory.isDirectory()) {
-        files = (char **) x_malloc(sizeof(char *) * 2);
-        if (files == NULL) {
-            Log_Println((char *) FPSTR(unableToAllocateMemForPlaylist), LOGLEVEL_ERROR);
-            System_IndicateError();
-            return NULL;
-        }
-        Log_Println((char *) FPSTR(fileModeDetected), LOGLEVEL_INFO);
-        strncpy(fileNameBuf, (char *) fileOrDirectory.name(), sizeof(fileNameBuf) / sizeof(fileNameBuf[0]));
-        if (fileValid(fileNameBuf))
-        {
+    // Don't read from cachefile (if cachefile doesn't exist, playmode doesn't fit or caching isn't desired)
+    if (!readFromCacheFile) {
+        Log_Println((char *) FPSTR(playlistGenModeUncached), LOGLEVEL_NOTICE);
+        // File-mode
+        if (!fileOrDirectory.isDirectory()) {
             files = (char **) x_malloc(sizeof(char *) * 2);
-            files[1] = x_strdup(fileNameBuf);
-        }
-        files[0] = x_strdup("1"); // Number of files is always 1 in file-mode
-
-        return ++files;
-    }
+            if (files == NULL) {
+                Log_Println((char *) FPSTR(unableToAllocateMemForPlaylist), LOGLEVEL_ERROR);
+                System_IndicateError();
+                return NULL;
+            }
+            Log_Println((char *) FPSTR(fileModeDetected), LOGLEVEL_INFO);
+            strncpy(fileNameBuf, (char *) fileOrDirectory.name(), sizeof(fileNameBuf) / sizeof(fileNameBuf[0]));
+            if (fileValid(fileNameBuf)) {
+                files = (char **) x_malloc(sizeof(char *) * 2);
+                files[1] = x_strdup(fileNameBuf);
+            }
+            files[0] = x_strdup("1"); // Number of files is always 1 in file-mode
 
-    // Directory-mode
-    uint16_t allocCount = 1;
-    uint16_t allocSize = 512;
-    if (psramInit()) {
-        allocSize = 16384; // There's enough PSRAM. So we don't have to care...
-    }
-    char *serializedPlaylist;
+            return ++files;
+        }
 
-    serializedPlaylist = (char *) x_calloc(allocSize, sizeof(char));
+        // Directory-mode
+        uint16_t allocCount = 1;
+        uint16_t allocSize = 4096;
+        if (psramInit()) {
+            allocSize = 16384; // There's enough PSRAM. So we don't have to care...
+        }
 
-    while (true) {
-        File fileItem = fileOrDirectory.openNextFile();
-        if (!fileItem) {
-            break;
+        serializedPlaylist = (char *) x_calloc(allocSize, sizeof(char));
+        File cacheFile;
+        if (readFromCacheFile) {
+            cacheFile = gFSystem.open(cacheFileNameBuf, FILE_WRITE);
         }
-        if (fileItem.isDirectory()) {
-            continue;
-        } else {
-            strncpy(fileNameBuf, (char *) fileItem.name(), sizeof(fileNameBuf) / sizeof(fileNameBuf[0]));
 
-            // Don't support filenames that start with "." and only allow .mp3
-            if (fileValid(fileNameBuf)) {
-                /*snprintf(Log_Buffer, Log_BufferLength, "%s: %s", (char *) FPSTR(nameOfFileFound), fileNameBuf);
-                Log_Println(Log_Buffer, LOGLEVEL_INFO);*/
-                if ((strlen(serializedPlaylist) + strlen(fileNameBuf) + 2) >= allocCount * allocSize) {
-                    serializedPlaylist = (char *) realloc(serializedPlaylist, ++allocCount * allocSize);
-                    Log_Println((char *) FPSTR(reallocCalled), LOGLEVEL_DEBUG);
-                    if (serializedPlaylist == NULL) {
-                        Log_Println((char *) FPSTR(unableToAllocateMemForLinearPlaylist), LOGLEVEL_ERROR);
-                        System_IndicateError();
-                        return files;
+        while (true) {
+            File fileItem = fileOrDirectory.openNextFile();
+            if (!fileItem) {
+                break;
+            }
+            if (fileItem.isDirectory()) {
+                continue;
+            } else {
+                strncpy(fileNameBuf, (char *) fileItem.name(), sizeof(fileNameBuf) / sizeof(fileNameBuf[0]));
+
+                // Don't support filenames that start with "." and only allow .mp3
+                if (fileValid(fileNameBuf)) {
+                    /*snprintf(Log_Buffer, Log_BufferLength, "%s: %s", (char *) FPSTR(nameOfFileFound), fileNameBuf);
+                    Log_Println(Log_Buffer, LOGLEVEL_INFO);*/
+                    if ((strlen(serializedPlaylist) + strlen(fileNameBuf) + 2) >= allocCount * allocSize) {
+                        serializedPlaylist = (char *) realloc(serializedPlaylist, ++allocCount * allocSize);
+                        Log_Println((char *) FPSTR(reallocCalled), LOGLEVEL_DEBUG);
+                        if (serializedPlaylist == NULL) {
+                            Log_Println((char *) FPSTR(unableToAllocateMemForLinearPlaylist), LOGLEVEL_ERROR);
+                            System_IndicateError();
+                            return files;
+                        }
+                    }
+                    strcat(serializedPlaylist, stringDelimiter);
+                    strcat(serializedPlaylist, fileNameBuf);
+                    if (cacheFile && readFromCacheFile) {
+                        cacheFile.print(stringDelimiter);
+                        cacheFile.print(fileNameBuf);       // Write linear playlist to cacheFile
                     }
                 }
-                strcat(serializedPlaylist, stringDelimiter);
-                strcat(serializedPlaylist, fileNameBuf);
             }
         }
+
+        if (cacheFile && readFromCacheFile) {
+            cacheFile.close();
+        }
     }
 
     // Get number of elements out of serialized playlist

src/SdCard.h

@@ -11,4 +11,4 @@ extern fs::FS gFSystem;
 void SdCard_Init(void);
 void SdCard_Exit(void);
 sdcard_type_t SdCard_GetType(void);
-char **SdCard_ReturnPlaylist(const char *fileName);
+char **SdCard_ReturnPlaylist(const char *fileName, const uint32_t _playMode);

src/logmessages.h

@@ -185,3 +185,6 @@ extern const char portExpanderFound[];
 extern const char portExpanderNotFound[];
 extern const char portExpanderInterruptEnabled[];
 extern const char warningRefactoring[];
+extern const char playlistGenModeUncached[];
+extern const char playlistGenModeCached[];
+extern const char playlistCacheFoundBut0[];

src/main.cpp

@@ -150,7 +150,7 @@ void setup()
     Serial.println(F(" | |___   ___) | |  __/  | |_| | | | | | | | | (_) |"));
     Serial.println(F(" |_____| |____/  |_|      \\__,_| |_| |_| |_|  \\___/ "));
     Serial.println(F(" Rfid-controlled musicplayer\n"));
-    Serial.println(F(" Rev 20210622-1\n"));
+    Serial.println(F(" Rev 20210630-1\n"));
 
     // print wake-up reason
     printWakeUpReason();

src/settings.h

@@ -41,6 +41,7 @@
     #define USEROTARY_ENABLE                // If rotary-encoder is used (don't forget to review WAKEUP_BUTTON if you disable this feature!)
     #define BLUETOOTH_ENABLE                // If enabled and bluetooth-mode is active, you can stream to your ESPuino via bluetooth (a2dp-sink).
     //#define IR_CONTROL_ENABLE             // Enables remote control
+    #define CACHED_PLAYLIST_ENABLE          // Enables playlist-caching (infos: https://forum.espuino.de/t/neues-feature-cached-playlist/515)
 
 
     //################## select SD card mode #############################
@@ -164,7 +165,7 @@
 
     // Where to store the backup-file for NVS-records
     constexpr const char backupFile[] PROGMEM = "/backup.txt"; // File is written every time a (new) RFID-assignment via GUI is done
-
+    constexpr const char playlistCacheFile[] PROGMEM = "playlistcache.csv"; // Filename that is used for caching playlists
 
     //#################### Settings for optional Modules##############################
     // (optinal) Neopixel