Adding support for cached-playlist

README.md

@@ -10,9 +10,9 @@
 * Partition-layout for ESP32 is changed along with this branch. This step was necessary in order to resize (enlarge) the memory-region where especially the assignments for the RFID-tags are saved. As all permanent settings (e.g. WiFi-settings) are saved there too, it's necessary to re-enter WiFi-credentials after update. But the most important thing is to recover the assignments for the RFID-tags. Please consult my [migration-document](https://forum.espuino.de/t/wechsel-zum-refactoring-branch-was-ist-zu-beachten/510).
 ## Changelog
 Last three events:
+* 30.06.2021: Added directive `CACHED_PLAYLIST_ENABLE` for faster playlist-generation
 * 22.06.2021: Changed ESP32' partition-layout in order to provider bigger NVS-storage.
 * 15.06.2021: Added interrupt-handling to PCA9555
-* 08.06.2021: Added global support for PA/HP-enable
 ## Known bugs
 * Some webstreams don't run. Guess it's a combination of saturated connection-pool and lack of heap-memory. Works probably better if ESP32-WROVER (e.g. Lolin D32 pro) is used, as this chip has PSRAM. Advice: Don't enable modules (e.g. MQTT) if you don't need them as this could save memory (and trouble).
 ## ESPuino - what's that?

changelog.md

@@ -3,7 +3,8 @@
 * xx.05.2021: Fixing/stabilizing code
 * 08.06.2021: Added global support for PA/HP-enable
 * 15.06.2021: Added interrupt-handling to PCA9555
-* 22.06.2021: Changed ESP32's partition-layout in order to provider bigger NVS-storage.
+* 22.06.2021: Changed ESP32's partition-layout in order to provider bigger NVS-storage
+* 30.06.2021: Added directive `CACHED_PLAYLIST_ENABLE` for faster playlist-generation
 ## Old (monolithic main.cpp)
 * 11.07.2020: Added support for reversed Neopixel addressing.
 * 09.10.2020: mqttUser / mqttPassword can now be configured via webgui.

src/AudioPlayer.cpp

@@ -718,7 +718,7 @@ void AudioPlayer_TrackQueueDispatcher(const char *_itemToPlay, const uint32_t _l
     #endif
 
     if (_playMode != WEBSTREAM) {
-        musicFiles = SdCard_ReturnPlaylist(filename);
+        musicFiles = SdCard_ReturnPlaylist(filename, _playMode);
     } else {
         musicFiles = AudioPlayer_ReturnPlaylistFromWebstream(filename);
     }

src/LogMessages_DE.cpp

@@ -189,5 +189,8 @@
     const char portExpanderNotFound[] PROGMEM = "Port-expander nicht gefunden";
     const char portExpanderInterruptEnabled[] PROGMEM = "Interrupt für Port-Expander aktiviert";
     const char warningRefactoring[] PROGMEM = "!!!!WICHTIG!!!! Beachte bitte https://forum.espuino.de/t/wechsel-zum-refactoring-branch-was-ist-zu-beachten/510 !!!!WICHTIG!!!!";
+    const char playlistGenModeUncached[] PROGMEM = "Playlist-Generierung: uncached";
+    const char playlistGenModeCached[] PROGMEM = "Playlist-Generierung: cached";
+    const char playlistCacheFoundBut0[] PROGMEM = "Playlist-Cache-File gefunden, jedoch 0 Bytes groß";
 
 #endif

src/LogMessages_EN.cpp

@@ -189,6 +189,8 @@
     const char portExpanderNotFound[] PROGMEM = "Unable to detect port-expander";
     const char portExpanderInterruptEnabled[] PROGMEM = "Enabled interrupt-handling for port-expander";
     const char warningRefactoring[] PROGMEM = "!!!!IMPORTANT!!!! Please review https://forum.espuino.de/t/wechsel-zum-refactoring-branch-was-ist-zu-beachten/510 !!!!IMPORTANT!!!!";
-
+    const char playlistGenModeUncached[] PROGMEM = "Playlist-generation: uncached";
+    const char playlistGenModeCached[] PROGMEM = "Playlist-generation: cached";
+    const char playlistCacheFoundBut0[] PROGMEM = "Playlist-cache-file found but 0 bytes";
 
 #endif

src/SdCard.cpp

@@ -82,11 +82,62 @@ bool fileValid(const char *_fileItem) {
 
 /* Puts SD-file(s) or directory into a playlist
     First element of array always contains the number of payload-items. */
-char **SdCard_ReturnPlaylist(const char *fileName) {
+char **SdCard_ReturnPlaylist(const char *fileName, const uint32_t _playMode) {
     static char **files;
+    char *serializedPlaylist;
     char fileNameBuf[255];
+    char cacheFileNameBuf[275];
+    bool readFromCacheFile = false;
 
+    // Look if file/folder requested really exists. If not => break.
     File fileOrDirectory = gFSystem.open(fileName);
+    if (!fileOrDirectory) {
+        Log_Println((char *) FPSTR(dirOrFileDoesNotExist), LOGLEVEL_ERROR);
+        return NULL;
+    }
+
+    #ifdef CACHED_PLAYLIST_ENABLE
+        strncpy(cacheFileNameBuf, fileName, sizeof(cacheFileNameBuf));
+        strcat(cacheFileNameBuf, "/");
+        strcat(cacheFileNameBuf, (const char*) FPSTR(playlistCacheFile));       // Build absolute path of cacheFile
+
+        // Decide if to use cacheFile. It needs to exist first...
+        if (gFSystem.exists(cacheFileNameBuf)) {     // Check if cacheFile (already) exists
+            readFromCacheFile = true;
+        }
+
+        // ...and playmode has to be != random/single (as random along with caching doesn't make sense at all)
+        if (_playMode == ALL_TRACKS_OF_DIR_RANDOM ||
+            _playMode == ALL_TRACKS_OF_DIR_RANDOM_LOOP ||
+            _playMode == SINGLE_TRACK ||
+            _playMode == SINGLE_TRACK_LOOP) {
+                readFromCacheFile = false;
+        }
+
+        // Read linear playlist (csv with #-delimiter) from cachefile (faster!)
+        if (readFromCacheFile) {
+            File cacheFile = gFSystem.open(cacheFileNameBuf);
+            if (cacheFile) {
+                uint32_t cacheFileSize = cacheFile.size();
+
+                if (!(cacheFileSize >= 1)) {        // Make sure it's greater than 0 bytes
+                    Log_Println((char *) FPSTR(playlistCacheFoundBut0), LOGLEVEL_ERROR);
+                    readFromCacheFile = false;
+                } else {
+                    Log_Println((char *) FPSTR(playlistGenModeCached), LOGLEVEL_NOTICE);
+                    serializedPlaylist = (char *) x_calloc(cacheFileSize+10, sizeof(char));
+
+                    char buf;
+                    uint32_t fPos = 0;
+                    while (cacheFile.available() > 0) {
+                        buf = cacheFile.read();
+                        serializedPlaylist[fPos++] = buf;
+                    }
+                }
+                cacheFile.close();
+            }
+        }
+    #endif
 
     snprintf(Log_Buffer, Log_BufferLength, "%s: %u", (char *) FPSTR(freeMemory), ESP.getFreeHeap());
     Log_Println(Log_Buffer, LOGLEVEL_DEBUG);
@@ -99,11 +150,9 @@ char **SdCard_ReturnPlaylist(const char *fileName) {
         Log_Println(Log_Buffer, LOGLEVEL_DEBUG);
     }
 
-    if (!fileOrDirectory) {
-        Log_Println((char *) FPSTR(dirOrFileDoesNotExist), LOGLEVEL_ERROR);
-        return NULL;
-    }
-
+    // Don't read from cachefile (if cachefile doesn't exist, playmode doesn't fit or caching isn't desired)
+    if (!readFromCacheFile) {
+        Log_Println((char *) FPSTR(playlistGenModeUncached), LOGLEVEL_NOTICE);
         // File-mode
         if (!fileOrDirectory.isDirectory()) {
             files = (char **) x_malloc(sizeof(char *) * 2);
@@ -114,8 +163,7 @@ char **SdCard_ReturnPlaylist(const char *fileName) {
             }
             Log_Println((char *) FPSTR(fileModeDetected), LOGLEVEL_INFO);
             strncpy(fileNameBuf, (char *) fileOrDirectory.name(), sizeof(fileNameBuf) / sizeof(fileNameBuf[0]));
-        if (fileValid(fileNameBuf))
-        {
+            if (fileValid(fileNameBuf)) {
                 files = (char **) x_malloc(sizeof(char *) * 2);
                 files[1] = x_strdup(fileNameBuf);
             }
@@ -126,13 +174,16 @@ char **SdCard_ReturnPlaylist(const char *fileName) {
 
         // Directory-mode
         uint16_t allocCount = 1;
-    uint16_t allocSize = 512;
+        uint16_t allocSize = 4096;
         if (psramInit()) {
             allocSize = 16384; // There's enough PSRAM. So we don't have to care...
         }
-    char *serializedPlaylist;
 
         serializedPlaylist = (char *) x_calloc(allocSize, sizeof(char));
+        File cacheFile;
+        if (readFromCacheFile) {
+            cacheFile = gFSystem.open(cacheFileNameBuf, FILE_WRITE);
+        }
 
         while (true) {
             File fileItem = fileOrDirectory.openNextFile();
@@ -159,10 +210,19 @@ char **SdCard_ReturnPlaylist(const char *fileName) {
                     }
                     strcat(serializedPlaylist, stringDelimiter);
                     strcat(serializedPlaylist, fileNameBuf);
+                    if (cacheFile && readFromCacheFile) {
+                        cacheFile.print(stringDelimiter);
+                        cacheFile.print(fileNameBuf);       // Write linear playlist to cacheFile
+                    }
                 }
             }
         }
 
+        if (cacheFile && readFromCacheFile) {
+            cacheFile.close();
+        }
+    }
+
     // Get number of elements out of serialized playlist
     uint32_t cnt = 0;
     for (uint32_t k = 0; k < (strlen(serializedPlaylist)); k++) {

src/SdCard.h

@@ -11,4 +11,4 @@ extern fs::FS gFSystem;
 void SdCard_Init(void);
 void SdCard_Exit(void);
 sdcard_type_t SdCard_GetType(void);
-char **SdCard_ReturnPlaylist(const char *fileName);
+char **SdCard_ReturnPlaylist(const char *fileName, const uint32_t _playMode);

src/logmessages.h

@@ -185,3 +185,6 @@ extern const char portExpanderFound[];
 extern const char portExpanderNotFound[];
 extern const char portExpanderInterruptEnabled[];
 extern const char warningRefactoring[];
+extern const char playlistGenModeUncached[];
+extern const char playlistGenModeCached[];
+extern const char playlistCacheFoundBut0[];

src/main.cpp

@@ -150,7 +150,7 @@ void setup()
     Serial.println(F(" | |___   ___) | |  __/  | |_| | | | | | | | | (_) |"));
     Serial.println(F(" |_____| |____/  |_|      \\__,_| |_| |_| |_|  \\___/ "));
     Serial.println(F(" Rfid-controlled musicplayer\n"));
-    Serial.println(F(" Rev 20210622-1\n"));
+    Serial.println(F(" Rev 20210630-1\n"));
 
     // print wake-up reason
     printWakeUpReason();

src/settings.h

@@ -41,6 +41,7 @@
     #define USEROTARY_ENABLE                // If rotary-encoder is used (don't forget to review WAKEUP_BUTTON if you disable this feature!)
     #define BLUETOOTH_ENABLE                // If enabled and bluetooth-mode is active, you can stream to your ESPuino via bluetooth (a2dp-sink).
     //#define IR_CONTROL_ENABLE             // Enables remote control
+    #define CACHED_PLAYLIST_ENABLE          // Enables playlist-caching (infos: https://forum.espuino.de/t/neues-feature-cached-playlist/515)
 
 
     //################## select SD card mode #############################
@@ -164,7 +165,7 @@
 
     // Where to store the backup-file for NVS-records
     constexpr const char backupFile[] PROGMEM = "/backup.txt"; // File is written every time a (new) RFID-assignment via GUI is done
-
+    constexpr const char playlistCacheFile[] PROGMEM = "playlistcache.csv"; // Filename that is used for caching playlists
 
     //#################### Settings for optional Modules##############################
     // (optinal) Neopixel