parse github contents efficiently, update readme

README.md

@@ -5,8 +5,9 @@ Download Flipper Zero apps directly to your Flipper Zero using WiFi. You no long
 - App Catalog
 - Install Apps
 - Delete Apps 
-- Install Developer Board Flashes
-- Install Custom Apps (coming soon)
+- Install WiFi Developer Board Firmware
+- Install Video Game Module Firmware
+- Install GitHub Repositories (Beta)
 - Install Official Firmware (coming soon)
 
 ## Installation
@@ -38,7 +39,7 @@ Download Flipper Zero apps directly to your Flipper Zero using WiFi. You no long
 - UX Improvements
 
 **v0.8**
-- Download custom apps/assets from a GitHub URL
+- Download GitHub repositories
 
 **1.0**
-- Download Official Firmware/Firmware Updates
+- Download Official Flipper Zero Firmware

flipper_http/flipper_http.h

@@ -23,7 +23,7 @@
 #define BAUDRATE (115200)                 // UART baudrate
 #define RX_BUF_SIZE 2048                  // UART RX buffer size
 #define RX_LINE_BUFFER_SIZE 2048          // UART RX line buffer size (increase for large responses)
-#define MAX_FILE_SHOW 10000               // Maximum data from file to show
+#define MAX_FILE_SHOW 12000               // Maximum data from file to show
 #define FILE_BUFFER_SIZE 512              // File buffer size
 
 // Forward declaration for callback

github/flip_store_github.c

@@ -69,122 +69,230 @@ bool flip_store_get_github_contents(FlipperHTTP *fhttp, const char *author, cons
     return flipper_http_get_request_with_headers(fhttp, link, "{\"Content-Type\":\"application/json\"}");
 }
 
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+
+// Assuming necessary headers and definitions for FuriString, FURI_LOG, etc., are included.
+
 bool flip_store_parse_github_contents(char *file_path, const char *author, const char *repo)
 {
-    // Parse list of files and prepare to download
+    FURI_LOG_I(TAG, "Parsing Github contents from %s - %s.", author, repo);
+    if (!file_path || !author || !repo)
+    {
+        FURI_LOG_E(TAG, "Invalid arguments.");
+        return false;
+    }
+
+    // Load JSON data from file
     FuriString *git_data = flipper_http_load_from_file(file_path);
     if (git_data == NULL)
     {
         FURI_LOG_E(TAG, "Failed to load received data from file.");
         return false;
     }
+
+    // Allocate a new FuriString to hold the entire JSON structure
     FuriString *git_data_str = furi_string_alloc();
     if (!git_data_str)
     {
-        FURI_LOG_E("Game", "Failed to allocate json_data string");
+        FURI_LOG_E(TAG, "Failed to allocate git_data_str.");
+        furi_string_free(git_data);
         return false;
     }
+
+    // Construct the full JSON string
     furi_string_cat_str(git_data_str, "{\"json_data\":");
-    if (memmgr_get_free_heap() < furi_string_size(git_data) + furi_string_size(git_data_str) + 2)
+    furi_string_cat(git_data_str, git_data);
+    furi_string_cat_str(git_data_str, "}");
+    furi_string_free(git_data); // Free the original git_data as it's now part of git_data_str
+
+    // Check available memory
+    const size_t additional_bytes = strlen("{\"json_data\":") + 1; // +1 for the closing "}"
+    if (memmgr_get_free_heap() < furi_string_size(git_data_str) + additional_bytes)
     {
         FURI_LOG_E(TAG, "Not enough memory to allocate git_data_str.");
-        furi_string_free(git_data);
         furi_string_free(git_data_str);
         return false;
     }
-    furi_string_cat(git_data_str, git_data);
-    furi_string_free(git_data);
-    furi_string_cat_str(git_data_str, "}");
-    //
+
     int file_count = 0;
-    for (int i = 0; i < MAX_GITHUB_FILES; i++)
+    char dir[512]; // Increased size to accommodate longer paths if necessary
+    FURI_LOG_I(TAG, "Looping through Github files.");
+    FURI_LOG_I(TAG, "Available memory: %d bytes", memmgr_get_free_heap());
+
+    // Get the C-string and its length for processing
+    char *data = (char *)furi_string_get_cstr(git_data_str);
+    size_t data_len = furi_string_size(git_data_str);
+
+    size_t pos = 0; // Current position in the data string
+    // Locate the start of the JSON array
+    char *array_start = strchr(data, '[');
+    if (!array_start)
     {
-        FuriString *json_data_array = get_json_array_value_furi("json_data", i, git_data_str);
-        if (!json_data_array)
-        {
-            break;
-        }
-        FuriString *type = get_json_value_furi("type", json_data_array);
-        if (!type)
-        {
-            FURI_LOG_E(TAG, "Failed to get type.");
-            furi_string_free(json_data_array);
-            break;
-        }
-        // skip directories for now
-        if (strcmp(furi_string_get_cstr(type), "file") != 0)
+        FURI_LOG_E(TAG, "Invalid JSON format: '[' not found.");
+        furi_string_free(git_data_str);
+        return false;
+    }
+    pos = array_start - data; // Update position to the start of the array
+    size_t brace_count = 0;
+    size_t obj_start = 0;
+    bool in_string = false; // To handle braces inside strings
+
+    while (pos < data_len && file_count < MAX_GITHUB_FILES)
+    {
+        char current = data[pos];
+
+        // Toggle in_string flag if a quote is found (handling escaped quotes)
+        if (current == '"' && (pos == 0 || data[pos - 1] != '\\'))
         {
-            furi_string_free(type);
-            furi_string_free(json_data_array);
-            continue;
+            in_string = !in_string;
         }
-        furi_string_free(type);
-        FuriString *download_url = get_json_value_furi("download_url", json_data_array);
-        if (!download_url)
-        {
-            FURI_LOG_E(TAG, "Failed to get download_url.");
 
-            furi_string_free(json_data_array);
-            break;
-        }
-        FuriString *name = get_json_value_furi("name", json_data_array);
-        if (!name)
+        if (!in_string)
         {
-            FURI_LOG_E(TAG, "Failed to get name.");
+            if (current == '{')
+            {
+                if (brace_count == 0)
+                {
+                    obj_start = pos; // Potential start of a JSON object
+                }
+                brace_count++;
+            }
+            else if (current == '}')
+            {
+                brace_count--;
+                if (brace_count == 0)
+                {
+                    size_t obj_end = pos;
+                    size_t obj_length = obj_end - obj_start + 1;
+                    // Extract the JSON object substring
+                    char *obj_str = malloc(obj_length + 1);
+                    if (!obj_str)
+                    {
+                        FURI_LOG_E(TAG, "Memory allocation failed for obj_str.");
+                        break;
+                    }
+                    strncpy(obj_str, data + obj_start, obj_length);
+                    obj_str[obj_length] = '\0'; // Null-terminate
 
-            furi_string_free(json_data_array);
-            furi_string_free(download_url);
-            break;
-        }
-        furi_string_free(json_data_array);
+                    FuriString *json_data_array = furi_string_alloc();
+                    furi_string_set(json_data_array, obj_str); // Set the string to the allocated memory
+                    free(obj_str);                             // Free the temporary C-string
 
-        // create json to save
-        FuriString *json = furi_string_alloc();
-        if (!json)
-        {
-            FURI_LOG_E(TAG, "Failed to allocate json.");
+                    if (!json_data_array)
+                    {
+                        FURI_LOG_E(TAG, "Failed to initialize json_data_array.");
+                        break;
+                    }
 
-            furi_string_free(download_url);
-            furi_string_free(name);
-            break;
-        }
+                    FURI_LOG_I(TAG, "Loaded json data array value %d. Available memory: %d bytes", file_count, memmgr_get_free_heap());
 
-        furi_string_cat_str(json, "{\"name\":\"");
-        furi_string_cat(json, name);
+                    // Extract "type" field
+                    FuriString *type = get_json_value_furi("type", json_data_array);
+                    if (!type)
+                    {
+                        FURI_LOG_E(TAG, "Failed to get type.");
+                        furi_string_free(json_data_array);
+                        break;
+                    }
 
-        furi_string_cat_str(json, "\",\"link\":\"");
-        furi_string_cat(json, download_url);
-        furi_string_free(download_url);
+                    // Skip non-file types (e.g., directories)
+                    if (strcmp(furi_string_get_cstr(type), "file") != 0)
+                    {
+                        furi_string_free(type);
+                        furi_string_free(json_data_array);
+                        pos = obj_end + 1; // Move past this object
+                        continue;
+                    }
+                    furi_string_free(type);
 
-        furi_string_cat_str(json, "\"}");
+                    // Extract "download_url" and "name"
+                    FuriString *download_url = get_json_value_furi("download_url", json_data_array);
+                    if (!download_url)
+                    {
+                        FURI_LOG_E(TAG, "Failed to get download_url.");
+                        furi_string_free(json_data_array);
+                        break;
+                    }
 
-        // save the json to the data folder: /ext/apps_data/flip_store/data/author/repo
-        char dir[256];
-        snprintf(dir, sizeof(dir), STORAGE_EXT_PATH_PREFIX "/apps_data/flip_store/data/%s/%s/file%d.json", author, repo, file_count);
-        if (!save_char_with_path(dir, furi_string_get_cstr(json)))
-        {
-            FURI_LOG_E(TAG, "Failed to save json.");
+                    FuriString *name = get_json_value_furi("name", json_data_array);
+                    if (!name)
+                    {
+                        FURI_LOG_E(TAG, "Failed to get name.");
+                        furi_string_free(json_data_array);
+                        furi_string_free(download_url);
+                        break;
+                    }
+
+                    furi_string_free(json_data_array);
+                    FURI_LOG_I(TAG, "Received name and download_url. Available memory: %d bytes", memmgr_get_free_heap());
+
+                    // Create JSON to save
+                    FuriString *json = furi_string_alloc();
+                    if (!json)
+                    {
+                        FURI_LOG_E(TAG, "Failed to allocate json.");
+                        furi_string_free(download_url);
+                        furi_string_free(name);
+                        break;
+                    }
+
+                    furi_string_cat_str(json, "{\"name\":\"");
+                    furi_string_cat(json, name);
+                    furi_string_cat_str(json, "\",\"link\":\"");
+                    furi_string_cat(json, download_url);
+                    furi_string_cat_str(json, "\"}");
+
+                    FURI_LOG_I(TAG, "Created json. Available memory: %d bytes", memmgr_get_free_heap());
+
+                    // Save the JSON to the data folder: /ext/apps_data/flip_store/data/author/repo/fileX.json
+                    snprintf(dir, sizeof(dir), STORAGE_EXT_PATH_PREFIX "/apps_data/flip_store/data/%s/%s/file%d.json", author, repo, file_count);
+                    if (!save_char_with_path(dir, furi_string_get_cstr(json)))
+                    {
+                        FURI_LOG_E(TAG, "Failed to save json.");
+                    }
+
+                    FURI_LOG_I(TAG, "Saved file %s.", furi_string_get_cstr(name));
+
+                    // Free allocated resources
+                    furi_string_free(name);
+                    furi_string_free(download_url);
+                    furi_string_free(json);
+
+                    file_count++;
+
+                    // This can be expensive for large strings; consider memory constraints
+                    size_t remaining_length = data_len - (obj_end + 1);
+                    memmove(data + obj_start, data + obj_end + 1, remaining_length + 1); // +1 to include null terminator
+                    data_len -= (obj_end + 1 - obj_start);
+                    pos = obj_start; // Reset position to the start of the modified string
+                    continue;
+                }
+            }
         }
 
-        // free the json
-        furi_string_free(name);
-        furi_string_free(json);
-        file_count++;
+        pos++;
     }
+
     furi_string_free(git_data_str);
-    // save file count
+
+    // Save file count
     char file_count_str[16];
     snprintf(file_count_str, sizeof(file_count_str), "%d", file_count);
-    char file_count_dir[256]; // /ext/apps_data/flip_store/data/author/file_count.txt
+    char file_count_dir[512]; // Increased size for longer paths
     snprintf(file_count_dir, sizeof(file_count_dir), STORAGE_EXT_PATH_PREFIX "/apps_data/flip_store/data/%s/file_count.txt", author);
+
+    FURI_LOG_I(TAG, "Successfully parsed %d files.", file_count);
     return save_char_with_path(file_count_dir, file_count_str);
 }
 
 bool flip_store_install_all_github_files(FlipperHTTP *fhttp, const char *author, const char *repo)
 {
-    if (!fhttp)
+    FURI_LOG_I(TAG, "Installing all Github files.");
+    if (!fhttp || !author || !repo)
     {
-        FURI_LOG_E(TAG, "FlipperHTTP is NULL");
+        FURI_LOG_E(TAG, "Invalid arguments.");
         return false;
     }
     // get the file count
@@ -199,17 +307,25 @@ bool flip_store_install_all_github_files(FlipperHTTP *fhttp, const char *author,
     int count = atoi(furi_string_get_cstr(file_count));
     furi_string_free(file_count);
 
+    bool parse()
+    {
+        return true;
+    }
+
     // install all files
+    char file_dir[256]; // /ext/apps_data/flip_store/data/author/repo/file.json
+    FURI_LOG_I(TAG, "Installing %d files.", count);
     for (int i = 0; i < count; i++)
     {
-        char file_dir[256]; // /ext/apps_data/flip_store/data/author/repo/file.json
         snprintf(file_dir, sizeof(file_dir), STORAGE_EXT_PATH_PREFIX "/apps_data/flip_store/data/%s/%s/file%d.json", author, repo, i);
+        FURI_LOG_I(TAG, "Loading file %s.", file_dir);
         FuriString *file = flipper_http_load_from_file(file_dir);
         if (file == NULL)
         {
             FURI_LOG_E(TAG, "Failed to load file.");
             return false;
         }
+        FURI_LOG_I(TAG, "Loaded file %s.", file_dir);
         FuriString *name = get_json_value_furi("name", file);
         if (!name)
         {
@@ -227,14 +343,21 @@ bool flip_store_install_all_github_files(FlipperHTTP *fhttp, const char *author,
         }
         furi_string_free(file);
 
-        // download the file
-        if (!flip_store_download_github_file(fhttp, furi_string_get_cstr(name), author, repo, furi_string_get_cstr(link)))
+        bool fetch_file()
+        {
+            return flip_store_download_github_file(fhttp, furi_string_get_cstr(name), author, repo, furi_string_get_cstr(link));
+        }
+
+        // download the file and wait until it is downloaded
+        FURI_LOG_I(TAG, "Downloading file %s.", furi_string_get_cstr(name));
+        if (!flipper_http_process_response_async(fhttp, fetch_file, parse))
         {
             FURI_LOG_E(TAG, "Failed to download file.");
             furi_string_free(name);
             furi_string_free(link);
             return false;
         }
+        FURI_LOG_I(TAG, "Downloaded file %s.", furi_string_get_cstr(name));
         furi_string_free(name);
         furi_string_free(link);
     }

jsmn/jsmn_furi.c

@@ -481,7 +481,7 @@ FuriString *get_json_value_furi(const char *key, const FuriString *json_data)
         return NULL;
     }
     uint32_t max_tokens = json_token_count_furi(json_data);
-    if (!jsmn_memory_check(max_tokens))
+    if (!jsmn_memory_check(sizeof(jsmntok_t) * max_tokens))
     {
         FURI_LOG_E("JSMM.H", "Insufficient memory for JSON tokens.");
         return NULL;
@@ -551,7 +551,7 @@ FuriString *get_json_array_value_furi(const char *key, uint32_t index, const Fur
         return NULL;
     }
     uint32_t max_tokens = json_token_count_furi(array_str);
-    if (!jsmn_memory_check(max_tokens))
+    if (!jsmn_memory_check(sizeof(jsmntok_t) * max_tokens))
     {
         FURI_LOG_E("JSMM.H", "Insufficient memory for JSON tokens.");
         furi_string_free(array_str);
@@ -633,7 +633,7 @@ FuriString **get_json_array_values_furi(const char *key, const FuriString *json_
     }
 
     uint32_t max_tokens = json_token_count_furi(array_str);
-    if (!jsmn_memory_check(max_tokens))
+    if (!jsmn_memory_check(sizeof(jsmntok_t) * max_tokens))
     {
         FURI_LOG_E("JSMM.H", "Insufficient memory for JSON tokens.");
         furi_string_free(array_str);