Web Crawler - v0.6

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6
+- Added a DOWNLOAD method, which downloads the file at the specified path. This is best used for downloading images, binary files, and other non-text files.
+- Increased buffer size to handle larger responses.
+- Fixed the variable item lists to display as intended.
+
 ## 0.5 (Cleanup)
 - Refactored using the easy flipper library
 - New python library for the FlipperHTTP flash

assets/FlipperHTTP/flipper_http.h

@@ -15,8 +15,8 @@
 #define UART_CH (FuriHalSerialIdUsart)    // UART channel
 #define TIMEOUT_DURATION_TICKS (2 * 1000) // 2 seconds
 #define BAUDRATE (115200)                 // UART baudrate
-#define RX_BUF_SIZE 128                   // UART RX buffer size
-#define RX_LINE_BUFFER_SIZE 2048          // UART RX line buffer size (increase for large responses)
+#define RX_BUF_SIZE 1024                  // UART RX buffer size
+#define RX_LINE_BUFFER_SIZE 3000          // UART RX line buffer size (increase for large responses)
 
 // Forward declaration for callback
 typedef void (*FlipperHTTP_Callback)(const char *line, void *context);
@@ -48,6 +48,9 @@ bool flipper_http_post_request_with_headers(const char *url, const char *headers
 bool flipper_http_put_request_with_headers(const char *url, const char *headers, const char *payload);
 bool flipper_http_delete_request_with_headers(const char *url, const char *headers, const char *payload);
 //---
+bool flipper_http_get_request_bytes(const char *url, const char *headers);
+bool flipper_http_post_request_bytes(const char *url, const char *headers, const char *payload);
+//
 bool flipper_http_save_received_data(size_t bytes_received, const char line_buffer[]);
 static char *trim(const char *str);
 
@@ -98,13 +101,175 @@ typedef struct
 
     bool started_receiving_delete; // Indicates if a DELETE request has started
     bool just_started_delete;      // Indicates if DELETE data reception has just started
+
+    // Buffer to hold the raw bytes received from the UART
+    uint8_t *received_bytes;
+    size_t received_bytes_len; // Length of the received bytes
+
+    // File path to save the bytes received
+    char file_path[256];
+
+    bool save_data; // Flag to save the received data
+
+    bool is_bytes_request; // trigger for bytes request
 } FlipperHTTP;
 
-FlipperHTTP fhttp;
+static FlipperHTTP fhttp;
+// Global static array for the line buffer
+static char rx_line_buffer[RX_LINE_BUFFER_SIZE];
+#define FILE_BUFFER_SIZE 512
+static uint8_t file_buffer[FILE_BUFFER_SIZE];
 
 // fhttp.received_data holds the received data from HTTP requests
 // fhttp.last_response holds the last received data from the UART, which could be [GET/END], [POST/END], [PUT/END], [DELETE/END], etc
 
+// Function to append received data to file
+// make sure to initialize the file path before calling this function
+static bool append_to_file(const char *file_path, const void *data, size_t data_size)
+{
+    Storage *storage = furi_record_open(RECORD_STORAGE);
+    File *file = storage_file_alloc(storage);
+
+    // Open the file in append mode
+    if (!storage_file_open(file, file_path, FSAM_WRITE, FSOM_OPEN_APPEND))
+    {
+        FURI_LOG_E(HTTP_TAG, "Failed to open file for appending: %s", file_path);
+        storage_file_free(file);
+        furi_record_close(RECORD_STORAGE);
+        return false;
+    }
+
+    // Write the data to the file
+    if (storage_file_write(file, data, data_size) != data_size)
+    {
+        FURI_LOG_E(HTTP_TAG, "Failed to append data to file");
+        storage_file_close(file);
+        storage_file_free(file);
+        furi_record_close(RECORD_STORAGE);
+        return false;
+    }
+
+    storage_file_close(file);
+    storage_file_free(file);
+    furi_record_close(RECORD_STORAGE);
+
+    return true;
+}
+
+// UART worker thread
+/**
+ * @brief      Worker thread to handle UART data asynchronously.
+ * @return     0
+ * @param      context   The context to pass to the callback.
+ * @note       This function will handle received data asynchronously via the callback.
+ */
+// UART worker thread
+static int32_t flipper_http_worker(void *context)
+{
+    UNUSED(context);
+    size_t rx_line_pos = 0;
+    static size_t file_buffer_len = 0;
+
+    while (1)
+    {
+        uint32_t events = furi_thread_flags_wait(WorkerEvtStop | WorkerEvtRxDone, FuriFlagWaitAny, FuriWaitForever);
+        if (events & WorkerEvtStop)
+            break;
+        if (events & WorkerEvtRxDone)
+        {
+            // Continuously read from the stream buffer until it's empty
+            while (!furi_stream_buffer_is_empty(fhttp.flipper_http_stream))
+            {
+                // Read one byte at a time
+                char c = 0;
+                size_t received = furi_stream_buffer_receive(fhttp.flipper_http_stream, &c, 1, 0);
+
+                if (received == 0)
+                {
+                    // No more data to read
+                    break;
+                }
+
+                // Append the received byte to the file if saving is enabled
+                if (fhttp.save_data)
+                {
+                    // Add byte to the buffer
+                    file_buffer[file_buffer_len++] = c;
+                    // Write to file if buffer is full
+                    if (file_buffer_len >= FILE_BUFFER_SIZE)
+                    {
+                        if (!append_to_file(fhttp.file_path, file_buffer, file_buffer_len))
+                        {
+                            FURI_LOG_E(HTTP_TAG, "Failed to append data to file");
+                        }
+                        file_buffer_len = 0;
+                    }
+                }
+
+                // Handle line buffering only if callback is set (text data)
+                if (fhttp.handle_rx_line_cb)
+                {
+                    // Handle line buffering
+                    if (c == '\n' || rx_line_pos >= RX_LINE_BUFFER_SIZE - 1)
+                    {
+                        rx_line_buffer[rx_line_pos] = '\0'; // Null-terminate the line
+
+                        // Invoke the callback with the complete line
+                        fhttp.handle_rx_line_cb(rx_line_buffer, fhttp.callback_context);
+
+                        // Reset the line buffer position
+                        rx_line_pos = 0;
+                    }
+                    else
+                    {
+                        rx_line_buffer[rx_line_pos++] = c; // Add character to the line buffer
+                    }
+                }
+            }
+        }
+    }
+
+    if (fhttp.save_data)
+    {
+        // Write the remaining data to the file
+        if (file_buffer_len > 0)
+        {
+            if (!append_to_file(fhttp.file_path, file_buffer, file_buffer_len))
+            {
+                FURI_LOG_E(HTTP_TAG, "Failed to append remaining data to file");
+            }
+        }
+    }
+
+    // remove [POST/END] and/or [GET/END] from the file
+    if (fhttp.save_data)
+    {
+        char *end = NULL;
+        if ((end = strstr(fhttp.file_path, "[POST/END]")) != NULL)
+        {
+            *end = '\0';
+        }
+        else if ((end = strstr(fhttp.file_path, "[GET/END]")) != NULL)
+        {
+            *end = '\0';
+        }
+    }
+
+    // remove newline from the from the end of the file
+    if (fhttp.save_data)
+    {
+        char *end = NULL;
+        if ((end = strstr(fhttp.file_path, "\n")) != NULL)
+        {
+            *end = '\0';
+        }
+    }
+
+    // Reset the file buffer length
+    file_buffer_len = 0;
+
+    return 0;
+}
 // Timer callback function
 /**
  * @brief      Callback function for the GET timeout timer.
@@ -154,62 +319,6 @@ static void _flipper_http_rx_callback(FuriHalSerialHandle *handle, FuriHalSerial
     }
 }
 
-// UART worker thread
-/**
- * @brief      Worker thread to handle UART data asynchronously.
- * @return     0
- * @param      context   The context to pass to the callback.
- * @note       This function will handle received data asynchronously via the callback.
- */
-static int32_t flipper_http_worker(void *context)
-{
-    UNUSED(context);
-    size_t rx_line_pos = 0;
-    char *rx_line_buffer = (char *)malloc(RX_LINE_BUFFER_SIZE);
-
-    if (!rx_line_buffer)
-    {
-        // Handle malloc failure
-        FURI_LOG_E(HTTP_TAG, "Failed to allocate memory for rx_line_buffer");
-        return -1;
-    }
-
-    while (1)
-    {
-        uint32_t events = furi_thread_flags_wait(WorkerEvtStop | WorkerEvtRxDone, FuriFlagWaitAny, FuriWaitForever);
-        if (events & WorkerEvtStop)
-            break;
-        if (events & WorkerEvtRxDone)
-        {
-            size_t len = furi_stream_buffer_receive(fhttp.flipper_http_stream, fhttp.rx_buf, RX_BUF_SIZE, 0);
-            for (size_t i = 0; i < len; i++)
-            {
-                char c = fhttp.rx_buf[i];
-                if (c == '\n' || rx_line_pos >= RX_LINE_BUFFER_SIZE - 1)
-                {
-                    rx_line_buffer[rx_line_pos] = '\0';
-                    // Invoke the callback with the complete line
-                    if (fhttp.handle_rx_line_cb)
-                    {
-                        fhttp.handle_rx_line_cb(rx_line_buffer, fhttp.callback_context);
-                    }
-                    // Reset the line buffer
-                    rx_line_pos = 0;
-                }
-                else
-                {
-                    rx_line_buffer[rx_line_pos++] = c;
-                }
-            }
-        }
-    }
-
-    // Free the allocated memory before exiting the thread
-    free(rx_line_buffer);
-
-    return 0;
-}
-
 // UART initialization function
 /**
  * @brief      Initialize UART.
@@ -757,6 +866,41 @@ bool flipper_http_get_request_with_headers(const char *url, const char *headers)
     // The response will be handled asynchronously via the callback
     return true;
 }
+// Function to send a GET request with headers and return bytes
+/**
+ * @brief      Send a GET request to the specified URL.
+ * @return     true if the request was successful, false otherwise.
+ * @param      url  The URL to send the GET request to.
+ * @param      headers  The headers to send with the GET request.
+ * @note       The received data will be handled asynchronously via the callback.
+ */
+bool flipper_http_get_request_bytes(const char *url, const char *headers)
+{
+    if (!url || !headers)
+    {
+        FURI_LOG_E("FlipperHTTP", "Invalid arguments provided to flipper_http_get_request_bytes.");
+        return false;
+    }
+
+    // Prepare GET request command with headers
+    char command[256];
+    int ret = snprintf(command, sizeof(command), "[GET/BYTES]{\"url\":\"%s\",\"headers\":%s}", url, headers);
+    if (ret < 0 || ret >= (int)sizeof(command))
+    {
+        FURI_LOG_E("FlipperHTTP", "Failed to format GET request command with headers.");
+        return false;
+    }
+
+    // Send GET request via UART
+    if (!flipper_http_send_data(command))
+    {
+        FURI_LOG_E("FlipperHTTP", "Failed to send GET request command with headers.");
+        return false;
+    }
+
+    // The response will be handled asynchronously via the callback
+    return true;
+}
 // Function to send a POST request with headers
 /**
  * @brief      Send a POST request to the specified URL.
@@ -793,6 +937,42 @@ bool flipper_http_post_request_with_headers(const char *url, const char *headers
     // The response will be handled asynchronously via the callback
     return true;
 }
+// Function to send a POST request with headers and return bytes
+/**
+ * @brief      Send a POST request to the specified URL.
+ * @return     true if the request was successful, false otherwise.
+ * @param      url  The URL to send the POST request to.
+ * @param      headers  The headers to send with the POST request.
+ * @param      payload  The data to send with the POST request.
+ * @note       The received data will be handled asynchronously via the callback.
+ */
+bool flipper_http_post_request_bytes(const char *url, const char *headers, const char *payload)
+{
+    if (!url || !headers || !payload)
+    {
+        FURI_LOG_E("FlipperHTTP", "Invalid arguments provided to flipper_http_post_request_bytes.");
+        return false;
+    }
+
+    // Prepare POST request command with headers and data
+    char command[256];
+    int ret = snprintf(command, sizeof(command), "[POST/BYTES]{\"url\":\"%s\",\"headers\":%s,\"payload\":%s}", url, headers, payload);
+    if (ret < 0 || ret >= (int)sizeof(command))
+    {
+        FURI_LOG_E("FlipperHTTP", "Failed to format POST request command with headers and data.");
+        return false;
+    }
+
+    // Send POST request via UART
+    if (!flipper_http_send_data(command))
+    {
+        FURI_LOG_E("FlipperHTTP", "Failed to send POST request command with headers and data.");
+        return false;
+    }
+
+    // The response will be handled asynchronously via the callback
+    return true;
+}
 // Function to send a PUT request with headers
 /**
  * @brief      Send a PUT request to the specified URL.
@@ -912,8 +1092,12 @@ void flipper_http_rx_callback(const char *line, void *context)
 
             if (fhttp.received_data)
             {
-                // uncomment if you want to save the received data to the external storage
-                // flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                /* uncomment if you want to save the received data to the external storage
+                if (!fhttp.is_bytes_request)
+                {
+                    flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                }
+                */
                 fhttp.started_receiving_get = false;
                 fhttp.just_started_get = false;
                 fhttp.state = IDLE;
@@ -974,8 +1158,12 @@ void flipper_http_rx_callback(const char *line, void *context)
 
             if (fhttp.received_data)
             {
-                // uncomment if you want to save the received data to the external storage
-                // flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                /* uncomment if you want to save the received data to the external storage
+                if (!fhttp.is_bytes_request)
+                {
+                    flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                }
+                */
                 fhttp.started_receiving_post = false;
                 fhttp.just_started_post = false;
                 fhttp.state = IDLE;
@@ -1167,6 +1355,9 @@ void flipper_http_rx_callback(const char *line, void *context)
         furi_timer_start(fhttp.get_timeout_timer, TIMEOUT_DURATION_TICKS);
         fhttp.state = RECEIVING;
         fhttp.received_data = NULL;
+
+        // for GET request, save data only if it's a bytes request
+        fhttp.save_data = fhttp.is_bytes_request;
         return;
     }
     else if (strstr(line, "[POST/SUCCESS]") != NULL)
@@ -1176,6 +1367,9 @@ void flipper_http_rx_callback(const char *line, void *context)
         furi_timer_start(fhttp.get_timeout_timer, TIMEOUT_DURATION_TICKS);
         fhttp.state = RECEIVING;
         fhttp.received_data = NULL;
+
+        // for POST request, save data only if it's a bytes request
+        fhttp.save_data = fhttp.is_bytes_request;
         return;
     }
     else if (strstr(line, "[PUT/SUCCESS]") != NULL)

flipper_http.h

@@ -15,8 +15,8 @@
 #define UART_CH (FuriHalSerialIdUsart)    // UART channel
 #define TIMEOUT_DURATION_TICKS (2 * 1000) // 2 seconds
 #define BAUDRATE (115200)                 // UART baudrate
-#define RX_BUF_SIZE 128                   // UART RX buffer size
-#define RX_LINE_BUFFER_SIZE 2048          // UART RX line buffer size (increase for large responses)
+#define RX_BUF_SIZE 1024                  // UART RX buffer size
+#define RX_LINE_BUFFER_SIZE 3000          // UART RX line buffer size (increase for large responses)
 
 // Forward declaration for callback
 typedef void (*FlipperHTTP_Callback)(const char *line, void *context);
@@ -47,6 +47,9 @@ bool flipper_http_post_request_with_headers(const char *url, const char *headers
 bool flipper_http_put_request_with_headers(const char *url, const char *headers, const char *payload);
 bool flipper_http_delete_request_with_headers(const char *url, const char *headers, const char *payload);
 //---
+bool flipper_http_get_request_bytes(const char *url, const char *headers);
+bool flipper_http_post_request_bytes(const char *url, const char *headers, const char *payload);
+//
 bool flipper_http_save_received_data(size_t bytes_received, const char line_buffer[]);
 static char *trim(const char *str);
 
@@ -97,13 +100,176 @@ typedef struct
 
     bool started_receiving_delete; // Indicates if a DELETE request has started
     bool just_started_delete;      // Indicates if DELETE data reception has just started
-} FlipperHTTP;
 
-FlipperHTTP fhttp;
+    // Buffer to hold the raw bytes received from the UART
+    uint8_t *received_bytes;
+    size_t received_bytes_len; // Length of the received bytes
+
+    // File path to save the bytes received
+    char file_path[256];
+
+    bool save_data; // Flag to save the received data
+
+    bool is_bytes_request; // trigger for bytes request
+} FlipperHTTP;
 
 // fhttp.received_data holds the received data from HTTP requests
 // fhttp.last_response holds the last received data from the UART, which could be [GET/END], [POST/END], [PUT/END], [DELETE/END], etc
 
+static FlipperHTTP fhttp;
+
+// Function to append received data to file
+// make sure to initialize the file path before calling this function
+static bool append_to_file(const char *file_path, const void *data, size_t data_size)
+{
+    Storage *storage = furi_record_open(RECORD_STORAGE);
+    File *file = storage_file_alloc(storage);
+
+    // Open the file in append mode
+    if (!storage_file_open(file, file_path, FSAM_WRITE, FSOM_OPEN_APPEND))
+    {
+        FURI_LOG_E(HTTP_TAG, "Failed to open file for appending: %s", file_path);
+        storage_file_free(file);
+        furi_record_close(RECORD_STORAGE);
+        return false;
+    }
+
+    // Write the data to the file
+    if (storage_file_write(file, data, data_size) != data_size)
+    {
+        FURI_LOG_E(HTTP_TAG, "Failed to append data to file");
+        storage_file_close(file);
+        storage_file_free(file);
+        furi_record_close(RECORD_STORAGE);
+        return false;
+    }
+
+    storage_file_close(file);
+    storage_file_free(file);
+    furi_record_close(RECORD_STORAGE);
+
+    return true;
+}
+// Global static array for the line buffer
+static char rx_line_buffer[RX_LINE_BUFFER_SIZE];
+#define FILE_BUFFER_SIZE 512
+static uint8_t file_buffer[FILE_BUFFER_SIZE];
+
+// UART worker thread
+/**
+ * @brief      Worker thread to handle UART data asynchronously.
+ * @return     0
+ * @param      context   The context to pass to the callback.
+ * @note       This function will handle received data asynchronously via the callback.
+ */
+// UART worker thread
+static int32_t flipper_http_worker(void *context)
+{
+    UNUSED(context);
+    size_t rx_line_pos = 0;
+    static size_t file_buffer_len = 0;
+
+    while (1)
+    {
+        uint32_t events = furi_thread_flags_wait(WorkerEvtStop | WorkerEvtRxDone, FuriFlagWaitAny, FuriWaitForever);
+        if (events & WorkerEvtStop)
+            break;
+        if (events & WorkerEvtRxDone)
+        {
+            // Continuously read from the stream buffer until it's empty
+            while (!furi_stream_buffer_is_empty(fhttp.flipper_http_stream))
+            {
+                // Read one byte at a time
+                char c = 0;
+                size_t received = furi_stream_buffer_receive(fhttp.flipper_http_stream, &c, 1, 0);
+
+                if (received == 0)
+                {
+                    // No more data to read
+                    break;
+                }
+
+                // Append the received byte to the file if saving is enabled
+                if (fhttp.save_data)
+                {
+                    // Add byte to the buffer
+                    file_buffer[file_buffer_len++] = c;
+                    // Write to file if buffer is full
+                    if (file_buffer_len >= FILE_BUFFER_SIZE)
+                    {
+                        if (!append_to_file(fhttp.file_path, file_buffer, file_buffer_len))
+                        {
+                            FURI_LOG_E(HTTP_TAG, "Failed to append data to file");
+                        }
+                        file_buffer_len = 0;
+                    }
+                }
+
+                // Handle line buffering only if callback is set (text data)
+                if (fhttp.handle_rx_line_cb)
+                {
+                    // Handle line buffering
+                    if (c == '\n' || rx_line_pos >= RX_LINE_BUFFER_SIZE - 1)
+                    {
+                        rx_line_buffer[rx_line_pos] = '\0'; // Null-terminate the line
+
+                        // Invoke the callback with the complete line
+                        fhttp.handle_rx_line_cb(rx_line_buffer, fhttp.callback_context);
+
+                        // Reset the line buffer position
+                        rx_line_pos = 0;
+                    }
+                    else
+                    {
+                        rx_line_buffer[rx_line_pos++] = c; // Add character to the line buffer
+                    }
+                }
+            }
+        }
+    }
+
+    if (fhttp.save_data)
+    {
+        // Write the remaining data to the file
+        if (file_buffer_len > 0)
+        {
+            if (!append_to_file(fhttp.file_path, file_buffer, file_buffer_len))
+            {
+                FURI_LOG_E(HTTP_TAG, "Failed to append remaining data to file");
+            }
+        }
+    }
+
+    // remove [POST/END] and/or [GET/END] from the file
+    if (fhttp.save_data)
+    {
+        char *end = NULL;
+        if ((end = strstr(fhttp.file_path, "[POST/END]")) != NULL)
+        {
+            *end = '\0';
+        }
+        else if ((end = strstr(fhttp.file_path, "[GET/END]")) != NULL)
+        {
+            *end = '\0';
+        }
+    }
+
+    // remove newline from the from the end of the file
+    if (fhttp.save_data)
+    {
+        char *end = NULL;
+        if ((end = strstr(fhttp.file_path, "\n")) != NULL)
+        {
+            *end = '\0';
+        }
+    }
+
+    // Reset the file buffer length
+    file_buffer_len = 0;
+
+    return 0;
+}
+
 // Timer callback function
 /**
  * @brief      Callback function for the GET timeout timer.
@@ -153,62 +319,6 @@ static void _flipper_http_rx_callback(FuriHalSerialHandle *handle, FuriHalSerial
     }
 }
 
-// UART worker thread
-/**
- * @brief      Worker thread to handle UART data asynchronously.
- * @return     0
- * @param      context   The context to pass to the callback.
- * @note       This function will handle received data asynchronously via the callback.
- */
-static int32_t flipper_http_worker(void *context)
-{
-    UNUSED(context);
-    size_t rx_line_pos = 0;
-    char *rx_line_buffer = (char *)malloc(RX_LINE_BUFFER_SIZE);
-
-    if (!rx_line_buffer)
-    {
-        // Handle malloc failure
-        FURI_LOG_E(HTTP_TAG, "Failed to allocate memory for rx_line_buffer");
-        return -1;
-    }
-
-    while (1)
-    {
-        uint32_t events = furi_thread_flags_wait(WorkerEvtStop | WorkerEvtRxDone, FuriFlagWaitAny, FuriWaitForever);
-        if (events & WorkerEvtStop)
-            break;
-        if (events & WorkerEvtRxDone)
-        {
-            size_t len = furi_stream_buffer_receive(fhttp.flipper_http_stream, fhttp.rx_buf, RX_BUF_SIZE, 0);
-            for (size_t i = 0; i < len; i++)
-            {
-                char c = fhttp.rx_buf[i];
-                if (c == '\n' || rx_line_pos >= RX_LINE_BUFFER_SIZE - 1)
-                {
-                    rx_line_buffer[rx_line_pos] = '\0';
-                    // Invoke the callback with the complete line
-                    if (fhttp.handle_rx_line_cb)
-                    {
-                        fhttp.handle_rx_line_cb(rx_line_buffer, fhttp.callback_context);
-                    }
-                    // Reset the line buffer
-                    rx_line_pos = 0;
-                }
-                else
-                {
-                    rx_line_buffer[rx_line_pos++] = c;
-                }
-            }
-        }
-    }
-
-    // Free the allocated memory before exiting the thread
-    free(rx_line_buffer);
-
-    return 0;
-}
-
 // UART initialization function
 /**
  * @brief      Initialize UART.
@@ -737,6 +847,41 @@ bool flipper_http_get_request_with_headers(const char *url, const char *headers)
     // The response will be handled asynchronously via the callback
     return true;
 }
+// Function to send a GET request with headers and return bytes
+/**
+ * @brief      Send a GET request to the specified URL.
+ * @return     true if the request was successful, false otherwise.
+ * @param      url  The URL to send the GET request to.
+ * @param      headers  The headers to send with the GET request.
+ * @note       The received data will be handled asynchronously via the callback.
+ */
+bool flipper_http_get_request_bytes(const char *url, const char *headers)
+{
+    if (!url || !headers)
+    {
+        FURI_LOG_E("FlipperHTTP", "Invalid arguments provided to flipper_http_get_request_bytes.");
+        return false;
+    }
+
+    // Prepare GET request command with headers
+    char command[256];
+    int ret = snprintf(command, sizeof(command), "[GET/BYTES]{\"url\":\"%s\",\"headers\":%s}", url, headers);
+    if (ret < 0 || ret >= (int)sizeof(command))
+    {
+        FURI_LOG_E("FlipperHTTP", "Failed to format GET request command with headers.");
+        return false;
+    }
+
+    // Send GET request via UART
+    if (!flipper_http_send_data(command))
+    {
+        FURI_LOG_E("FlipperHTTP", "Failed to send GET request command with headers.");
+        return false;
+    }
+
+    // The response will be handled asynchronously via the callback
+    return true;
+}
 // Function to send a POST request with headers
 /**
  * @brief      Send a POST request to the specified URL.
@@ -773,6 +918,42 @@ bool flipper_http_post_request_with_headers(const char *url, const char *headers
     // The response will be handled asynchronously via the callback
     return true;
 }
+// Function to send a POST request with headers and return bytes
+/**
+ * @brief      Send a POST request to the specified URL.
+ * @return     true if the request was successful, false otherwise.
+ * @param      url  The URL to send the POST request to.
+ * @param      headers  The headers to send with the POST request.
+ * @param      payload  The data to send with the POST request.
+ * @note       The received data will be handled asynchronously via the callback.
+ */
+bool flipper_http_post_request_bytes(const char *url, const char *headers, const char *payload)
+{
+    if (!url || !headers || !payload)
+    {
+        FURI_LOG_E("FlipperHTTP", "Invalid arguments provided to flipper_http_post_request_bytes.");
+        return false;
+    }
+
+    // Prepare POST request command with headers and data
+    char command[256];
+    int ret = snprintf(command, sizeof(command), "[POST/BYTES]{\"url\":\"%s\",\"headers\":%s,\"payload\":%s}", url, headers, payload);
+    if (ret < 0 || ret >= (int)sizeof(command))
+    {
+        FURI_LOG_E("FlipperHTTP", "Failed to format POST request command with headers and data.");
+        return false;
+    }
+
+    // Send POST request via UART
+    if (!flipper_http_send_data(command))
+    {
+        FURI_LOG_E("FlipperHTTP", "Failed to send POST request command with headers and data.");
+        return false;
+    }
+
+    // The response will be handled asynchronously via the callback
+    return true;
+}
 // Function to send a PUT request with headers
 /**
  * @brief      Send a PUT request to the specified URL.
@@ -876,7 +1057,7 @@ void flipper_http_rx_callback(const char *line, void *context)
     }
 
     // Uncomment below line to log the data received over UART
-    // FURI_LOG_I(HTTP_TAG, "Received UART line: %s", line);
+    FURI_LOG_I(HTTP_TAG, "Received UART line: %s", line);
 
     // Check if we've started receiving data from a GET request
     if (fhttp.started_receiving_get)
@@ -892,8 +1073,10 @@ void flipper_http_rx_callback(const char *line, void *context)
 
             if (fhttp.received_data)
             {
-                // uncomment if you want to save the received data to the external storage
-                // flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                if (!fhttp.is_bytes_request)
+                {
+                    flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                }
                 fhttp.started_receiving_get = false;
                 fhttp.just_started_get = false;
                 fhttp.state = IDLE;
@@ -954,8 +1137,10 @@ void flipper_http_rx_callback(const char *line, void *context)
 
             if (fhttp.received_data)
             {
-                // uncomment if you want to save the received data to the external storage
-                // flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                if (!fhttp.is_bytes_request)
+                {
+                    flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                }
                 fhttp.started_receiving_post = false;
                 fhttp.just_started_post = false;
                 fhttp.state = IDLE;
@@ -1017,7 +1202,7 @@ void flipper_http_rx_callback(const char *line, void *context)
             if (fhttp.received_data)
             {
                 // uncomment if you want to save the received data to the external storage
-                // flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
                 fhttp.started_receiving_put = false;
                 fhttp.just_started_put = false;
                 fhttp.state = IDLE;
@@ -1079,7 +1264,7 @@ void flipper_http_rx_callback(const char *line, void *context)
             if (fhttp.received_data)
             {
                 // uncomment if you want to save the received data to the external storage
-                // flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
+                flipper_http_save_received_data(strlen(fhttp.received_data), fhttp.received_data);
                 fhttp.started_receiving_delete = false;
                 fhttp.just_started_delete = false;
                 fhttp.state = IDLE;
@@ -1147,6 +1332,9 @@ void flipper_http_rx_callback(const char *line, void *context)
         furi_timer_start(fhttp.get_timeout_timer, TIMEOUT_DURATION_TICKS);
         fhttp.state = RECEIVING;
         fhttp.received_data = NULL;
+
+        // for GET request, save data only if it's a bytes request
+        fhttp.save_data = fhttp.is_bytes_request;
         return;
     }
     else if (strstr(line, "[POST/SUCCESS]") != NULL)
@@ -1156,6 +1344,9 @@ void flipper_http_rx_callback(const char *line, void *context)
         furi_timer_start(fhttp.get_timeout_timer, TIMEOUT_DURATION_TICKS);
         fhttp.state = RECEIVING;
         fhttp.received_data = NULL;
+
+        // for POST request, save data only if it's a bytes request
+        fhttp.save_data = fhttp.is_bytes_request;
         return;
     }
     else if (strstr(line, "[PUT/SUCCESS]") != NULL)

web_crawler_callback.h

@@ -74,6 +74,8 @@ static void web_crawler_view_draw_callback(Canvas *canvas, void *context)
             {
                 canvas_draw_str(canvas, 0, 10, "Sending GET request...");
 
+                fhttp.is_bytes_request = false;
+
                 // Perform GET request and handle the response
                 if (app_instance->headers == NULL || app_instance->headers[0] == '\0' || strstr(app_instance->headers, " ") == NULL)
                 {
@@ -88,6 +90,8 @@ static void web_crawler_view_draw_callback(Canvas *canvas, void *context)
             {
                 canvas_draw_str(canvas, 0, 10, "Sending POST request...");
 
+                fhttp.is_bytes_request = false;
+
                 // Perform POST request and handle the response
                 get_success = flipper_http_post_request_with_headers(app_instance->path, app_instance->headers, app_instance->payload);
             }
@@ -95,16 +99,30 @@ static void web_crawler_view_draw_callback(Canvas *canvas, void *context)
             {
                 canvas_draw_str(canvas, 0, 10, "Sending PUT request...");
 
+                fhttp.is_bytes_request = false;
+
                 // Perform PUT request and handle the response
                 get_success = flipper_http_put_request_with_headers(app_instance->path, app_instance->headers, app_instance->payload);
             }
-            else
+            else if (strstr(app_instance->http_method, "DELETE") != NULL)
             {
                 canvas_draw_str(canvas, 0, 10, "Sending DELETE request...");
 
+                fhttp.is_bytes_request = false;
+
                 // Perform DELETE request and handle the response
                 get_success = flipper_http_delete_request_with_headers(app_instance->path, app_instance->headers, app_instance->payload);
             }
+            else
+            {
+                // download file
+                canvas_draw_str(canvas, 0, 10, "Downloading file...");
+
+                fhttp.is_bytes_request = true;
+
+                // Perform GET request and handle the response
+                get_success = flipper_http_get_request_bytes(app_instance->path, app_instance->headers);
+            }
 
             canvas_draw_str(canvas, 0, 20, "Sent!");
 
@@ -149,6 +167,12 @@ static void web_crawler_view_draw_callback(Canvas *canvas, void *context)
                         canvas_draw_str(canvas, 0, 50, "Update your WiFi settings.");
                         canvas_draw_str(canvas, 0, 60, "Press BACK to return.");
                     }
+                    else if (strstr(fhttp.last_response, "[ERROR] GET request failed with error: connection refused") != NULL)
+                    {
+                        canvas_draw_str(canvas, 0, 10, "[ERROR] Connection refused.");
+                        canvas_draw_str(canvas, 0, 50, "Choose another URL.");
+                        canvas_draw_str(canvas, 0, 60, "Press BACK to return.");
+                    }
                     else
                     {
                         canvas_draw_str(canvas, 0, 10, "[ERROR] Failed to sync data.");
@@ -562,6 +586,16 @@ static void web_crawler_set_file_type_update(void *context)
 
     rename_received_data(app->file_rename, app->file_rename, app->file_type, old_file_type);
 
+    // set the file path for fhttp.file_path
+    if (app->file_rename && app->file_type)
+    {
+        char file_path[256];
+        snprintf(file_path, sizeof(file_path), "%s%s%s", RECEIVED_DATA_PATH, app->file_rename, app->file_type);
+        file_path[sizeof(file_path) - 1] = '\0'; // Null-terminate
+        strncpy(fhttp.file_path, file_path, sizeof(fhttp.file_path) - 1);
+        fhttp.file_path[sizeof(fhttp.file_path) - 1] = '\0'; // Null-terminate
+    }
+
     // Return to the Configure view
     view_dispatcher_switch_to_view(app->view_dispatcher, WebCrawlerViewVariableItemListFile);
 }
@@ -604,6 +638,16 @@ static void web_crawler_set_file_rename_update(void *context)
 
     rename_received_data(old_name, app->file_rename, app->file_type, app->file_type);
 
+    // set the file path for fhttp.file_path
+    if (app->file_rename && app->file_type)
+    {
+        char file_path[256];
+        snprintf(file_path, sizeof(file_path), "%s%s%s", RECEIVED_DATA_PATH, app->file_rename, app->file_type);
+        file_path[sizeof(file_path) - 1] = '\0'; // Null-terminate
+        strncpy(fhttp.file_path, file_path, sizeof(fhttp.file_path) - 1);
+        fhttp.file_path[sizeof(fhttp.file_path) - 1] = '\0'; // Null-terminate
+    }
+
     // Return to the Configure view
     view_dispatcher_switch_to_view(app->view_dispatcher, WebCrawlerViewVariableItemListFile);
 }

web_crawler_e.h

@@ -6,7 +6,7 @@
 #include <storage/storage.h>
 
 #define TAG "WebCrawler"
-static char *http_method_names[] = {"GET", "POST", "PUT", "DELETE"};
+static char *http_method_names[] = {"GET", "POST", "PUT", "DELETE", "DOWNLOAD"};
 
 // Define the submenu items for our WebCrawler application
 typedef enum
@@ -32,7 +32,7 @@ typedef enum
     WebCrawlerViewTextInput,               // Text input for Path
     WebCrawlerViewTextInputSSID,           // Text input for SSID
     WebCrawlerViewTextInputPassword,       // Text input for Password
-    WebCrawlerViewFileRead,                // Text input for File Read
+    WebCrawlerViewFileRead,                // File Read
     WebCrawlerViewTextInputFileType,       // Text input for File Type
     WebCrawlerViewTextInputFileRename,     // Text input for File Rename
     WebCrawlerViewTextInputHeaders,        // Text input for Headers

web_crawler_i.h

@@ -32,7 +32,7 @@ WebCrawlerApp *web_crawler_app_alloc()
     app->temp_buffer_size_password = 64;
     app->temp_buffer_size_file_type = 16;
     app->temp_buffer_size_file_rename = 128;
-    app->temp_buffer_size_http_method = 8;
+    app->temp_buffer_size_http_method = 16;
     app->temp_buffer_size_headers = 256;
     app->temp_buffer_size_payload = 256;
     if (!easy_flipper_set_buffer(&app->temp_buffer_path, app->temp_buffer_size_path))
@@ -146,7 +146,7 @@ WebCrawlerApp *web_crawler_app_alloc()
 
     // set variable items
     app->path_item = variable_item_list_add(app->variable_item_list_request, "Path", 0, NULL, NULL);
-    app->http_method_item = variable_item_list_add(app->variable_item_list_request, "HTTP Method", 4, web_crawler_http_method_change, app);
+    app->http_method_item = variable_item_list_add(app->variable_item_list_request, "HTTP Method", 5, web_crawler_http_method_change, app);
     app->headers_item = variable_item_list_add(app->variable_item_list_request, "Headers", 0, NULL, NULL);
     app->payload_item = variable_item_list_add(app->variable_item_list_request, "Payload", 0, NULL, NULL);
     //
@@ -176,7 +176,7 @@ WebCrawlerApp *web_crawler_app_alloc()
     variable_item_set_current_value_text(app->file_delete_item, ""); // Initialize
 
     // Allocate Submenu views
-    if (!easy_flipper_set_submenu(&app->submenu_main, WebCrawlerViewSubmenuMain, "Web Crawler v0.5", web_crawler_exit_app_callback, &app->view_dispatcher))
+    if (!easy_flipper_set_submenu(&app->submenu_main, WebCrawlerViewSubmenuMain, "Web Crawler v0.6", web_crawler_exit_app_callback, &app->view_dispatcher))
     {
         return NULL;
     }
@@ -243,16 +243,16 @@ WebCrawlerApp *web_crawler_app_alloc()
     else
     {
         // Update the configuration items based on loaded settings
-        if (app->path[0] != '\0')
+        if (app->path_item)
         {
             variable_item_set_current_value_text(app->path_item, app->path);
         }
         else
         {
-            variable_item_set_current_value_text(app->path_item, ""); // Initialize
+            variable_item_set_current_value_text(app->path_item, "https://httpbin.org/get"); // Initialize
         }
 
-        if (app->ssid[0] != '\0')
+        if (app->ssid_item)
         {
             variable_item_set_current_value_text(app->ssid_item, app->ssid);
         }
@@ -261,7 +261,7 @@ WebCrawlerApp *web_crawler_app_alloc()
             variable_item_set_current_value_text(app->ssid_item, ""); // Initialize
         }
 
-        if (app->file_type[0] != '\0')
+        if (app->file_type_item)
         {
             variable_item_set_current_value_text(app->file_type_item, app->file_type);
         }
@@ -270,7 +270,7 @@ WebCrawlerApp *web_crawler_app_alloc()
             variable_item_set_current_value_text(app->file_type_item, ".txt"); // Initialize
         }
 
-        if (app->file_rename[0] != '\0')
+        if (app->file_rename_item)
         {
             variable_item_set_current_value_text(app->file_rename_item, app->file_rename);
         }
@@ -279,7 +279,7 @@ WebCrawlerApp *web_crawler_app_alloc()
             variable_item_set_current_value_text(app->file_rename_item, "received_data"); // Initialize
         }
 
-        if (app->http_method[0] != '\0')
+        if (app->http_method_item)
         {
             variable_item_set_current_value_text(app->http_method_item, app->http_method);
         }
@@ -288,7 +288,7 @@ WebCrawlerApp *web_crawler_app_alloc()
             variable_item_set_current_value_text(app->http_method_item, "GET"); // Initialize
         }
 
-        if (app->headers[0] != '\0')
+        if (app->headers_item)
         {
             variable_item_set_current_value_text(app->headers_item, app->headers);
         }
@@ -297,7 +297,7 @@ WebCrawlerApp *web_crawler_app_alloc()
             variable_item_set_current_value_text(app->headers_item, "{\n\t\"Content-Type\": \"application/json\"\n}"); // Initialize
         }
 
-        if (app->payload[0] != '\0')
+        if (app->payload_item)
         {
             variable_item_set_current_value_text(app->payload_item, app->payload);
         }
@@ -306,6 +306,27 @@ WebCrawlerApp *web_crawler_app_alloc()
             variable_item_set_current_value_text(app->payload_item, "{\n\t\"key\": \"value\"\n}"); // Initialize
         }
 
+        // set the file path for fhttp.file_path
+        char file_path[128];
+        snprintf(file_path, sizeof(file_path), "%s%s%s", RECEIVED_DATA_PATH, app->file_rename, app->file_type);
+        snprintf(fhttp.file_path, sizeof(fhttp.file_path), "%s", file_path);
+
+        // update temp buffers
+        strncpy(app->temp_buffer_path, app->path, app->temp_buffer_size_path - 1);
+        app->temp_buffer_path[app->temp_buffer_size_path - 1] = '\0';
+        strncpy(app->temp_buffer_ssid, app->ssid, app->temp_buffer_size_ssid - 1);
+        app->temp_buffer_ssid[app->temp_buffer_size_ssid - 1] = '\0';
+        strncpy(app->temp_buffer_file_type, app->file_type, app->temp_buffer_size_file_type - 1);
+        app->temp_buffer_file_type[app->temp_buffer_size_file_type - 1] = '\0';
+        strncpy(app->temp_buffer_file_rename, app->file_rename, app->temp_buffer_size_file_rename - 1);
+        app->temp_buffer_file_rename[app->temp_buffer_size_file_rename - 1] = '\0';
+        strncpy(app->temp_buffer_http_method, app->http_method, app->temp_buffer_size_http_method - 1);
+        app->temp_buffer_http_method[app->temp_buffer_size_http_method - 1] = '\0';
+        strncpy(app->temp_buffer_headers, app->headers, app->temp_buffer_size_headers - 1);
+        app->temp_buffer_headers[app->temp_buffer_size_headers - 1] = '\0';
+        strncpy(app->temp_buffer_payload, app->payload, app->temp_buffer_size_payload - 1);
+        app->temp_buffer_payload[app->temp_buffer_size_payload - 1] = '\0';
+
         // Password handling can be omitted for security or handled securely
     }