[FL-2831] Resources cleanup in updater (#1796)

* updater: remove files from existing resources Manifest file before deploying new resources
* toolbox: tar: single file extraction API

Co-authored-by: あく <alleteam@gmail.com>

applications/system/updater/util/update_task.h

@@ -10,7 +10,7 @@ extern "C" {
 #include <stdbool.h>
 #include <m-string.h>
 
-#define UPDATE_DELAY_OPERATION_OK 300
+#define UPDATE_DELAY_OPERATION_OK 10
 #define UPDATE_DELAY_OPERATION_ERROR INT_MAX
 
 typedef enum {

applications/system/updater/util/update_task_worker_backup.c

@@ -9,6 +9,7 @@
 #include <update_util/dfu_file.h>
 #include <update_util/lfs_backup.h>
 #include <update_util/update_operation.h>
+#include <update_util/resources/manifest.h>
 #include <toolbox/tar/tar_archive.h>
 #include <toolbox/crc32_calc.h>
 
@@ -50,10 +51,46 @@ static bool update_task_resource_unpack_cb(const char* name, bool is_directory,
     update_task_set_progress(
         unpack_progress->update_task,
         UpdateTaskStageProgress,
-        unpack_progress->processed_files * 100 / (unpack_progress->total_files + 1));
+        /* For this stage, last 70% of progress = extraction */
+        30 + (unpack_progress->processed_files * 70) / (unpack_progress->total_files + 1));
     return true;
 }
 
+static void
+    update_task_cleanup_resources(UpdateTask* update_task, uint32_t n_approx_file_entries) {
+    ResourceManifestReader* manifest_reader = resource_manifest_reader_alloc(update_task->storage);
+    do {
+        FURI_LOG_I(TAG, "Cleaning up old manifest");
+        if(!resource_manifest_reader_open(manifest_reader, EXT_PATH("Manifest"))) {
+            FURI_LOG_W(TAG, "No existing manifest");
+            break;
+        }
+
+        /* We got # of entries in TAR file. Approx 1/4th is dir entries, we skip them */
+        n_approx_file_entries = n_approx_file_entries * 3 / 4 + 1;
+        uint32_t n_processed_files = 0;
+
+        ResourceManifestEntry* entry_ptr = NULL;
+        while((entry_ptr = resource_manifest_reader_next(manifest_reader))) {
+            if(entry_ptr->type == ResourceManifestEntryTypeFile) {
+                update_task_set_progress(
+                    update_task,
+                    UpdateTaskStageProgress,
+                    /* For this stage, first 30% of progress = cleanup */
+                    (n_processed_files++ * 30) / (n_approx_file_entries + 1));
+
+                string_t file_path;
+                string_init(file_path);
+                path_concat(STORAGE_EXT_PATH_PREFIX, string_get_cstr(entry_ptr->name), file_path);
+                FURI_LOG_D(TAG, "Removing %s", string_get_cstr(file_path));
+                storage_simply_remove(update_task->storage, string_get_cstr(file_path));
+                string_clear(file_path);
+            }
+        }
+    } while(false);
+    resource_manifest_reader_free(manifest_reader);
+}
+
 static bool update_task_post_update(UpdateTask* update_task) {
     bool success = false;
 
@@ -88,6 +125,8 @@ static bool update_task_post_update(UpdateTask* update_task) {
 
             progress.total_files = tar_archive_get_entries_count(archive);
             if(progress.total_files > 0) {
+                update_task_cleanup_resources(update_task, progress.total_files);
+
                 CHECK_RESULT(tar_archive_unpack_to(archive, STORAGE_EXT_PATH_PREFIX, NULL));
             }
         }

applications/system/updater/util/update_task_worker_flasher.c

@@ -308,7 +308,7 @@ bool update_task_validate_optionbytes(UpdateTask* update_task) {
                 }
             }
         } else {
-            FURI_LOG_I(
+            FURI_LOG_D(
                 TAG,
                 "OB MATCH: #%d: real %08X == %08X (exp.)",
                 idx,

firmware/targets/f7/api_symbols.csv

@@ -2269,6 +2269,7 @@ Function,+,tar_archive_get_entries_count,int32_t,TarArchive*
 Function,+,tar_archive_open,_Bool,"TarArchive*, const char*, TarOpenMode"
 Function,+,tar_archive_set_file_callback,void,"TarArchive*, tar_unpack_file_cb, void*"
 Function,+,tar_archive_store_data,_Bool,"TarArchive*, const char*, const uint8_t*, const int32_t"
+Function,+,tar_archive_unpack_file,_Bool,"TarArchive*, const char*, const char*"
 Function,+,tar_archive_unpack_to,_Bool,"TarArchive*, const char*, Storage_name_converter"
 Function,-,tempnam,char*,"const char*, const char*"
 Function,+,text_box_alloc,TextBox*,

firmware/targets/f7/furi_hal/furi_hal_cortex.c

@@ -6,6 +6,9 @@ void furi_hal_cortex_init_early() {
     CoreDebug->DEMCR |= CoreDebug_DEMCR_TRCENA_Msk;
     DWT->CTRL |= DWT_CTRL_CYCCNTENA_Msk;
     DWT->CYCCNT = 0U;
+
+    /* Enable instruction prefetch */
+    SET_BIT(FLASH->ACR, FLASH_ACR_PRFTEN);
 }
 
 void furi_hal_cortex_delay_us(uint32_t microseconds) {

lib/toolbox/tar/tar_archive.c

@@ -168,7 +168,44 @@ typedef struct {
     Storage_name_converter converter;
 } TarArchiveDirectoryOpParams;
 
+static bool archive_extract_current_file(TarArchive* archive, const char* dst_path) {
+    mtar_t* tar = &archive->tar;
+    File* out_file = storage_file_alloc(archive->storage);
+    uint8_t* readbuf = malloc(FILE_BLOCK_SIZE);
+
+    bool success = true;
+    uint8_t n_tries = FILE_OPEN_NTRIES;
+    do {
+        while(n_tries-- > 0) {
+            if(storage_file_open(out_file, dst_path, FSAM_WRITE, FSOM_CREATE_ALWAYS)) {
+                break;
+            }
+            FURI_LOG_W(TAG, "Failed to open '%s', reties: %d", dst_path, n_tries);
+            storage_file_close(out_file);
+            furi_delay_ms(FILE_OPEN_RETRY_DELAY);
+        }
+
+        if(!storage_file_is_open(out_file)) {
+            success = false;
+            break;
+        }
+
+        while(!mtar_eof_data(tar)) {
+            int32_t readcnt = mtar_read_data(tar, readbuf, FILE_BLOCK_SIZE);
+            if(!readcnt || !storage_file_write(out_file, readbuf, readcnt)) {
+                success = false;
+                break;
+            }
+        }
+    } while(false);
+    storage_file_free(out_file);
+    free(readbuf);
+
+    return success;
+}
+
 static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, void* param) {
+    UNUSED(tar);
     TarArchiveDirectoryOpParams* op_params = param;
     TarArchive* archive = op_params->archive;
 
@@ -199,58 +236,22 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header,
         return 0;
     }
 
-    string_init(full_extracted_fname);
+    FURI_LOG_D(TAG, "Extracting %d bytes to '%s'", header->size, header->name);
 
     string_t converted_fname;
     string_init_set(converted_fname, header->name);
     if(op_params->converter) {
         op_params->converter(converted_fname);
     }
-    path_concat(op_params->work_dir, string_get_cstr(converted_fname), full_extracted_fname);
-    string_clear(converted_fname);
 
-    FURI_LOG_D(TAG, "Extracting %d bytes to '%s'", header->size, header->name);
-    File* out_file = storage_file_alloc(archive->storage);
-    uint8_t* readbuf = malloc(FILE_BLOCK_SIZE);
-
-    bool failed = false;
-    uint8_t n_tries = FILE_OPEN_NTRIES;
-    do {
-        while(n_tries-- > 0) {
-            if(storage_file_open(
-                   out_file,
-                   string_get_cstr(full_extracted_fname),
-                   FSAM_WRITE,
-                   FSOM_CREATE_ALWAYS)) {
-                break;
-            }
-            FURI_LOG_W(
-                TAG,
-                "Failed to open '%s', reties: %d",
-                string_get_cstr(full_extracted_fname),
-                n_tries);
-            storage_file_close(out_file);
-            furi_delay_ms(FILE_OPEN_RETRY_DELAY);
-        }
-
-        if(!storage_file_is_open(out_file)) {
-            failed = true;
-            break;
-        }
+    string_init(full_extracted_fname);
+    path_concat(op_params->work_dir, string_get_cstr(converted_fname), full_extracted_fname);
 
-        while(!mtar_eof_data(tar)) {
-            int32_t readcnt = mtar_read_data(tar, readbuf, FILE_BLOCK_SIZE);
-            if(!readcnt || !storage_file_write(out_file, readbuf, readcnt)) {
-                failed = true;
-                break;
-            }
-        }
-    } while(false);
+    bool success = archive_extract_current_file(archive, string_get_cstr(full_extracted_fname));
 
-    storage_file_free(out_file);
-    free(readbuf);
+    string_clear(converted_fname);
     string_clear(full_extracted_fname);
-    return failed ? -1 : 0;
+    return success ? 0 : -1;
 }
 
 bool tar_archive_unpack_to(
@@ -369,3 +370,16 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch
     storage_file_free(directory);
     return success;
 }
+
+bool tar_archive_unpack_file(
+    TarArchive* archive,
+    const char* archive_fname,
+    const char* destination) {
+    furi_assert(archive);
+    furi_assert(archive_fname);
+    furi_assert(destination);
+    if(mtar_find(&archive->tar, archive_fname) != MTAR_ESUCCESS) {
+        return false;
+    }
+    return archive_extract_current_file(archive, destination);
+}

lib/toolbox/tar/tar_archive.h

@@ -41,6 +41,11 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch
 
 int32_t tar_archive_get_entries_count(TarArchive* archive);
 
+bool tar_archive_unpack_file(
+    TarArchive* archive,
+    const char* archive_fname,
+    const char* destination);
+
 /* Optional per-entry callback on unpacking - return false to skip entry */
 typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* context);
 

lib/update_util/resources/manifest.c

@@ -0,0 +1,115 @@
+#include "manifest.h"
+
+#include <toolbox/stream/buffered_file_stream.h>
+#include <toolbox/hex.h>
+
+struct ResourceManifestReader {
+    Storage* storage;
+    Stream* stream;
+    string_t linebuf;
+    ResourceManifestEntry entry;
+};
+
+ResourceManifestReader* resource_manifest_reader_alloc(Storage* storage) {
+    ResourceManifestReader* resource_manifest =
+        (ResourceManifestReader*)malloc(sizeof(ResourceManifestReader));
+    resource_manifest->storage = storage;
+    resource_manifest->stream = buffered_file_stream_alloc(resource_manifest->storage);
+    memset(&resource_manifest->entry, 0, sizeof(ResourceManifestEntry));
+    string_init(resource_manifest->entry.name);
+    string_init(resource_manifest->linebuf);
+    return resource_manifest;
+}
+
+void resource_manifest_reader_free(ResourceManifestReader* resource_manifest) {
+    furi_assert(resource_manifest);
+
+    string_clear(resource_manifest->linebuf);
+    string_clear(resource_manifest->entry.name);
+    buffered_file_stream_close(resource_manifest->stream);
+    stream_free(resource_manifest->stream);
+    free(resource_manifest);
+}
+
+bool resource_manifest_reader_open(ResourceManifestReader* resource_manifest, const char* filename) {
+    furi_assert(resource_manifest);
+
+    return buffered_file_stream_open(
+        resource_manifest->stream, filename, FSAM_READ, FSOM_OPEN_EXISTING);
+}
+
+/* Read entries in format of
+ * F:<hash>:<size>:<name>
+ * D:<name> 
+ */
+ResourceManifestEntry* resource_manifest_reader_next(ResourceManifestReader* resource_manifest) {
+    furi_assert(resource_manifest);
+
+    string_reset(resource_manifest->entry.name);
+    resource_manifest->entry.type = ResourceManifestEntryTypeUnknown;
+    resource_manifest->entry.size = 0;
+    memset(resource_manifest->entry.hash, 0, sizeof(resource_manifest->entry.hash));
+
+    do {
+        if(!stream_read_line(resource_manifest->stream, resource_manifest->linebuf)) {
+            return NULL;
+        }
+
+        /* Trim end of line */
+        string_strim(resource_manifest->linebuf);
+
+        char type_code = string_get_char(resource_manifest->linebuf, 0);
+        switch(type_code) {
+        case 'F':
+            resource_manifest->entry.type = ResourceManifestEntryTypeFile;
+            break;
+        case 'D':
+            resource_manifest->entry.type = ResourceManifestEntryTypeDirectory;
+            break;
+        default: /* Skip other entries - version, timestamp, etc */
+            continue;
+        };
+
+        if(resource_manifest->entry.type == ResourceManifestEntryTypeFile) {
+            /* Parse file entry
+              F:<hash>:<size>:<name> */
+
+            /* Remove entry type code */
+            string_right(resource_manifest->linebuf, 2);
+
+            if(string_search_char(resource_manifest->linebuf, ':') !=
+               sizeof(resource_manifest->entry.hash) * 2) {
+                /* Invalid hash */
+                continue;
+            }
+
+            /* Read hash */
+            hex_chars_to_uint8(
+                string_get_cstr(resource_manifest->linebuf), resource_manifest->entry.hash);
+
+            /* Remove hash */
+            string_right(
+                resource_manifest->linebuf, sizeof(resource_manifest->entry.hash) * 2 + 1);
+
+            resource_manifest->entry.size = atoi(string_get_cstr(resource_manifest->linebuf));
+
+            /* Remove size */
+            size_t offs = string_search_char(resource_manifest->linebuf, ':');
+            string_right(resource_manifest->linebuf, offs + 1);
+
+            string_set(resource_manifest->entry.name, resource_manifest->linebuf);
+        } else if(resource_manifest->entry.type == ResourceManifestEntryTypeDirectory) {
+            /* Parse directory entry
+               D:<name> */
+
+            /* Remove entry type code */
+            string_right(resource_manifest->linebuf, 2);
+
+            string_set(resource_manifest->entry.name, resource_manifest->linebuf);
+        }
+
+        return &resource_manifest->entry;
+    } while(true);
+
+    return NULL;
+}

lib/update_util/resources/manifest.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <storage/storage.h>
+
+#include <m-string.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum {
+    ResourceManifestEntryTypeUnknown = 0,
+    ResourceManifestEntryTypeDirectory,
+    ResourceManifestEntryTypeFile,
+} ResourceManifestEntryType;
+
+typedef struct {
+    ResourceManifestEntryType type;
+    string_t name;
+    uint32_t size;
+    uint8_t hash[16];
+} ResourceManifestEntry;
+
+typedef struct ResourceManifestReader ResourceManifestReader;
+
+/**
+ * @brief Initialize resource manifest reader
+ * @param storage Storage API pointer
+ * @return allocated object
+ */
+ResourceManifestReader* resource_manifest_reader_alloc(Storage* storage);
+
+/**
+ * @brief Release resource manifest reader
+ * @param resource_manifest allocated object
+ */
+void resource_manifest_reader_free(ResourceManifestReader* resource_manifest);
+
+/**
+ * @brief Initialize resource manifest reader iteration
+ * @param resource_manifest allocated object
+ * @param filename manifest file name
+ * @return true if file opened
+ */
+bool resource_manifest_reader_open(ResourceManifestReader* resource_manifest, const char* filename);
+
+/**
+ * @brief Read next file/dir entry from manifest
+ * @param resource_manifest allocated object
+ * @return entry or NULL if end of file
+ */
+ResourceManifestEntry* resource_manifest_reader_next(ResourceManifestReader* resource_manifest);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif