[FL-2477] Updater support for resource bundles (#1131)

* Resource unpacking core
* Added more fields to manifest; updated dist scripts
* Python linter fixes
* Parsing manifest before unpacking
* Updated pipelines for separate resource build
* Removed raw path formatting
* Visual progress for resource extraction
* Renamed update status enum

Co-authored-by: Aleksandr Kutuzov <alleteam@gmail.com>

.github/workflows/build.yml

@@ -76,8 +76,7 @@ jobs:
         with:
           run: |
             set -e
-            make -C assets clean
-            make -C assets
+            make assets_manifest
             git diff --quiet || ( echo "Assets recompilation required."; exit 255 )
 
       - name: 'Build the firmware in docker'
@@ -118,7 +117,6 @@ jobs:
       - name: 'Bundle resources'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          ./scripts/assets.py manifest assets/resources
           tar czpf artifacts/flipper-z-any-resources-${{steps.names.outputs.suffix}}.tgz -C assets resources
 
       - name: 'Bundle core2 firmware'

Makefile

@@ -92,9 +92,19 @@ updater_clean:
 updater_debug:
 	@$(MAKE) -C $(PROJECT_ROOT)/firmware -j$(NPROCS) RAM_EXEC=1 debug
 
+.PHONY: updater_package_bin
+updater_package_bin: firmware_all updater
+	@$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)"
+
 .PHONY: updater_package
 updater_package: firmware_all updater
-	@$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)"
+	@$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) -a assets/resources --bundlever "$(VERSION_STRING)"
+
+.PHONY: assets_manifest
+assets_manifest:
+	@$(MAKE) -C $(PROJECT_ROOT)/assets clean
+	@$(MAKE) -C $(PROJECT_ROOT)/assets
+	@$(PROJECT_ROOT)/scripts/assets.py manifest assets/resources
 
 .PHONY: flash_radio
 flash_radio:

applications/updater/scenes/updater_scene_main.c

@@ -73,7 +73,7 @@ bool updater_scene_main_on_event(void* context, SceneManagerEvent event) {
 
         case UpdaterCustomEventRetryUpdate:
             if(!update_task_is_running(updater->update_task) &&
-               (update_task_get_state(updater->update_task)->stage != UpdateTaskStageComplete))
+               (update_task_get_state(updater->update_task)->stage != UpdateTaskStageCompleted))
                 update_task_start(updater->update_task);
             consumed = true;
             break;

applications/updater/util/update_task.c

@@ -19,7 +19,8 @@ static const char* update_task_stage_descr[] = {
     [UpdateTaskStageRadioCommit] = "Applying radio stack",
     [UpdateTaskStageLfsBackup] = "Backing up LFS",
     [UpdateTaskStageLfsRestore] = "Restoring LFS",
-    [UpdateTaskStageComplete] = "Complete",
+    [UpdateTaskStageAssetsUpdate] = "Updating assets",
+    [UpdateTaskStageCompleted] = "Completed!",
     [UpdateTaskStageError] = "Error",
 };
 

applications/updater/util/update_task.h

@@ -23,7 +23,8 @@ typedef enum {
     UpdateTaskStageRadioCommit,
     UpdateTaskStageLfsBackup,
     UpdateTaskStageLfsRestore,
-    UpdateTaskStageComplete,
+    UpdateTaskStageAssetsUpdate,
+    UpdateTaskStageCompleted,
     UpdateTaskStageError,
 } UpdateTaskStage;
 

applications/updater/util/update_task_workers.c

@@ -8,6 +8,7 @@
 #include <update_util/dfu_file.h>
 #include <update_util/lfs_backup.h>
 #include <update_util/update_operation.h>
+#include <toolbox/tar/tar_archive.h>
 
 #define CHECK_RESULT(x) \
     if(!(x)) {          \
@@ -19,6 +20,8 @@
 /* Written into DFU file by build pipeline */
 #define FLIPPER_ZERO_DFU_DEVICE_CODE 0xFFFF
 
+#define EXT_PATH "/ext"
+
 static const DfuValidationParams flipper_dfu_params = {
     .device = FLIPPER_ZERO_DFU_DEVICE_CODE,
     .product = STM_DFU_PRODUCT_ID,
@@ -85,7 +88,7 @@ int32_t update_task_worker_flash_writer(void* context) {
             CHECK_RESULT(dfu_file_process_targets(&page_task, update_task->file, valid_targets));
         }
 
-        update_task_set_progress(update_task, UpdateTaskStageComplete, 100);
+        update_task_set_progress(update_task, UpdateTaskStageCompleted, 100);
 
         furi_hal_rtc_set_boot_mode(FuriHalRtcBootModePostUpdate);
 
@@ -99,6 +102,95 @@ int32_t update_task_worker_flash_writer(void* context) {
     return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED;
 }
 
+static bool update_task_pre_update(UpdateTask* update_task) {
+    bool success = false;
+    string_t backup_file_path;
+    string_init(backup_file_path);
+    path_concat(
+        string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path);
+
+    update_task->state.total_stages = 1;
+    update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0);
+    furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops
+    if((success = lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) {
+        furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate);
+    }
+
+    string_clear(backup_file_path);
+    return success;
+}
+
+typedef struct {
+    UpdateTask* update_task;
+    int32_t total_files, processed_files;
+} TarUnpackProgress;
+
+static bool update_task_resource_unpack_cb(const char* name, bool is_directory, void* context) {
+    UNUSED(name);
+    UNUSED(is_directory);
+    TarUnpackProgress* unpack_progress = context;
+    unpack_progress->processed_files++;
+    update_task_set_progress(
+        unpack_progress->update_task,
+        UpdateTaskStageProgress,
+        unpack_progress->processed_files * 100 / (unpack_progress->total_files + 1));
+    return true;
+}
+
+static bool update_task_post_update(UpdateTask* update_task) {
+    bool success = false;
+
+    string_t file_path;
+    string_init(file_path);
+
+    update_task->state.total_stages = 2;
+
+    do {
+        CHECK_RESULT(update_task_parse_manifest(update_task));
+        path_concat(
+            string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, file_path);
+
+        bool unpack_resources = !string_empty_p(update_task->manifest->resource_bundle);
+        if(unpack_resources) {
+            update_task->state.total_stages++;
+        }
+
+        update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0);
+        furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal);
+
+        CHECK_RESULT(lfs_backup_unpack(update_task->storage, string_get_cstr(file_path)));
+
+        if(unpack_resources) {
+            TarUnpackProgress progress = {
+                .update_task = update_task,
+                .total_files = 0,
+                .processed_files = 0,
+            };
+            update_task_set_progress(update_task, UpdateTaskStageAssetsUpdate, 0);
+
+            path_concat(
+                string_get_cstr(update_task->update_path),
+                string_get_cstr(update_task->manifest->resource_bundle),
+                file_path);
+
+            update_task_set_progress(update_task, UpdateTaskStageProgress, 0);
+            TarArchive* archive = tar_archive_alloc(update_task->storage);
+            tar_archive_set_file_callback(archive, update_task_resource_unpack_cb, &progress);
+            success = tar_archive_open(archive, string_get_cstr(file_path), TAR_OPEN_MODE_READ);
+            if(success) {
+                progress.total_files = tar_archive_get_entries_count(archive);
+                if(progress.total_files > 0) {
+                    tar_archive_unpack_to(archive, EXT_PATH);
+                }
+            }
+            tar_archive_free(archive);
+        }
+    } while(false);
+
+    string_clear(file_path);
+    return success;
+}
+
 int32_t update_task_worker_backup_restore(void* context) {
     furi_assert(context);
     UpdateTask* update_task = context;
@@ -112,37 +204,22 @@ int32_t update_task_worker_backup_restore(void* context) {
     }
 
     update_task->state.current_stage_idx = 0;
-    update_task->state.total_stages = 1;
 
     if(!update_operation_get_current_package_path(update_task->storage, update_task->update_path)) {
         return UPDATE_TASK_FAILED;
     }
 
-    string_t backup_file_path;
-    string_init(backup_file_path);
-    path_concat(
-        string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path);
-
     if(boot_mode == FuriHalRtcBootModePreUpdate) {
-        update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0);
-        furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops
-        if((success =
-                lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) {
-            furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate);
-        }
+        success = update_task_pre_update(update_task);
     } else if(boot_mode == FuriHalRtcBootModePostUpdate) {
-        update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0);
-        furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal);
-        success = lfs_backup_unpack(update_task->storage, string_get_cstr(backup_file_path));
+        success = update_task_post_update(update_task);
     }
 
     if(success) {
-        update_task_set_progress(update_task, UpdateTaskStageComplete, 100);
+        update_task_set_progress(update_task, UpdateTaskStageCompleted, 100);
     } else {
         update_task_set_progress(update_task, UpdateTaskStageError, update_task->state.progress);
     }
 
-    string_clear(backup_file_path);
-
     return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED;
-}
+}

lib/toolbox/tar/tar_archive.c

@@ -15,6 +15,8 @@
 typedef struct TarArchive {
     Storage* storage;
     mtar_t tar;
+    tar_unpack_file_cb unpack_cb;
+    void* unpack_cb_context;
 } TarArchive;
 
 /* API WRAPPER */
@@ -51,6 +53,7 @@ TarArchive* tar_archive_alloc(Storage* storage) {
     furi_check(storage);
     TarArchive* archive = malloc(sizeof(TarArchive));
     archive->storage = storage;
+    archive->unpack_cb = NULL;
     return archive;
 }
 
@@ -92,6 +95,28 @@ void tar_archive_free(TarArchive* archive) {
     }
 }
 
+void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context) {
+    furi_assert(archive);
+    archive->unpack_cb = callback;
+    archive->unpack_cb_context = context;
+}
+
+static int tar_archive_entry_counter(mtar_t* tar, const mtar_header_t* header, void* param) {
+    UNUSED(tar);
+    UNUSED(header);
+    int32_t* counter = param;
+    (*counter)++;
+    return 0;
+}
+
+int32_t tar_archive_get_entries_count(TarArchive* archive) {
+    int32_t counter = 0;
+    if(mtar_foreach(&archive->tar, tar_archive_entry_counter, &counter) != MTAR_ESUCCESS) {
+        counter = -1;
+    }
+    return counter;
+}
+
 bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath) {
     furi_assert(archive);
     return (mtar_write_dir_header(&archive->tar, dirpath) == MTAR_ESUCCESS);
@@ -142,14 +167,25 @@ typedef struct {
 
 static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, void* param) {
     TarArchiveDirectoryOpParams* op_params = param;
+    TarArchive* archive = op_params->archive;
     string_t fname;
 
+    bool skip_entry = false;
+    if(archive->unpack_cb) {
+        skip_entry = !archive->unpack_cb(
+            header->name, header->type == MTAR_TDIR, archive->unpack_cb_context);
+    }
+
+    if(skip_entry) {
+        FURI_LOG_W(TAG, "filter: skipping entry \"%s\"", header->name);
+        return 0;
+    }
+
     if(header->type == MTAR_TDIR) {
         string_init(fname);
         path_concat(op_params->work_dir, header->name, fname);
 
-        bool create_res =
-            storage_simply_mkdir(op_params->archive->storage, string_get_cstr(fname));
+        bool create_res = storage_simply_mkdir(archive->storage, string_get_cstr(fname));
         string_clear(fname);
         return create_res ? 0 : -1;
     }
@@ -162,7 +198,7 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header,
     string_init(fname);
     path_concat(op_params->work_dir, header->name, fname);
     FURI_LOG_I(TAG, "Extracting %d bytes to '%s'", header->size, header->name);
-    File* out_file = storage_file_alloc(op_params->archive->storage);
+    File* out_file = storage_file_alloc(archive->storage);
     uint8_t* readbuf = malloc(FILE_BLOCK_SIZE);
 
     bool failed = false;
@@ -303,4 +339,4 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch
     free(name);
     storage_file_free(directory);
     return success;
-}
+}

lib/toolbox/tar/tar_archive.h

@@ -34,6 +34,13 @@ bool tar_archive_add_file(
 
 bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const char* path_prefix);
 
+int32_t tar_archive_get_entries_count(TarArchive* archive);
+
+/* Optional per-entry callback on unpacking - return false to skip entry */
+typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* context);
+
+void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context);
+
 /* Low-level API */
 bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath);
 

lib/update_util/update_manifest.c

@@ -4,12 +4,24 @@
 #include <flipper_format/flipper_format.h>
 #include <flipper_format/flipper_format_i.h>
 
+#define MANIFEST_KEY_INFO "Info"
+#define MANIFEST_KEY_TARGET "Target"
+#define MANIFEST_KEY_LOADER_FILE "Loader"
+#define MANIFEST_KEY_LOADER_CRC "Loader CRC"
+#define MANIFEST_KEY_DFU_FILE "Firmware"
+#define MANIFEST_KEY_RADIO_FILE "Radio"
+#define MANIFEST_KEY_RADIO_ADDRESS "Radio address"
+#define MANIFEST_KEY_RADIO_VERSION "Radio version"
+#define MANIFEST_KEY_RADIO_CRC "Radio CRC"
+#define MANIFEST_KEY_ASSETS_FILE "Assets"
+
 UpdateManifest* update_manifest_alloc() {
     UpdateManifest* update_manifest = malloc(sizeof(UpdateManifest));
     string_init(update_manifest->version);
     string_init(update_manifest->firmware_dfu_image);
     string_init(update_manifest->radio_image);
     string_init(update_manifest->staged_loader_file);
+    string_init(update_manifest->resource_bundle);
     update_manifest->target = 0;
     update_manifest->valid = false;
     return update_manifest;
@@ -21,6 +33,7 @@ void update_manifest_free(UpdateManifest* update_manifest) {
     string_clear(update_manifest->firmware_dfu_image);
     string_clear(update_manifest->radio_image);
     string_clear(update_manifest->staged_loader_file);
+    string_clear(update_manifest->resource_bundle);
     free(update_manifest);
 }
 
@@ -36,21 +49,47 @@ static bool
     string_init(filetype);
     update_manifest->valid =
         flipper_format_read_header(flipper_file, filetype, &version) &&
-        flipper_format_read_string(flipper_file, "Info", update_manifest->version) &&
-        flipper_format_read_uint32(flipper_file, "Target", &update_manifest->target, 1) &&
-        flipper_format_read_string(flipper_file, "Loader", update_manifest->staged_loader_file) &&
+        flipper_format_read_string(flipper_file, MANIFEST_KEY_INFO, update_manifest->version) &&
+        flipper_format_read_uint32(
+            flipper_file, MANIFEST_KEY_TARGET, &update_manifest->target, 1) &&
+        flipper_format_read_string(
+            flipper_file, MANIFEST_KEY_LOADER_FILE, update_manifest->staged_loader_file) &&
         flipper_format_read_hex(
             flipper_file,
-            "Loader CRC",
+            MANIFEST_KEY_LOADER_CRC,
             (uint8_t*)&update_manifest->staged_loader_crc,
             sizeof(uint32_t));
     string_clear(filetype);
 
-    /* Optional fields - we can have dfu, radio, or both */
-    flipper_format_read_string(flipper_file, "Firmware", update_manifest->firmware_dfu_image);
-    flipper_format_read_string(flipper_file, "Radio", update_manifest->radio_image);
-    flipper_format_read_hex(
-        flipper_file, "Radio address", (uint8_t*)&update_manifest->radio_address, sizeof(uint32_t));
+    if(update_manifest->valid) {
+        /* Optional fields - we can have dfu, radio, or both */
+        flipper_format_read_string(
+            flipper_file, MANIFEST_KEY_DFU_FILE, update_manifest->firmware_dfu_image);
+        flipper_format_read_string(
+            flipper_file, MANIFEST_KEY_RADIO_FILE, update_manifest->radio_image);
+        flipper_format_read_hex(
+            flipper_file,
+            MANIFEST_KEY_RADIO_ADDRESS,
+            (uint8_t*)&update_manifest->radio_address,
+            sizeof(uint32_t));
+        flipper_format_read_hex(
+            flipper_file,
+            MANIFEST_KEY_RADIO_VERSION,
+            (uint8_t*)&update_manifest->radio_version,
+            sizeof(uint32_t));
+        flipper_format_read_hex(
+            flipper_file,
+            MANIFEST_KEY_RADIO_CRC,
+            (uint8_t*)&update_manifest->radio_crc,
+            sizeof(uint32_t));
+        flipper_format_read_string(
+            flipper_file, MANIFEST_KEY_ASSETS_FILE, update_manifest->resource_bundle);
+
+        update_manifest->valid =
+            (!string_empty_p(update_manifest->firmware_dfu_image) ||
+             !string_empty_p(update_manifest->radio_image) ||
+             !string_empty_p(update_manifest->resource_bundle));
+    }
 
     return update_manifest->valid;
 }
@@ -83,4 +122,4 @@ bool update_manifest_init_mem(
     flipper_format_free(flipper_file);
 
     return update_manifest->valid;
-}
+}

lib/update_util/update_manifest.h

@@ -21,6 +21,9 @@ typedef struct {
     string_t firmware_dfu_image;
     string_t radio_image;
     uint32_t radio_address;
+    uint32_t radio_version;
+    uint32_t radio_crc;
+    string_t resource_bundle;
     bool valid;
 } UpdateManifest;
 

scripts/dist.py

@@ -18,6 +18,7 @@ class Main(App):
         self.parser_copy.add_argument("-t", dest="target", required=True)
         self.parser_copy.add_argument("-p", dest="projects", nargs="+", required=True)
         self.parser_copy.add_argument("-s", dest="suffix", required=True)
+        self.parser_copy.add_argument("-a", dest="assets", required=False)
         self.parser_copy.add_argument(
             "--bundlever",
             dest="version",
@@ -83,6 +84,13 @@ class Main(App):
                 "-stage",
                 self.get_dist_filepath(self.get_project_filename("updater", "bin")),
             ]
+            if self.args.assets:
+                bundle_args.extend(
+                    (
+                        "-a",
+                        self.args.assets,
+                    )
+                )
             self.logger.info(
                 f"Use this directory to self-update your Flipper:\n\t{bundle_dir}"
             )

scripts/update.py

@@ -3,12 +3,16 @@
 from flipper.app import App
 from flipper.utils.fff import FlipperFormatFile
 from os.path import basename, join, exists
-from os import makedirs
+import os
 import shutil
 import zlib
+import tarfile
 
 
 class Main(App):
+    #  No compression, plain tar
+    ASSET_TAR_MODE = "w:"
+
     def init(self):
         self.subparsers = self.parser.add_subparsers(help="sub-command help")
 
@@ -20,24 +24,41 @@ class Main(App):
         self.parser_generate.add_argument("-d", dest="directory", required=True)
         self.parser_generate.add_argument("-v", dest="version", required=True)
         self.parser_generate.add_argument("-t", dest="target", required=True)
-        self.parser_generate.add_argument("-dfu", dest="dfu", required=True)
+        self.parser_generate.add_argument("-dfu", dest="dfu", required=False)
+        self.parser_generate.add_argument("-a", dest="assets", required=False)
         self.parser_generate.add_argument("-stage", dest="stage", required=True)
-        self.parser_generate.add_argument("-radio", dest="radiobin", required=False)
+        self.parser_generate.add_argument(
+            "-radio", dest="radiobin", default="", required=False
+        )
         self.parser_generate.add_argument(
             "-radioaddr", dest="radioaddr", required=False
         )
+        self.parser_generate.add_argument(
+            "-radiover", dest="radioversion", required=False
+        )
 
         self.parser_generate.set_defaults(func=self.generate)
 
     def generate(self):
         stage_basename = basename(self.args.stage)
         dfu_basename = basename(self.args.dfu)
+        radiobin_basename = basename(self.args.radiobin)
+        assets_basename = ""
 
         if not exists(self.args.directory):
-            makedirs(self.args.directory)
+            os.makedirs(self.args.directory)
 
         shutil.copyfile(self.args.stage, join(self.args.directory, stage_basename))
         shutil.copyfile(self.args.dfu, join(self.args.directory, dfu_basename))
+        if radiobin_basename:
+            shutil.copyfile(
+                self.args.radiobin, join(self.args.directory, radiobin_basename)
+            )
+        if self.args.assets:
+            assets_basename = "assets.tar"
+            self.package_assets(
+                self.args.assets, join(self.args.directory, assets_basename)
+            )
 
         file = FlipperFormatFile()
         file.setHeader("Flipper firmware upgrade configuration", 1)
@@ -47,12 +68,24 @@ class Main(App):
         file.writeComment("little-endian hex!")
         file.writeKey("Loader CRC", self.int2ffhex(self.crc(self.args.stage)))
         file.writeKey("Firmware", dfu_basename)
-        file.writeKey("Radio", self.args.radiobin or "")
+        file.writeKey("Radio", radiobin_basename or "")
         file.writeKey("Radio address", self.int2ffhex(self.args.radioaddr or 0))
-        file.save("%s/update.fuf" % self.args.directory)
+        file.writeKey("Radio version", self.int2ffhex(self.args.radioversion or 0))
+        if radiobin_basename:
+            file.writeKey("Radio CRC", self.int2ffhex(self.crc(self.args.radiobin)))
+        else:
+            file.writeKey("Radio CRC", self.int2ffhex(0))
+        file.writeKey("Assets", assets_basename)
+        file.save(join(self.args.directory, "update.fuf"))
 
         return 0
 
+    def package_assets(self, srcdir: str, dst_name: str):
+        with tarfile.open(
+            dst_name, self.ASSET_TAR_MODE, format=tarfile.USTAR_FORMAT
+        ) as tarball:
+            tarball.add(srcdir, arcname="")
+
     @staticmethod
     def int2ffhex(value: int):
         hexstr = "%08X" % value