fbt: initial blackmagic support (#1362)

* fbt: added separate script for Windows env setup; moved flash targets from firmware.scons to SConstruct; added Blackmagic support with automatic probe port resolution; added apps.c rebuild on any manifest.fam changes; fixed simultaneous flash & debug ops
* fbt: added networked BlackmagicResolver mode; added `get_blackmagic` target for IDE integration
* fbt: cleanup
* fbt: docs update; fixed blackmagic lookup on certain usb hubs
* fbt: removed explicit python serial port import
* fbt: cleanup
* fbt: raising exception on multiple serial blackmagic probes

SConstruct

@@ -28,15 +28,41 @@ SConscript("site_scons/cc.scons", exports={"ENV": coreenv})
 # Store root dir in environment for certain tools
 coreenv["ROOT_DIR"] = Dir(".")
 
+
 # Create a separate "dist" environment and add construction envs to it
 distenv = coreenv.Clone(
-    tools=["fbt_dist", "openocd"],
-    GDBOPTS="-ex 'target extended-remote | ${OPENOCD} -c \"gdb_port pipe\" ${OPENOCD_OPTS}' "
-    '-ex "set confirm off" ',
+    tools=["fbt_dist", "openocd", "blackmagic"],
+    OPENOCD_GDB_PIPE=["|openocd -c 'gdb_port pipe' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}"],
+    GDBOPTS_BASE=[
+        "-ex",
+        "target extended-remote ${GDBREMOTE}",
+        "-ex",
+        "set confirm off",
+    ],
+    GDBOPTS_BLACKMAGIC=[
+        "-ex",
+        "monitor swdp_scan",
+        "-ex",
+        "monitor debug_bmp enable",
+        "-ex",
+        "attach 1",
+        "-ex",
+        "set mem inaccessible-by-default off",
+    ],
+    GDBPYOPTS=[
+        "-ex",
+        "source debug/FreeRTOS/FreeRTOS.py",
+        "-ex",
+        "source debug/PyCortexMDebug/PyCortexMDebug.py",
+        "-ex",
+        "svd_load ${SVD_FILE}",
+        "-ex",
+        "compare-sections",
+    ],
     ENV=os.environ,
 )
 
-firmware_out = distenv.AddFwProject(
+firmware_env = distenv.AddFwProject(
     base_env=coreenv,
     fw_type="firmware",
     fw_env_key="FW_ENV",
@@ -44,7 +70,7 @@ firmware_out = distenv.AddFwProject(
 
 # If enabled, initialize updater-related targets
 if GetOption("fullenv"):
-    updater_out = distenv.AddFwProject(
+    updater_env = distenv.AddFwProject(
         base_env=coreenv,
         fw_type="updater",
         fw_env_key="UPD_ENV",
@@ -72,19 +98,32 @@ if GetOption("fullenv"):
 
     selfupdate_dist = distenv.DistCommand(
         "updater_package",
-        (distenv["DIST_DEPENDS"], firmware_out["FW_RESOURCES"]),
+        (distenv["DIST_DEPENDS"], firmware_env["FW_RESOURCES"]),
         DIST_EXTRA=dist_arguments,
     )
 
     # Updater debug
-    distenv.AddDebugTarget("updater_debug", updater_out, False)
+    distenv.PhonyTarget(
+        "updater_debug",
+        "${GDBPYCOM}",
+        source=updater_env["FW_ELF"],
+        GDBREMOTE="${OPENOCD_GDB_PIPE}",
+    )
+
+    distenv.PhonyTarget(
+        "updater_blackmagic",
+        "${GDBPYCOM}",
+        source=updater_env["FW_ELF"],
+        GDBOPTS=distenv.subst("$GDBOPTS_BLACKMAGIC"),
+        GDBREMOTE="${BLACKMAGIC_ADDR}",
+    )
 
     # Installation over USB & CLI
     usb_update_package = distenv.UsbInstall(
         "#build/usbinstall.flag",
         (
             distenv["DIST_DEPENDS"],
-            firmware_out["FW_RESOURCES"],
+            firmware_env["FW_RESOURCES"],
             selfupdate_dist,
         ),
     )
@@ -104,15 +143,47 @@ copro_dist = distenv.CoproBuilder(
 )
 distenv.Alias("copro_dist", copro_dist)
 
+firmware_flash = distenv.AddOpenOCDFlashTarget(firmware_env)
+distenv.Alias("flash", firmware_flash)
+
+firmware_bm_flash = distenv.PhonyTarget(
+    "flash_blackmagic",
+    "$GDB $GDBOPTS $SOURCES $GDBFLASH",
+    source=firmware_env["FW_ELF"],
+    GDBOPTS="${GDBOPTS_BASE} ${GDBOPTS_BLACKMAGIC}",
+    GDBREMOTE="${BLACKMAGIC_ADDR}",
+    GDBFLASH=[
+        "-ex",
+        "load",
+        "-ex",
+        "quit",
+    ],
+)
+
 # Debugging firmware
-distenv.AddDebugTarget("debug", firmware_out)
+firmware_debug = distenv.PhonyTarget(
+    "debug",
+    "${GDBPYCOM}",
+    source=firmware_env["FW_ELF"],
+    GDBOPTS="${GDBOPTS_BASE}",
+    GDBREMOTE="${OPENOCD_GDB_PIPE}",
+)
+distenv.Depends(firmware_debug, firmware_flash)
+
+distenv.PhonyTarget(
+    "blackmagic",
+    "${GDBPYCOM}",
+    source=firmware_env["FW_ELF"],
+    GDBOPTS="${GDBOPTS_BASE} ${GDBOPTS_BLACKMAGIC}",
+    GDBREMOTE="${BLACKMAGIC_ADDR}",
+)
+
 # Debug alien elf
 distenv.PhonyTarget(
     "debug_other",
-    "$GDBPYCOM",
-    GDBPYOPTS=
-    # '-ex "source ${ROOT_DIR.abspath}/debug/FreeRTOS/FreeRTOS.py" '
-    '-ex "source debug/PyCortexMDebug/PyCortexMDebug.py" ',
+    "${GDBPYCOM}",
+    GDBPYOPTS='-ex "source debug/PyCortexMDebug/PyCortexMDebug.py" ',
+    GDBREMOTE="${OPENOCD_GDB_PIPE}",
 )
 
 # Just start OpenOCD
@@ -125,11 +196,19 @@ distenv.PhonyTarget(
 distenv.PhonyTarget(
     "lint",
     "${PYTHON3} scripts/lint.py check ${LINT_SOURCES}",
-    LINT_SOURCES=firmware_out["LINT_SOURCES"],
+    LINT_SOURCES=firmware_env["LINT_SOURCES"],
 )
 
 distenv.PhonyTarget(
     "format",
     "${PYTHON3} scripts/lint.py format ${LINT_SOURCES}",
-    LINT_SOURCES=firmware_out["LINT_SOURCES"],
+    LINT_SOURCES=firmware_env["LINT_SOURCES"],
+)
+
+
+# Find blackmagic probe
+
+distenv.PhonyTarget(
+    "get_blackmagic",
+    "@echo $( ${BLACKMAGIC_ADDR} $)",
 )

documentation/fbt.md

@@ -34,7 +34,9 @@ FBT keeps track of internal dependencies, so you only need to build the highest-
 - `debug` - build and flash firmware, then attach with gdb with firmware's .elf loaded
 - `debug_updater` - attach gdb with updater's .elf loaded. _Requires `--with-updater` option_
 - `debug_other` - attach gdb without loading any .elf. Allows to manually add external elf files with `add-symbol-file` in gdb.
+- `blackmagic` - debug firmware with Blackmagic probe (WiFi dev board)
 - `openocd` - just start OpenOCD
+- `get_blackmagic` - output blackmagic address in gdb remote format. Useful for IDE integration
 
 ### Firmware targets
 
@@ -43,6 +45,7 @@ FBT keeps track of internal dependencies, so you only need to build the highest-
     - Check out `--extra-ext-apps` for force adding extra apps to external build 
     - `firmware_snake_game_list`, etc - generate source + assembler listing for app's .elf
 - `flash`, `firmware_flash` - flash current version to attached device with OpenOCD over ST-Link
+- `flash_blackmagic` - flash current version to attached device with Blackmagic probe
 - `firmware_cdb` - generate compilation database
 - `firmware_all`, `updater_all` - build basic set of binaries
 - `firmware_list`, `updater_list` - generate source + assembler listing

fbt

@@ -6,7 +6,6 @@ SCRIPTDIR="$( dirname -- "$0"; )";
 SCONS_EP=${SCRIPTDIR}/lib/scons/scripts/scons.py
 
 if [[ -d .git ]]; then
-    echo Updating git submodules
     git submodule update --init
 else # Not in a git repo
     echo Not in a git repo, please clone with git clone --recursive

fbt.cmd

@@ -3,8 +3,7 @@
 set SCONS_EP=%~dp0\lib\scons\scripts\scons.py
 
 if exist ".git" (
-	echo Updating git submodules
-	git submodule update --init	
+	git submodule update --init
 )
 
 set "SCONS_DEFAULT_FLAGS=-Q --warn=target-not-built"

fbt_options.py

@@ -41,10 +41,24 @@ COPRO_STACK_BIN_DIR = posixpath.join(
 # Supported toolchain versions
 FBT_TOOLCHAIN_VERSIONS = (" 10.3.",)
 
-OPENOCD_OPTS = '-f interface/stlink.cfg -c "transport select hla_swd" -f debug/stm32wbx.cfg -c "stm32wbx.cpu configure -rtos auto" -c "init"'
+OPENOCD_OPTS = [
+    "-f",
+    "interface/stlink.cfg",
+    "-c",
+    "transport select hla_swd",
+    "-f",
+    "debug/stm32wbx.cfg",
+    "-c",
+    "stm32wbx.cpu configure -rtos auto",
+    "-c",
+    "init",
+]
 
 SVD_FILE = "debug/STM32WB55_CM4.svd"
 
+# Look for blackmagic probe on serial ports
+BLACKMAGIC = "auto"
+
 FIRMWARE_APPS = {
     "default": [
         "crypto_start",

firmware.scons

@@ -6,7 +6,7 @@ from fbt.util import link_dir
 
 # Building initial C environment for libs
 env = ENV.Clone(
-    tools=["compilation_db", "fwbin", "openocd", "fbt_apps"],
+    tools=["compilation_db", "fwbin", "fbt_apps"],
     COMPILATIONDB_USE_ABSPATH=True,
     BUILD_DIR=fw_build_meta["build_dir"],
     IS_BASE_FIRMWARE=fw_build_meta["type"] == "firmware",
@@ -139,6 +139,8 @@ apps_c = fwenv.ApplicationsC(
     "applications/applications.c",
     Value(fwenv["APPS"]),
 )
+# Adding dependency on manifest files so apps.c is rebuilt when any manifest is changed
+fwenv.Depends(apps_c, fwenv.GlobRecursive("*.fam", "applications"))
 
 sources = [apps_c]
 # Gather sources only from app folders from current configuration
@@ -235,7 +237,7 @@ AddPostAction(fwelf, Action("@$SIZECOM"))
 AddPostAction(fwelf, Action(link_latest_dir, None))
 
 link_dir_command = fwenv["LINK_DIR_CMD"] = fwenv.PhonyTarget(
-    "${FIRMWARE_BUILD_CFG}" + "_latest",
+    fwenv.subst("${FIRMWARE_BUILD_CFG}_latest"),
     Action(lambda target, source, env: link_elf_dir_as_latest(env, source[0]), None),
     source=fwelf,
 )
@@ -249,18 +251,6 @@ Alias(fwenv["FIRMWARE_BUILD_CFG"] + "_dfu", fwdfu)
 fwdump = fwenv.ObjDump("${FIRMWARE_BUILD_CFG}")
 Alias(fwenv["FIRMWARE_BUILD_CFG"] + "_list", fwdump)
 
-# Additional FW-related pseudotargets
-flash = fwenv["FW_FLASH"] = fwenv.OpenOCDFlash(
-    "#build/oocd-${FIRMWARE_BUILD_CFG}-flash.flag",
-    "${FIRMWARE_BUILD_CFG}",
-    OPENOCD_COMMAND='-c "program ${SOURCE.posix} reset exit ${IMAGE_BASE_ADDRESS}"',
-)
-if fwenv["FORCE"]:
-    fwenv.AlwaysBuild(flash)
-fwenv.Alias(fwenv["FIRMWARE_BUILD_CFG"] + "_flash", flash)
-if fwenv["IS_BASE_FIRMWARE"]:
-    fwenv.Alias("flash", flash)
-
 
 # Compile DB generation
 fwcdb = fwenv["FW_CDB"] = fwenv.CompilationDatabase("compile_commands.json")

site_scons/commandline.scons

@@ -162,6 +162,12 @@ vars.Add(
     default="",
 )
 
+vars.Add(
+    "BLACKMAGIC",
+    help="Blackmagic probe location",
+    default="auto",
+)
+
 vars.Add(
     "UPDATE_SPLASH",
     help="Directory name with slideshow frames to render after installing update package",

site_scons/environ.scons

@@ -78,5 +78,6 @@ coreenv["TEMPFILEARGESCFUNC"] = util.tempfile_arg_esc_func
 util.wrap_tempfile(coreenv, "LINKCOM")
 util.wrap_tempfile(coreenv, "ARCOM")
 
+coreenv["SINGLEQUOTEFUNC"] = util.single_quote
 
 Return("coreenv")

site_scons/fbt/util.py

@@ -42,3 +42,7 @@ def random_alnum(length):
     return "".join(
         random.choice(string.ascii_letters + string.digits) for _ in range(length)
     )
+
+
+def single_quote(arg_list):
+    return " ".join(f"'{arg}'" if " " in arg else str(arg) for arg in arg_list)

site_scons/site_tools/blackmagic.py

@@ -0,0 +1,74 @@
+from SCons.Errors import StopError
+
+
+class BlackmagicResolver:
+    BLACKMAGIC_HOSTNAME = "blackmagic.local"
+
+    def __init__(self, env):
+        self.env = env
+
+    # On Win:
+    #    'location': '1-5:x.0', 'name': 'COM4',
+    #    'location': '1-5:x.2', 'name': 'COM13',
+    # On Linux:
+    #    'location': '1-1.2:1.0', 'name': 'ttyACM0',
+    #    'location': '1-1.2:1.2', 'name': 'ttyACM1',
+    # On MacOS:
+    #    'location': '0-1.3', 'name': 'cu.usbmodemblackmagic1',
+    #    'location': '0-1.3', 'name': 'cu.usbmodemblackmagic3',
+    def _find_probe(self):
+        import serial.tools.list_ports as list_ports
+
+        ports = list(list_ports.grep("blackmagic"))
+        if len(ports) == 0:
+            # Blackmagic probe serial port not found, will be handled later
+            pass
+        elif len(ports) > 2:
+            raise StopError("More than one Blackmagic probe found")
+        else:
+            # If you're getting any issues with auto lookup, uncomment this
+            # print("\n".join([f"{p.device} {vars(p)}" for p in ports]))
+            return sorted(ports, key=lambda p: f"{p.location}_{p.name}")[0]
+
+    # Look up blackmagic probe hostname with dns
+    def _resolve_hostname(self):
+        import socket
+
+        try:
+            return socket.gethostbyname(self.BLACKMAGIC_HOSTNAME)
+        except socket.gaierror:
+            print("Failed to resolve Blackmagic hostname")
+            return None
+
+    def get_serial(self):
+        if not (probe := self._find_probe()):
+            return None
+        # print(f"Found Blackmagic probe on {probe.device}")
+        if self.env.subst("$PLATFORM") == "win32":
+            return f"\\\\.\\{probe.device}"
+        return probe.device
+
+    def get_networked(self):
+        if not (probe := self._resolve_hostname()):
+            return None
+
+        return f"tcp:{probe}:2345"
+
+    def __str__(self):
+        # print("distenv blackmagic", self.env.subst("$BLACKMAGIC"))
+        if (blackmagic := self.env.subst("$BLACKMAGIC")) != "auto":
+            return blackmagic
+
+        # print("Looking for Blackmagic...")
+        if probe := self.get_serial() or self.get_networked():
+            return probe
+
+        raise Exception("Please specify BLACKMAGIC=...")
+
+
+def generate(env):
+    env.SetDefault(BLACKMAGIC_ADDR=BlackmagicResolver(env))
+
+
+def exists(env):
+    return True

site_scons/site_tools/fbt_dist.py

@@ -54,20 +54,20 @@ def AddFwProject(env, base_env, fw_type, fw_env_key):
     return project_env
 
 
-def AddDebugTarget(env, alias, targetenv, force_flash=True):
-    debug_target = env.PhonyTarget(
-        alias,
-        "$GDBPYCOM",
-        source=targetenv["FW_ELF"],
-        GDBPYOPTS='-ex "source debug/FreeRTOS/FreeRTOS.py" '
-        '-ex "source debug/PyCortexMDebug/PyCortexMDebug.py" '
-        '-ex "svd_load ${SVD_FILE}" '
-        '-ex "compare-sections"',
+def AddOpenOCDFlashTarget(env, targetenv, **kw):
+    openocd_target = env.OpenOCDFlash(
+        "#build/oocd-${BUILD_CFG}-flash.flag",
+        targetenv["FW_BIN"],
+        OPENOCD_COMMAND=[
+            "-c",
+            "program ${SOURCE.posix} reset exit ${BASE_ADDRESS}",
+        ],
+        BUILD_CFG=targetenv.subst("$FIRMWARE_BUILD_CFG"),
+        BASE_ADDRESS=targetenv.subst("$IMAGE_BASE_ADDRESS"),
+        **kw,
     )
-    if force_flash:
-        env.Depends(debug_target, targetenv["FW_FLASH"])
-
-    return debug_target
+    env.Alias(targetenv.subst("${FIRMWARE_BUILD_CFG}_flash"), openocd_target)
+    return openocd_target
 
 
 def DistCommand(env, name, source, **kw):
@@ -85,8 +85,9 @@ def DistCommand(env, name, source, **kw):
 
 def generate(env):
     env.AddMethod(AddFwProject)
-    env.AddMethod(AddDebugTarget)
     env.AddMethod(DistCommand)
+    env.AddMethod(AddOpenOCDFlashTarget)
+
     env.SetDefault(
         COPRO_MCU_FAMILY="STM32WB5x",
     )

site_scons/site_tools/sconsmodular.py

@@ -34,9 +34,9 @@ def PhonyTarget(env, name, action, source=None, **kw):
         source = []
     phony_name = "phony_" + name
     env.Pseudo(phony_name)
-    return env.AlwaysBuild(
-        env.Alias(name, env.Command(phony_name, source, action, **kw))
-    )
+    command = env.Command(phony_name, source, action, **kw)
+    env.AlwaysBuild(env.Alias(name, command))
+    return command
 
 
 def generate(env):