Import of the watch repository from Pebble

src/fw/debug/advanced_logging.c

@@ -0,0 +1,181 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flash_logging.h"
+
+#include "console/dbgserial.h"
+#include "drivers/flash.h"
+#include "os/mutex.h"
+#include "system/passert.h"
+#include "util/shared_circular_buffer.h"
+
+#include "services/common/system_task.h"
+
+static SharedCircularBuffer s_buffer;
+static SharedCircularBufferClient s_buffer_client;
+// 526 bytes is enough to buffer up the worst case bunch of reboot messages - a watchdog reset with a timer
+// callback. During normal operation, since log messages are hashed, most are only 30-40 bytes long with the longest
+// being about 80 bytes, so this is enough for 7-15 or so messages.
+static uint8_t s_buffer_storage[550];
+static PebbleMutex *s_buffer_mutex = INVALID_MUTEX_HANDLE; //!< Protects s_buffer
+static PebbleMutex *s_flash_write_mutex = INVALID_MUTEX_HANDLE; //!< Protects log line consistency
+static bool s_is_flash_write_scheduled; //!< true if handle_buffer_sync KernelBG callback is scheduled
+
+static void write_message(void) {
+  // Note that we should enter this function with the buffer mutex held.
+
+  const uint8_t *data_read;
+  uint16_t read_length;
+
+  // Read the header part
+  bool result = shared_circular_buffer_read(&s_buffer, &s_buffer_client, sizeof(uint8_t), &data_read, &read_length);
+  PBL_ASSERTN(result);
+  PBL_ASSERT(read_length == sizeof(uint8_t), "read_length %u sizeof(uint8_t) %u", read_length, sizeof(uint8_t));
+  uint8_t msg_length = *data_read;
+
+  if (shared_circular_buffer_get_read_space_remaining(&s_buffer, &s_buffer_client) < msg_length + sizeof(uint8_t)) {
+    return; // Not ready yet, consume nothing.
+  }
+
+  // Flash_logging_log_start can trigger a flash erase. Release the buffer mutex
+  // to allow logging while the (slow) erase completes.
+  mutex_unlock(s_buffer_mutex);
+  uint32_t flash_addr = flash_logging_log_start(msg_length);
+  mutex_lock(s_buffer_mutex);
+  if (flash_addr == FLASH_LOG_INVALID_ADDR) {
+    return;
+  }
+
+  shared_circular_buffer_consume(&s_buffer, &s_buffer_client, read_length);
+
+  while (msg_length > 0) {
+    mutex_unlock(s_buffer_mutex);
+
+    // Note that this buffer read really should be done with the buffer mutex held.
+    // This works only because writes to the buffer do not advance slackers.
+    result = shared_circular_buffer_read(&s_buffer, &s_buffer_client, msg_length, &data_read, &read_length);
+    PBL_ASSERTN(result);
+    msg_length -= read_length;
+
+    flash_logging_write(data_read, flash_addr, read_length);
+    flash_addr += read_length;
+
+    mutex_lock(s_buffer_mutex);
+    shared_circular_buffer_consume(&s_buffer, &s_buffer_client, read_length);
+  }
+
+  // We should still be holding the buffer lock here...
+}
+
+static void handle_buffer_sync(void *data) {
+  const bool is_async = (uintptr_t) data;
+
+  mutex_lock(s_flash_write_mutex);
+  mutex_lock(s_buffer_mutex);
+
+  while (shared_circular_buffer_get_read_space_remaining(&s_buffer, &s_buffer_client) > 0) {
+    write_message();
+    // The above function mucks with the mutex
+    mutex_assert_held_by_curr_task(s_buffer_mutex, true /* is_held */);
+  }
+
+  if (is_async) {
+    s_is_flash_write_scheduled = false;
+  }
+
+  mutex_unlock(s_buffer_mutex);
+  mutex_unlock(s_flash_write_mutex);
+}
+
+
+void advanced_logging_init(void) {
+  flash_logging_init();
+
+  shared_circular_buffer_init(&s_buffer, s_buffer_storage, sizeof(s_buffer_storage));
+  shared_circular_buffer_add_client(&s_buffer, &s_buffer_client);
+
+  s_buffer_mutex = mutex_create();
+  s_flash_write_mutex = mutex_create();
+}
+
+// Return true on success
+static bool write_buffer_locking(char* buffer, int length, bool async) {
+  bool success = false;
+
+  do {
+    mutex_lock(s_buffer_mutex);
+    if (shared_circular_buffer_get_write_space_remaining(&s_buffer) >= length + 1) {
+      // Ideally we could figure out a way to skip out on this copy but then you'd potentially need to sniprintf
+      // into a non-contiguous buffer... whatever, we have CPU to burn.
+      uint8_t msg_length = length;
+
+      // Do not advance slackers. Data loss and/or corruption will occur! See write_message()
+      shared_circular_buffer_write(&s_buffer, &msg_length, sizeof(uint8_t), false /*advance_slackers*/);
+      shared_circular_buffer_write(&s_buffer, (const uint8_t*) buffer, length, false /*advance_slackers*/);
+
+      success = true;
+    }
+    mutex_unlock(s_buffer_mutex);
+
+    // If we failed to buffer this message, flush the buffer to cache to make room.
+    // Otherwise, if this is a sync message, flush this message to flash.
+    if (!success || !async) {
+      handle_buffer_sync((void *)(uintptr_t) false /* !is_async */);
+    }
+  } while (!success); // Loop until the buffer copy succeeds. If sync, also wait until this message
+                      // is written to flash.
+                      // It's highly unlikely that another task will win the race and completely
+                      // fill the buffer between the flash write and the next buffer write attempt.
+                      // If so, there are bigger issues.
+
+  if (async) {
+    mutex_lock(s_buffer_mutex);
+    if (!s_is_flash_write_scheduled) {
+      s_is_flash_write_scheduled = true;
+      system_task_add_callback(handle_buffer_sync, (void *)(uintptr_t) true /* is_async */);
+    }
+    mutex_unlock(s_buffer_mutex);
+  }
+
+  return success;
+}
+
+void pbl_log_advanced(char* buffer, int length, bool async) {
+  if (s_buffer_mutex == INVALID_MUTEX_HANDLE) {
+    return;
+  }
+  write_buffer_locking(buffer, length, async);
+}
+
+char pbl_log_get_level_char(const uint8_t log_level) {
+  switch (log_level) {
+  case LOG_LEVEL_ALWAYS:
+    return '*';
+  case LOG_LEVEL_ERROR:
+    return 'E';
+  case LOG_LEVEL_WARNING:
+    return 'W';
+  case LOG_LEVEL_INFO:
+    return 'I';
+  case LOG_LEVEL_DEBUG:
+    return 'D';
+  case LOG_LEVEL_DEBUG_VERBOSE:
+    return 'V';
+  default:
+    return '?';
+  }
+}
+

src/fw/debug/advanced_logging.h

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+
+void advanced_logging_init(void);
+
+void pbl_log_advanced(const char* buffer, int length, bool async);
+

src/fw/debug/app_logging.c

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "util/attributes.h"
+#include "system/logging.h"
+#include "applib/app_logging.h"
+
+#include <stdint.h>
+
+#include "kernel/logging_private.h"
+#include "kernel/memory_layout.h"
+#include "kernel/util/stack_info.h"
+#include "services/common/comm_session/session.h"
+#include "syscall/syscall_internal.h"
+
+static const uint16_t APP_LOGGING_ENDPOINT = 2006;
+
+static AppLoggingMode s_app_logging_mode = AppLoggingDisabled;
+
+static const uint32_t MIN_STACK_FOR_SEND_DATA = 400;
+
+DEFINE_SYSCALL(void, sys_app_log, size_t length, void *log_buffer) {
+  if (PRIVILEGE_WAS_ELEVATED) {
+    syscall_assert_userspace_buffer(log_buffer, length);
+  }
+
+  AppLogBinaryMessage *message = log_buffer;
+
+  // First log to serial, we always do this.
+  kernel_pbl_log_serial(&message->log_msg, false);
+
+  // Now check to see if app logging is enabled over bluetooth.
+  if (s_app_logging_mode == AppLoggingDisabled) {
+    return;
+  }
+
+  // Then log to the app logging endpoint (if we have enough stack space)
+  uint32_t stack_space = stack_free_bytes();
+  if (stack_space > MIN_STACK_FOR_SEND_DATA) {
+    CommSession *session = comm_session_get_system_session();
+    if (session) {
+      comm_session_send_data(session, APP_LOGGING_ENDPOINT, (uint8_t*)log_buffer, length, COMM_SESSION_DEFAULT_TIMEOUT);
+    }
+  }
+}
+
+void app_log_protocol_msg_callback(CommSession *session, const uint8_t *data, const size_t length) {
+  typedef struct PACKED AppLogCommand {
+    uint8_t commandType;
+  } AppLogCommand;
+
+  enum AppLogCommandType {
+    APP_LOG_COMMAND_DISABLE_LOGGING = 0,
+    APP_LOG_COMMAND_ENABLE_LOGGING = 1,
+  };
+
+  AppLogCommand *command = (AppLogCommand *)data;
+  switch(command->commandType) {
+  case APP_LOG_COMMAND_ENABLE_LOGGING:
+    s_app_logging_mode = AppLoggingEnabled;
+    break;
+  case APP_LOG_COMMAND_DISABLE_LOGGING:
+    s_app_logging_mode = AppLoggingDisabled;
+    break;
+  default:
+    PBL_LOG(LOG_LEVEL_WARNING, "Invalid app log command 0x%x", command->commandType);
+  }
+}
+

src/fw/debug/debug.c

@@ -0,0 +1,227 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "debug.h"
+#include "advanced_logging.h"
+
+#include "flash_logging.h"
+#include "debug_reboot_reason.h"
+
+#include "drivers/watchdog.h"
+#include "flash_region/flash_region.h"
+#include "kernel/events.h"
+#include "kernel/logging_private.h"
+#include "kernel/pbl_malloc.h"
+#include "kernel/pebble_tasks.h"
+#include "mfg/mfg_serials.h"
+#include "process_management/app_manager.h"
+#include "services/common/analytics/analytics.h"
+#include "services/common/comm_session/session.h"
+#include "services/common/comm_session/session_send_buffer.h"
+#include "services/common/system_task.h"
+#include "services/normal/process_management/app_storage.h"
+#include "system/bootbits.h"
+#include "system/hexdump.h"
+#include "system/logging.h"
+#include "system/passert.h"
+#include "system/reboot_reason.h"
+#include "system/version.h"
+#include "util/attributes.h"
+#include "util/build_id.h"
+
+#include <inttypes.h>
+
+static const uint16_t ENDPOINT_ID = 2002;
+
+typedef struct PACKED {
+  uint8_t command;
+  uint32_t cookie;
+} BluetoothHeader;
+
+typedef struct BluetoothDumpLineCallbackData {
+  bool in_progress;
+  CommSession *comm_session;
+  int generation;
+  uint32_t cookie;
+} BluetoothDumpLineCallbackData;
+
+BluetoothDumpLineCallbackData s_bt_dump_chunk_callback_data;
+
+static void prv_put_status_event(DebugInfoEventState state) {
+  PebbleEvent event = {
+    .type = PEBBLE_GATHER_DEBUG_INFO_EVENT,
+    .debug_info = {
+      .source = DebugInfoSourceFWLogs,
+      .state = state,
+    },
+  };
+  event_put(&event);
+}
+
+static bool prv_bt_log_dump_line_cb(uint8_t *message, uint32_t total_length) {
+  CommSession *session = s_bt_dump_chunk_callback_data.comm_session;
+
+  // keep us sending data quickly
+  comm_session_set_responsiveness(
+      s_bt_dump_chunk_callback_data.comm_session, BtConsumerPpLogDump, ResponseTimeMin, 5);
+
+  const uint16_t required_length = total_length + 1 + 4;
+  SendBuffer *sb = comm_session_send_buffer_begin_write(session, ENDPOINT_ID, required_length,
+                                                        COMM_SESSION_DEFAULT_TIMEOUT);
+  if (!sb) {
+    PBL_LOG(LOG_LEVEL_DEBUG, "Failed to get send buffer");
+    return false;
+  }
+
+  BluetoothHeader header = {
+    .command = 0x80,
+    .cookie = s_bt_dump_chunk_callback_data.cookie,
+  };
+  comm_session_send_buffer_write(sb, (const uint8_t *) &header, sizeof(header));
+  comm_session_send_buffer_write(sb, message, total_length);
+  comm_session_send_buffer_end_write(sb);
+  return true;
+}
+
+// Called by flash_dump_log_file() when the log has been completely dumped
+static void prv_bt_log_dump_completed_cb(bool success) {
+  BluetoothHeader header = {
+    .cookie = s_bt_dump_chunk_callback_data.cookie
+  };
+  // Send a "no logs" message if the generation did not exist and the remote supports
+  // "infinite log dumping"
+  CommSession *session = s_bt_dump_chunk_callback_data.comm_session;
+  if (!success && comm_session_has_capability(session, CommSessionInfiniteLogDumping)) {
+    header.command = 0x82;
+    comm_session_send_data(s_bt_dump_chunk_callback_data.comm_session, ENDPOINT_ID,
+                           (uint8_t *) &header, sizeof(header),
+                           COMM_SESSION_DEFAULT_TIMEOUT);
+  } else {
+    // Otherwise, just send a "done" message
+    header.command = 0x81;
+    comm_session_send_data(s_bt_dump_chunk_callback_data.comm_session, ENDPOINT_ID,
+                           (uint8_t *) &header, sizeof(header), COMM_SESSION_DEFAULT_TIMEOUT);
+  }
+
+  s_bt_dump_chunk_callback_data.in_progress = false;
+
+  // Ok to enter a lower power less responsive state
+  comm_session_set_responsiveness(
+      s_bt_dump_chunk_callback_data.comm_session, BtConsumerPpLogDump, ResponseTimeMax, 0);
+  prv_put_status_event(DebugInfoStateFinished);
+}
+
+static void prv_flash_logging_bluetooth_dump(
+    CommSession *session, int generation, uint32_t cookie) {
+  PBL_ASSERT_RUNNING_FROM_EXPECTED_TASK(PebbleTask_KernelBackground);
+  if (s_bt_dump_chunk_callback_data.in_progress) {
+    PBL_LOG(LOG_LEVEL_ERROR, "Already in the middle of dumping logs");
+    return;
+  }
+
+  prv_put_status_event(DebugInfoStateStarted);
+
+  // Temporarily disable logging so we don't log forever.
+  flash_logging_set_enabled(false);
+
+  s_bt_dump_chunk_callback_data.in_progress = true;
+  s_bt_dump_chunk_callback_data.generation = generation;
+  s_bt_dump_chunk_callback_data.comm_session = session;
+  s_bt_dump_chunk_callback_data.cookie = cookie;
+
+  flash_dump_log_file(s_bt_dump_chunk_callback_data.generation, prv_bt_log_dump_line_cb,
+                      prv_bt_log_dump_completed_cb);
+  flash_logging_set_enabled(true);
+}
+
+void dump_log_protocol_msg_callback(CommSession *session, const uint8_t* data, size_t length) {
+  uint32_t cookie;
+  int generation = 0;
+  if (data[0] == 0x10 || data[0] == 0x11) {
+    if (length != 6) {
+      PBL_LOG(LOG_LEVEL_ERROR, "Invalid dump log message received -- length %u", length);
+      return;
+    }
+
+    generation = data[1];
+    cookie = *((uint32_t*) (data + 2));
+  } else {
+    if (length != 5) {
+      PBL_LOG(LOG_LEVEL_ERROR, "Invalid dump log message received -- length %u", length);
+      return;
+    }
+
+    cookie = *((uint32_t*) (data + 1));
+  }
+
+  switch (*data) {
+  case 0x00:
+    prv_flash_logging_bluetooth_dump(session, 0, cookie);
+    break;
+  case 0x01:
+    prv_flash_logging_bluetooth_dump(session, 1, cookie);
+    break;
+  case 0x10:
+    prv_flash_logging_bluetooth_dump(session, generation, cookie);
+    break;
+  case 0x02:
+  case 0x03:
+  case 0x11:
+    break;
+  }
+}
+
+void debug_init(McuRebootReason mcu_reboot_reason) {
+  advanced_logging_init();
+
+  // Log the firmware version in the first flash log line:
+  DEBUG_LOG(LOG_LEVEL_INFO, "%s (platform: %u, hw: %s, sn: %s, pcba: %s)",
+      TINTIN_METADATA.version_tag,
+      TINTIN_METADATA.hw_platform,
+      mfg_get_hw_version(),
+      mfg_get_serial_number(),
+      mfg_get_pcba_serial_number());
+
+  // Log the firmware build id to flash:
+  char build_id_string[64];
+  version_copy_current_build_id_hex_string(build_id_string, 64);
+  DEBUG_LOG(LOG_LEVEL_INFO, "BUILD ID: %s", build_id_string);
+
+  debug_reboot_reason_print(mcu_reboot_reason);
+}
+
+void debug_print_last_launched_app(void) {
+  // Get the slot of the last launched app
+  // so we know what was running when we rebooted
+  uint32_t last_launched_app_slot = reboot_get_slot_of_last_launched_app();
+
+  // check if last app launched was a system app
+  if (last_launched_app_slot == (uint32_t)SYSTEM_APP_BANK_ID) {
+    DEBUG_LOG(LOG_LEVEL_INFO, "Last launched app: <System_App>");
+  } else if ((last_launched_app_slot != (uint32_t)INVALID_BANK_ID)) {
+    PebbleProcessInfo last_launched_app;
+    uint8_t build_id[BUILD_ID_EXPECTED_LEN];
+    AppStorageGetAppInfoResult result = app_storage_get_process_info(&last_launched_app,
+                                                                     build_id,
+                                                                     (AppInstallId)last_launched_app_slot,
+                                                                     PebbleTask_App);
+
+    if (result == GET_APP_INFO_SUCCESS) {
+      DEBUG_LOG(LOG_LEVEL_INFO, "Last launched app: %s", last_launched_app.name);
+      PBL_HEXDUMP(LOG_LEVEL_INFO, build_id, sizeof(build_id));
+    }
+  }
+}

src/fw/debug/debug.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "drivers/mcu_reboot_reason.h"
+#include <stdint.h>
+
+// TODO: Eventually move debug logging back to hashed logging
+// Currently broken out to directly log strings without hashing
+#ifdef PBL_LOG_ENABLED
+  #define DEBUG_LOG(level, fmt, ...) \
+      pbl_log(level, __FILE__, __LINE__, fmt, ## __VA_ARGS__)
+#else
+  #define DEBUG_LOG(level, fmt, ...)
+#endif
+
+void debug_init(McuRebootReason reason);
+
+void debug_print_last_launched_app(void);
+

src/fw/debug/debug_reboot_reason.c

@@ -0,0 +1,207 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "debug.h"
+#include "advanced_logging.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "comm/ble/gatt_service_changed.h"
+#include "drivers/pmic.h"
+#include "kernel/core_dump.h"
+#include "kernel/event_loop.h"
+#include "kernel/pbl_malloc.h"
+#include "popups/crashed_ui.h"
+#include "services/common/analytics/analytics.h"
+#include "system/logging.h"
+#include "system/reboot_reason.h"
+
+static void log_reboot_reason_cb(void *reason) {
+  AnalyticsEventCrash *crash_report = (AnalyticsEventCrash *)reason;
+  analytics_event_crash(crash_report->crash_code, crash_report->link_register);
+  kernel_free(crash_report);
+}
+
+static RebootReasonCode s_last_reboot_reason_code = RebootReasonCode_Unknown;
+RebootReasonCode reboot_reason_get_last_reboot_reason(void) {
+  return s_last_reboot_reason_code;
+}
+
+void debug_reboot_reason_print(McuRebootReason mcu_reboot_reason) {
+  RebootReason reason;
+  reboot_reason_get(&reason);
+  bool show_reset_alert = !reason.restarted_safely;
+  s_last_reboot_reason_code = reason.code;
+
+  // We're out of flash space, scrape a few bytes back!
+  static const char* rebooted_due_to = " rebooted due to ";
+
+  const char* restarted_safely_string = "Safely";
+  if (!reason.restarted_safely) {
+    restarted_safely_string = "Dangerously";
+  }
+
+  // Keep hourly logging to keep track of hours without crashes.
+  analytics_set(ANALYTICS_DEVICE_METRIC_SYSTEM_CRASH_CODE,
+                0xDEAD0000 | reason.code, AnalyticsClient_System);
+  uint32_t lr = reason.extra;
+
+  // Leave this NULL to do your own printing.
+  const char *reason_string = NULL;
+  switch (reason.code) {
+  // Normal stuff
+  case RebootReasonCode_Unknown:
+    reason_string = "We don't know why we %s rebooted.";
+    lr = mcu_reboot_reason.reset_mask;
+    break;
+  case RebootReasonCode_LowBattery:
+    reason_string = "%s%sLowBattery";
+    break;
+  case RebootReasonCode_SoftwareUpdate:
+    gatt_service_changed_server_handle_fw_update();
+    reason_string = "%s%sSoftwareUpdate";
+    break;
+  case RebootReasonCode_ResetButtonsHeld:
+    // Since we forced the reset, it isn't unexpected
+    show_reset_alert = false;
+    reason_string = "%s%sResetButtonsHeld";
+    break;
+  case RebootReasonCode_ShutdownMenuItem:
+    reason_string = "%s%sLowBattery";
+    break;
+  case RebootReasonCode_FactoryResetReset:
+    reason_string = "%s%sFactoryResetReset";
+    break;
+  case RebootReasonCode_FactoryResetShutdown:
+    reason_string = "%s%sFactoryResetShutdown";
+    break;
+  case RebootReasonCode_MfgShutdown:
+    reason_string = "%s%sMfgShutdown";
+    break;
+  case RebootReasonCode_Serial:
+    reason_string = "%s%sSerial";
+    break;
+  case RebootReasonCode_RemoteReset:
+    reason_string = "%s%sa Remote Reset";
+    break;
+  case RebootReasonCode_ForcedCoreDump:
+    reason_string = "%s%sa Forced Coredump";
+    break;
+  case RebootReasonCode_PrfIdle:
+    reason_string = "%s%sIdle PRF";
+    break;
+  // Error occurred
+  case RebootReasonCode_Assert:
+    show_reset_alert = true;
+    reason_string = "%s%sAssert: LR %#"PRIxPTR;
+    break;
+  case RebootReasonCode_HardFault:
+    show_reset_alert = true;
+    reason_string = "%s%sHardFault: LR %#"PRIxPTR;
+    break;
+  case RebootReasonCode_LauncherPanic:
+    show_reset_alert = true;
+    reason_string = "%s%sLauncherPanic: code 0x%"PRIx32;
+    break;
+  case RebootReasonCode_ClockFailure:
+    reason_string = "%s%sClock Failure";
+    break;
+  case RebootReasonCode_WorkerHardFault:
+    show_reset_alert = true;
+    reason_string = "%s%sWorker HardFault";
+    break;
+  case RebootReasonCode_OutOfMemory:
+    show_reset_alert = true;
+    reason_string = "%s%sOOM";
+    break;
+  case RebootReasonCode_BtCoredump:
+    show_reset_alert = true;
+    reason_string = "%s%sBT Coredump";
+    break;
+  default:
+    reason_string = "%s%sUnrecognized Reason";
+    break;
+  // Error occurred
+  case RebootReasonCode_Watchdog:
+    show_reset_alert = true;
+    DEBUG_LOG(LOG_LEVEL_INFO, "%s%sWatchdog: Bits 0x%" PRIx8 ", Mask 0x%" PRIx8,
+              restarted_safely_string, rebooted_due_to, reason.data8[0], reason.data8[1]);
+
+    if (reason.watchdog.stuck_task_pc != 0) {
+      DEBUG_LOG(LOG_LEVEL_INFO, "Stuck task PC: 0x%" PRIx32 ", LR: 0x%" PRIx32,
+                reason.watchdog.stuck_task_pc, reason.watchdog.stuck_task_lr);
+
+      if (reason.watchdog.stuck_task_callback) {
+        DEBUG_LOG(LOG_LEVEL_INFO, "Stuck callback: 0x%" PRIx32,
+                  reason.watchdog.stuck_task_callback);
+      }
+    }
+    break;
+  case RebootReasonCode_StackOverflow:
+    show_reset_alert = true;
+    PebbleTask task = (PebbleTask) reason.data8[0];
+    DEBUG_LOG(LOG_LEVEL_INFO, "%s%sStackOverflow: Task #%d (%s)", restarted_safely_string,
+              rebooted_due_to, task, pebble_task_get_name(task));
+    break;
+  case RebootReasonCode_EventQueueFull:
+    show_reset_alert = true;
+    DEBUG_LOG(LOG_LEVEL_INFO, "%s%sEvent Queue Full", restarted_safely_string, rebooted_due_to);
+    DEBUG_LOG(LOG_LEVEL_INFO, "Task: <%s> LR: 0x%"PRIx32" Current: 0x%"PRIx32" Dropped: 0x%"PRIx32,
+              pebble_task_get_name(reason.event_queue.destination_task),
+              reason.event_queue.push_lr,
+              reason.event_queue.current_event,
+              reason.event_queue.dropped_event);
+    break;
+  }
+  // Generic reason string
+  if (reason_string) {
+    DEBUG_LOG(LOG_LEVEL_INFO, reason_string, restarted_safely_string, rebooted_due_to,
+              reason.extra);
+  }
+
+  analytics_set(ANALYTICS_DEVICE_METRIC_SYSTEM_CRASH_LR, lr, AnalyticsClient_System);
+
+  // We need to wait for the logging service to initialize.
+  AnalyticsEventCrash *crash_report = kernel_malloc_check(sizeof(AnalyticsEventCrash));
+  *crash_report = (AnalyticsEventCrash) {
+    .crash_code = reason.code,
+    .link_register = lr
+  };
+  launcher_task_add_callback(log_reboot_reason_cb, crash_report);
+
+  if (is_unread_coredump_available()) {
+    DEBUG_LOG(LOG_LEVEL_INFO, "Unread coredump file is present!");
+  }
+
+  DEBUG_LOG(LOG_LEVEL_INFO, "MCU reset reason mask: 0x%x", (int)mcu_reboot_reason.reset_mask);
+#if CAPABILITY_HAS_PMIC
+  uint32_t pmic_reset_reason = pmic_get_last_reset_reason();
+  if (pmic_reset_reason != 0) {
+    DEBUG_LOG(LOG_LEVEL_INFO, "PMIC reset reason mask: 0x%x", (int)pmic_reset_reason);
+  }
+#endif
+
+#ifdef SHOW_PEBBLE_JUST_RESET_ALERT
+  // Trigger an alert display so that the user knows the watch rebooted due to a crash. This event
+  // will be caught and handled by the launcher.c event loop.
+  if (show_reset_alert) {
+    crashed_ui_show_pebble_reset();
+  }
+#endif
+
+  reboot_reason_clear();
+}

src/fw/debug/debug_reboot_reason.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "drivers/mcu_reboot_reason.h"
+
+void debug_reboot_reason_print(McuRebootReason mcu_reboot_reason);

src/fw/debug/default/flash_logging.c

@@ -0,0 +1,574 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "debug/flash_logging.h"
+
+#include "drivers/flash.h"
+#include "flash_region/flash_region.h"
+#include "kernel/pbl_malloc.h"
+#include "services/common/system_task.h"
+#include "syscall/syscall.h"
+#include "system/logging.h"
+#include "system/passert.h"
+#include "system/version.h"
+#include "util/attributes.h"
+#include "util/build_id.h"
+#include "util/size.h"
+#include "util/string.h"
+
+#include <inttypes.h>
+#include <stdio.h>
+
+// Notes:
+//
+// This implements a simple circular logging scheme format.
+//
+// The only assumption it makes is that you have at least two eraseable flash
+// units. However, the more units (i.e sectors) that you have, the smaller % of
+// logs that will be erased when the log buffer fills.
+//
+// On each boot, we create a file to hold all the messages for that boot. This
+// file is called a log generation or log.
+//
+// Within each eraseable unit multiple 'pages' exist. A log generation can span
+// one or more pages. Multiple log generations can be stored at any given
+// time. The oldest pages will be removed as the log buffer wraps around.
+//
+// Since our logging routines call into this module, we should NOT have any
+// PBL_LOGs in this file, else you could generate infinite loops!
+
+// Configuration Defines
+#define LOG_REGION_SIZE (FLASH_REGION_DEBUG_DB_END - FLASH_REGION_DEBUG_DB_BEGIN)
+#define ERASE_UNIT_SIZE (FLASH_DEBUG_DB_BLOCK_SIZE)
+
+#define DEFAULT_LOG_PAGE_SIZE (0x2000)
+#if ERASE_UNIT_SIZE < DEFAULT_LOG_PAGE_SIZE
+#define LOG_PAGE_SIZE ERASE_UNIT_SIZE
+#else
+#define LOG_PAGE_SIZE DEFAULT_LOG_PAGE_SIZE
+#endif
+
+#define MAX_POSSIBLE_LOG_GENS (LOG_REGION_SIZE / LOG_PAGE_SIZE)
+
+static bool s_flash_logging_enabled = false;
+
+typedef struct PACKED {
+  uint32_t  magic;
+  uint8_t   version;
+  uint8_t   build_id[BUILD_ID_EXPECTED_LEN];
+  uint8_t   log_file_id;
+  uint8_t   log_chunk_id; // For a given log file, the id of the page
+  uint8_t   log_flags; // this should be the last header field written
+} FlashLoggingHeader;
+
+// indicates the region is erased and no logs are stored in it
+#define LOG_MAGIC_PAGE_FREE 0xffffffff
+#define LOG_MAGIC   0x21474F4C /* LOG! */
+#define LOG_VERSION 0x1
+
+typedef struct PACKED {
+  uint8_t flags;
+  uint8_t length;
+} LogRecordHeader;
+
+#define LOG_FLAGS_VALID (0x1 << 0)
+
+typedef struct {
+  uint32_t page_start_addr; // absolute start addr of the page we are logging to
+  uint32_t offset_in_log_page; // the offset we writing to in a given page
+  uint32_t log_start_addr; // the starting address of the curr log being written
+  uint8_t  bytes_remaining; // the bytes left to write for the current log
+  uint8_t  log_chunk_id;  // the id of the current page being logged to
+  uint8_t  log_file_id; // the id of the current log generation
+} CurrentLoggingState;
+
+static CurrentLoggingState s_curr_state;
+
+#define CHUNK_ID_BITWIDTH (sizeof(((FlashLoggingHeader *)0)->log_chunk_id) * 8)
+#define LOG_ID_BITWIDTH   (sizeof(((FlashLoggingHeader *)0)->log_file_id) * 8)
+#define MAX_LOG_FILE_ID   (0x1UL << LOG_ID_BITWIDTH)
+#define MAX_PAGE_CHUNK_ID (0x1UL << CHUNK_ID_BITWIDTH)
+
+// we use 0xff... to indicate an unpopulated msg so define max msg len to be 1
+// less than that
+#define MAX_MSG_LEN  ((0x1UL << sizeof(((LogRecordHeader *)0)->length) * 8) - 2)
+
+// This is the state used while performing flash_log_file(). Each log message gets handled
+// by a separate system task callback
+typedef struct {
+  uint8_t   page_index;               // which page we are currently dumping
+  uint8_t   num_pages;                // number of pages to dump
+  uint8_t   retry_count;              // How many retries we have performed at this offset
+  bool      sent_build_id;            // True after we've sent the build ID
+  uint16_t  page_offset;              // current offset within the page
+  uint32_t  log_start_addr;           // start address of the log file we are dumping
+  DumpLineCallback  line_cb;          // Called to send each line
+  DumpCompletedCallback completed_cb; // Called when completed
+  uint8_t   msg_buf[MAX_MSG_LEN];     // Message buffer
+} DumpLogState;
+
+#define DUMP_LOG_MAX_RETRIES          3
+
+typedef enum {
+  DumpStatus_DoneFailure,
+  DumpStatus_InProgress,
+  DumpStatus_DoneSuccess,
+} DumpStatus;
+
+
+// Static asserts to make sure user has configured flash logging correctly for
+// the platform of interest
+
+_Static_assert((MAX_POSSIBLE_LOG_GENS >= 4) &&
+    (MAX_POSSIBLE_LOG_GENS < MAX_LOG_FILE_ID),
+    "Invalid number of log generation numbers");
+_Static_assert(MAX_POSSIBLE_LOG_GENS < MAX_PAGE_CHUNK_ID,
+    "Invalid number of chunk ids for serial distance to work");
+_Static_assert((LOG_REGION_SIZE / ERASE_UNIT_SIZE) >= 2,
+    "Need to have at least 2 eraseable units for flash logging to work");
+_Static_assert((LOG_REGION_SIZE % LOG_PAGE_SIZE) == 0,
+    "The log page size must be divisible by the log region size");
+_Static_assert(((FLASH_REGION_DEBUG_DB_END % ERASE_UNIT_SIZE) == 0) &&
+    ((FLASH_REGION_DEBUG_DB_END % ERASE_UNIT_SIZE) == 0),
+    "Space for flash logging must be aligned on an erase region boundary");
+_Static_assert(LOG_PAGE_SIZE <= ERASE_UNIT_SIZE,
+     "Log pages must fit within an erase unit");
+_Static_assert((ERASE_UNIT_SIZE % LOG_PAGE_SIZE) == 0,
+     "The log page size must be divisible by the erase unit size");
+
+//! Given the current address and amount to increment it by, handles wrapping
+//! and computes the valid flash address
+static uint32_t prv_get_page_addr(uint32_t curr_page_addr, uint32_t incr_by) {
+  uint32_t new_offset =
+      ((curr_page_addr - FLASH_REGION_DEBUG_DB_BEGIN) + incr_by) %
+      LOG_REGION_SIZE;
+  return (new_offset + FLASH_REGION_DEBUG_DB_BEGIN);
+}
+
+//! Given the header magic and version, returns true if the log is valid
+static bool prv_flash_log_valid(const FlashLoggingHeader *hdr) {
+  return (hdr->magic == LOG_MAGIC && hdr->version == LOG_VERSION);
+}
+
+static uint8_t prv_get_next_log_file_id(uint8_t file_id) {
+  return (file_id + 1) % MAX_LOG_FILE_ID;
+}
+
+static uint32_t prv_get_unit_base_address(uint32_t addr) {
+#if PLATFORM_SNOWY || PLATFORM_SPALDING
+  return flash_get_sector_base_address(addr);
+#elif PLATFORM_SILK || PLATFORM_CALCULUS || PLATFORM_ROBERT
+  return flash_get_subsector_base_address(addr);
+#else
+#error "Invalid platform!"
+#endif
+}
+
+static void prv_erase_unit(uint32_t addr) {
+#if PLATFORM_SNOWY || PLATFORM_SPALDING
+  flash_erase_sector_blocking(addr);
+#elif PLATFORM_SILK || PLATFORM_CALCULUS || PLATFORM_ROBERT
+  flash_erase_subsector_blocking(addr);
+#else
+#error "Invalid platform!"
+#endif
+}
+
+static void prv_format_flash_logging_region(void) {
+  uint32_t sector_addr;
+  for (sector_addr = FLASH_REGION_DEBUG_DB_BEGIN;
+       sector_addr < FLASH_REGION_DEBUG_DB_END; sector_addr += ERASE_UNIT_SIZE) {
+    prv_erase_unit(sector_addr);
+  }
+}
+
+static uint8_t generation_to_log_file_id(int generation) {
+  int log_id = (int)s_curr_state.log_file_id - generation;
+
+  if (log_id < 0) {
+    log_id += MAX_LOG_FILE_ID;
+  }
+
+  return (log_id);
+}
+
+//! Scans the flash log region and checks the FlashLoggingHeader magic and
+//! version of each log page for validity. If any header looks completely bogus,
+//! we format the log region to put us back into a known state
+//!
+//! @return addr of the first active section
+static uint32_t prv_validate_flash_log_region(uint8_t *first_log_file_id) {
+  uint32_t first_used_region = UINT32_MAX;
+
+  for (uint32_t offset = 0; offset < LOG_REGION_SIZE; offset += LOG_PAGE_SIZE) {
+    uint32_t flash_addr = FLASH_REGION_DEBUG_DB_BEGIN + offset;
+
+    FlashLoggingHeader hdr;
+    flash_read_bytes((uint8_t *)&hdr, flash_addr, sizeof(hdr));
+
+    if (!prv_flash_log_valid(&hdr)) { // is the region erased ?
+      FlashLoggingHeader erased_hdr;
+      memset(&erased_hdr, 0xff, sizeof(erased_hdr));
+      bool region_erased = (memcmp(&erased_hdr, &hdr, sizeof(hdr)) == 0);
+      if (!region_erased) { // unrecognized format, erase everything
+        prv_format_flash_logging_region();
+        return (UINT32_MAX); // no region in use after formatting
+      }
+    } else if (first_used_region == UINT32_MAX) {
+      first_used_region = offset;
+      *first_log_file_id = hdr.log_file_id;
+    }
+  }
+  return (first_used_region);
+}
+
+//! @param log_file_id - the id of the log file to find
+//! @param[out] start_page_addr - the address of the page the log starts on
+//! @return the number of pages in the log file requested or 0 if no
+//!   file is found
+static int prv_get_start_of_log_file(uint8_t log_file_id,
+    uint32_t *start_page_addr) {
+  uint8_t num_log_pages = 0;
+  uint8_t prev_chunk_id = 0;
+  uint32_t log_start_addr = FLASH_LOG_INVALID_ADDR;
+
+  for (uint32_t offset = 0; offset < LOG_REGION_SIZE; offset += LOG_PAGE_SIZE) {
+    uint32_t flash_addr = FLASH_REGION_DEBUG_DB_BEGIN + offset;
+
+    FlashLoggingHeader hdr;
+    flash_read_bytes((uint8_t *)&hdr, flash_addr, sizeof(hdr));
+
+    bool in_use_and_valid = prv_flash_log_valid(&hdr);
+
+    // if the page is not in use or the log id is not for the generation we
+    // are searching for, continue looking
+    if (!in_use_and_valid || (hdr.log_file_id != log_file_id)) {
+      continue;
+    }
+
+    num_log_pages++;
+
+    int32_t dist = serial_distance(prev_chunk_id, hdr.log_chunk_id,
+        LOG_ID_BITWIDTH);
+
+    if (log_start_addr == FLASH_LOG_INVALID_ADDR) {
+      // this is the first page we've found
+      log_start_addr = flash_addr;
+      dist = 0; // nothing else to compare against yet
+    }
+
+    if ((dist == 0) || (dist == 1)) {
+      prev_chunk_id = hdr.log_chunk_id;
+      continue; // keep looking
+    }
+
+    // we have found a gap in the number sequence which means we have found
+    // the beginning and end of the log generation. We must continue looping
+    // to make sure we count the total number of pages
+    prev_chunk_id = hdr.log_chunk_id;
+    log_start_addr = flash_addr;
+  }
+
+  *start_page_addr = log_start_addr;
+  return (num_log_pages);
+}
+
+//! Update page in flash to reflect settings of s_curr_state
+static void prv_allocate_page_for_use(void) {
+  FlashLoggingHeader hdr;
+
+  hdr.magic = LOG_MAGIC;
+  hdr.version = LOG_VERSION;
+  hdr.log_file_id = s_curr_state.log_file_id;
+  hdr.log_chunk_id = s_curr_state.log_chunk_id;
+  hdr.log_flags = ~(LOG_FLAGS_VALID);
+  s_curr_state.log_chunk_id++;
+
+  size_t len;
+  const uint8_t *build_id = version_get_build_id(&len);
+  memcpy(hdr.build_id, build_id, sizeof(hdr.build_id));
+
+  flash_write_bytes((uint8_t *)&hdr, s_curr_state.page_start_addr, sizeof(hdr));
+  s_curr_state.offset_in_log_page = sizeof(hdr);
+}
+
+void flash_logging_set_enabled(bool enabled) {
+  s_flash_logging_enabled = enabled;
+}
+
+void flash_logging_init(void) {
+  s_curr_state = (CurrentLoggingState){};
+
+  uint8_t prev_log_id = 0;
+  uint32_t first_used_region = prv_validate_flash_log_region(&prev_log_id);
+
+  if (first_used_region == UINT32_MAX) { // no logs exist so start at region 0
+    s_curr_state.page_start_addr = FLASH_REGION_DEBUG_DB_BEGIN;
+    goto done;
+  }
+
+  bool new_log_region_found = false;
+  bool multiple_gens_found = false;
+
+  for (uint32_t offset = 0; offset < LOG_REGION_SIZE; offset += LOG_PAGE_SIZE) {
+    uint32_t flash_addr = prv_get_page_addr(first_used_region, offset);
+
+    FlashLoggingHeader hdr;
+    flash_read_bytes((uint8_t *)&hdr, flash_addr, sizeof(hdr));
+
+    if (prv_flash_log_valid(&hdr)) {
+      // we use serial distance to find the gap in the numbering
+      int32_t dist = serial_distance(prev_log_id, hdr.log_file_id,
+        LOG_ID_BITWIDTH);
+
+      if ((dist == 0) || (dist == 1)) {
+        prev_log_id = hdr.log_file_id;
+        multiple_gens_found |= (dist != 0 && offset != 0);
+        continue; // keep looking
+      }
+
+      // we have found a page to use, but we need to erase the contents first
+      prv_erase_unit(prv_get_unit_base_address(flash_addr));
+    }
+
+    s_curr_state.log_file_id = prv_get_next_log_file_id(prev_log_id);
+    s_curr_state.page_start_addr = flash_addr;
+    new_log_region_found = true;
+    break;
+  }
+
+  // everything was in increasing order or there was only one log generation
+  // if there was only one log generation, we must find the oldest part of it
+  if (!new_log_region_found) {
+    if (multiple_gens_found || (prv_get_start_of_log_file(prev_log_id,
+        &s_curr_state.page_start_addr) == 0)) {
+      s_curr_state.page_start_addr = FLASH_REGION_DEBUG_DB_BEGIN;
+    }
+
+    s_curr_state.page_start_addr = prv_get_unit_base_address(s_curr_state.page_start_addr);
+    prv_erase_unit(s_curr_state.page_start_addr);
+
+    s_curr_state.log_file_id = prv_get_next_log_file_id(prev_log_id);
+  }
+
+done: // we have allocated a region to be used
+  prv_allocate_page_for_use();
+  flash_logging_set_enabled(true);
+}
+
+//! Writes the log record header to flash and advances the
+//! s_curr_state.offset_in_log_page field
+//!
+//! @param msg_length - the length of the message to be written
+static void prv_write_flash_log_record_header(uint8_t msg_length) {
+  LogRecordHeader record_hdr;
+  memset((uint8_t *)&record_hdr, 0xff, sizeof(record_hdr));
+
+  record_hdr.length = msg_length;
+
+  uint32_t addr = s_curr_state.page_start_addr + s_curr_state.offset_in_log_page;
+  flash_write_bytes((uint8_t *)&record_hdr, addr, sizeof(record_hdr));
+  s_curr_state.offset_in_log_page += sizeof(record_hdr);
+}
+
+uint32_t flash_logging_log_start(uint8_t msg_length) {
+  if ((msg_length == 0) || (msg_length > MAX_MSG_LEN) ||
+      !s_flash_logging_enabled) {
+    return FLASH_LOG_INVALID_ADDR;
+  }
+
+  // bytes_remaining should always be 0, but if for some reason this gets called
+  // again, just skip onto the next record spot
+  s_curr_state.offset_in_log_page += s_curr_state.bytes_remaining;
+
+  uint32_t payload_size = sizeof(LogRecordHeader) + msg_length;
+  if ((s_curr_state.offset_in_log_page + payload_size) <= LOG_PAGE_SIZE) {
+    goto done; // there is enough space in the current page
+  }
+
+  // out of space, mark end of page
+  uint32_t new_flash_addr = prv_get_page_addr(s_curr_state.page_start_addr,
+      LOG_PAGE_SIZE);
+
+  uint32_t curr_sector = s_curr_state.page_start_addr / ERASE_UNIT_SIZE;
+  uint32_t new_sector = new_flash_addr / ERASE_UNIT_SIZE;
+  if (curr_sector != new_sector) { // have we crossed into a new erase region ?
+    prv_erase_unit(prv_get_unit_base_address(new_flash_addr));
+  }
+
+  s_curr_state.page_start_addr = new_flash_addr;
+  prv_allocate_page_for_use();
+
+done:
+  s_curr_state.log_start_addr = s_curr_state.offset_in_log_page +
+      s_curr_state.page_start_addr;
+  s_curr_state.bytes_remaining = msg_length;
+
+  prv_write_flash_log_record_header(msg_length);
+  return (s_curr_state.log_start_addr);
+}
+
+bool flash_logging_write(const uint8_t *data_to_write, uint32_t flash_addr,
+     uint32_t read_length) {
+
+  if ((s_curr_state.bytes_remaining < read_length) || !s_flash_logging_enabled) {
+    return (false);
+  }
+
+  uint32_t addr = s_curr_state.page_start_addr + s_curr_state.offset_in_log_page;
+  flash_write_bytes(data_to_write, addr, read_length);
+
+  s_curr_state.offset_in_log_page += read_length;
+  s_curr_state.bytes_remaining -= read_length;
+
+  if (s_curr_state.bytes_remaining == 0) {
+    // we are done with the current log record, mark it valid
+    uint8_t flags = ~(LOG_FLAGS_VALID);
+    flash_write_bytes((uint8_t *)&flags, s_curr_state.log_start_addr,
+        sizeof(flags));
+  }
+
+  return (true);
+}
+
+// Extract the next log message out of flash and send it using the DumpLineCallback.
+// This system task callback is used by flash_dump_log_file()
+static void prv_dump_log_system_cb(void *context) {
+  DumpStatus status = DumpStatus_DoneFailure;
+  DumpLogState *state = (DumpLogState *)context;
+
+  // Get the start address of the current page
+  uint32_t flash_addr = prv_get_page_addr(state->log_start_addr, state->page_index * LOG_PAGE_SIZE);
+
+  if (!state->sent_build_id) {
+    // dump header data first
+
+    // use the read buffer to also hold build id str. Each byte of the build ID requires 2
+    // characters.
+    const int off = MEMBER_SIZE(DumpLogState, msg_buf)
+                    - 2 * MEMBER_SIZE(FlashLoggingHeader, build_id) - 1;
+    uint8_t build_id[MEMBER_SIZE(FlashLoggingHeader, build_id)];
+    uint32_t build_id_addr = flash_addr + offsetof(FlashLoggingHeader, build_id);
+
+    flash_read_bytes((uint8_t *)build_id, build_id_addr, sizeof(build_id));
+    byte_stream_to_hex_string((char *)&state->msg_buf[off], MAX_MSG_LEN - off, (uint8_t *)build_id,
+                              sizeof(build_id), false);
+    int len = pbl_log_get_bin_format((char *)state->msg_buf, MAX_MSG_LEN, LOG_LEVEL_INFO, "", 0,
+                                     "Build ID: %s", &state->msg_buf[off]);
+
+    if (!state->line_cb(state->msg_buf, len)) {
+      // Failed to send, if we expired our retry count, fail
+      if (++state->retry_count >= DUMP_LOG_MAX_RETRIES) {
+        goto exit;
+      }
+    } else {
+      // Go into reading the log messages now.
+      state->sent_build_id = true;
+      state->retry_count = 0;
+      state->page_offset = sizeof(FlashLoggingHeader);
+    }
+    status = DumpStatus_InProgress;
+    goto exit;
+  }
+
+  // Read next log message and send it out
+  LogRecordHeader rec;
+  flash_read_bytes((uint8_t *)&rec, flash_addr + state->page_offset, sizeof(rec));
+  bool page_done = false;
+  if ((rec.length > MAX_MSG_LEN) || (rec.length == 0)) {
+    // The record contents indicate the end of a page
+    page_done = true;
+
+  } else {
+    // This record has data, read it out
+    if ((~rec.flags & LOG_FLAGS_VALID) != 0) {
+      // read data and execute callback to dump data
+      flash_read_bytes(state->msg_buf, flash_addr + state->page_offset + sizeof(rec), rec.length);
+      if (!state->line_cb(state->msg_buf, rec.length)) {
+        if (++state->retry_count >= DUMP_LOG_MAX_RETRIES) {
+          goto exit;
+        }
+      }
+    }
+
+    // Onto the next record
+    status = DumpStatus_InProgress;
+    state->retry_count = 0;
+    state->page_offset += rec.length + sizeof(rec);
+  }
+
+
+  // If we're done with this page, onto the next
+  if (page_done || state->page_offset + sizeof(LogRecordHeader) >= LOG_PAGE_SIZE) {
+    state->page_index++;
+    state->page_offset = sizeof(FlashLoggingHeader);
+    if (state->page_index >= state->num_pages) {
+      status = DumpStatus_DoneSuccess;
+    } else {
+      status = DumpStatus_InProgress;
+      PBL_LOG(LOG_LEVEL_DEBUG, "Dumping page %d of %d", state->page_index, state->num_pages-1);
+    }
+  }
+
+exit:
+  if (status == DumpStatus_DoneFailure || status == DumpStatus_DoneSuccess) {
+    state->completed_cb(status == DumpStatus_DoneSuccess);
+    kernel_free(state);
+  } else {
+    // Keep going
+    system_task_add_callback(prv_dump_log_system_cb, state);
+  }
+}
+
+
+bool flash_dump_log_file(int generation, DumpLineCallback line_cb,
+                         DumpCompletedCallback completed_cb) {
+  uint8_t log_file_id = generation_to_log_file_id(generation);
+
+  uint32_t log_start_addr;
+  int num_log_pages = prv_get_start_of_log_file(log_file_id, &log_start_addr);
+  PBL_LOG(LOG_LEVEL_DEBUG, "Dumping generation %d, %d pages", generation, num_log_pages);
+
+  if (num_log_pages == 0) {
+    completed_cb(false);
+    return (false); // no match found
+  }
+
+  DumpLogState *state = kernel_malloc_check(sizeof(DumpLogState));
+
+  // Init our state
+  *state = (DumpLogState) {
+    .log_start_addr = log_start_addr,
+    .page_index = 0,
+    .num_pages = num_log_pages,
+    .page_offset = sizeof(FlashLoggingHeader),
+    .line_cb = line_cb,
+    .completed_cb = completed_cb,
+  };
+
+  // Kick it off
+  system_task_add_callback(prv_dump_log_system_cb, state);
+  return (true);
+}
+
+//! For unit tests
+void test_flash_logging_get_info(uint32_t *tot_size, uint32_t *erase_unit_size,
+    uint32_t *chunk_size, uint32_t *page_hdr_size) {
+  *tot_size = LOG_REGION_SIZE;
+  *erase_unit_size = ERASE_UNIT_SIZE;
+  *chunk_size = LOG_PAGE_SIZE;
+  *page_hdr_size = sizeof(FlashLoggingHeader);
+}

src/fw/debug/flash_logging.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <inttypes.h>
+#include <stdbool.h>
+
+//!
+//! To work as expected:
+//!  - Flash logging needs to span at least two erase units
+//!  - Flash logging needs to be aligned on erase unit boundaries
+//!
+
+void flash_logging_init(void);
+
+#define FLASH_LOG_INVALID_ADDR UINT32_MAX
+
+//! Find space for a log message with a given size and write the header.
+//!
+//! @return The flash address the message should start at or
+//! FLASH_LOG_INVALID_ADDR if no address could be allocated
+uint32_t flash_logging_log_start(uint8_t msg_length);
+
+//! Performs a log message write
+//!
+//! @return True if the message write was successful, false otherwise
+bool flash_logging_write(const uint8_t *data_to_write, uint32_t flash_addr,
+    uint32_t data_length);
+
+//! Allows a user to disable/enable flash logging after flash_logging_init()
+//! has been called.
+void flash_logging_set_enabled(bool enabled);
+
+typedef bool (*DumpLineCallback)(uint8_t *message, uint32_t total_length);
+typedef void (*DumpCompletedCallback)(bool success);
+
+//! Dump the flash logs of a given generation number
+//!
+//! @param generation - The saved logs to dump. Generation number indicates
+//!    what boot we want to grab logs from where 0 indicates the current boot, 1
+//!    indicates the previous boot, etc
+//!
+//! @param line_cb - The callback to invoke on each log message found for the
+//!    specified generation
+//!
+//! @param completed_cb - The callback to invoke after all messages have been sent to
+//!     the line_cb. This is also called (with false) if the generation does not exist.
+//!
+//! @return True if the log generation existed
+bool flash_dump_log_file(int generation, DumpLineCallback line_cb,
+                         DumpCompletedCallback completed_cb);

src/fw/debug/legacy/debug_db.c

@@ -0,0 +1,229 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "debug_db.h"
+
+#include "drivers/flash.h"
+#include "flash_region/flash_region.h"
+#include "system/logging.h"
+#include "system/passert.h"
+#include "util/attributes.h"
+#include "util/math.h"
+#include "system/version.h"
+
+#include <inttypes.h>
+
+//! @file debug_db.c
+//!
+//! The flash space is divided into multiple files, and those files are further divided into multiple chunks. Every time
+//! the system boots up a different file is used. This leaves the file from the previous boot intact in case we previously
+//! crashed.
+//!
+//! Files are referred to in multiple ways. The "file generation" is how recent the file is. 0 is the generation of the current
+//! boot, 1 is the generation of the previous boot, and so on. The "file index" is which physical slot the file is in. File index
+//! 0 has the lowest address in flash, where DEBUG_DB_NUM_FILES-1 has the highest. The "file id" is an id that is used to identify
+//! which generation the file is in. See debug_db_determine_current_index for the logic that is used to convert file ids into
+//! generations.
+//!
+//! The layout for each file looks like the following.
+//!
+//!  Header
+//!     +  Metrics
+//!     v     v      Logs
+//!    +--+--------+-------------------------------------+
+//!    |  |        |                                     |
+//!    |  |        |                                     |
+//!    +--+--------+-------------------------------------+
+//!
+
+#define FILE_SIZE_BYTES ((FLASH_REGION_DEBUG_DB_END - FLASH_REGION_DEBUG_DB_BEGIN) / DEBUG_DB_NUM_FILES)
+
+#define FILE_ID_BIT_WIDTH 4
+#define VERSION_ID_BIT_WIDTH 2
+
+#define CURRENT_VERSION_ID 1
+
+
+typedef struct PACKED {
+  uint8_t magic:2; //<! Set to 0x2 if valid.
+
+  //! The file id of this file. We always replace the oldest of the two files if they're both
+  //! valid. Serial distance is used to determine if the id overflowed or not.
+  uint8_t file_id:FILE_ID_BIT_WIDTH;
+
+  //! Just in case we have to change this struct in the future.
+  uint8_t version_id:VERSION_ID_BIT_WIDTH;
+} FileHeaderBasic;
+
+typedef struct PACKED {
+  char version_tag[FW_METADATA_VERSION_TAG_BYTES];
+  uint8_t is_recovery;
+} FileHeaderDetails;
+
+typedef struct PACKED {
+  FileHeaderBasic basic;
+  FileHeaderDetails details;
+} FileHeader;
+
+//! This value is chosen because older style (pre In-N-Out) filesystems set the first bit to zero to indicate that it's a
+//! valid chunk. We should consider those invalid (different format) so we want to see a 1 there if it's actually a post-In-N-Out
+//! file. Then, we set the second bit to 0 to differentiate it from unformatted SPI flash, as newly erased SPI flash will have the
+//! value 0x03 (both bits set).
+static const uint8_t VALID_FILE_HEADER_MAGIC = 0x02;
+
+//! Which file we're writing to this boot. [0 - DEBUG_DB_NUM_FILES)
+static int s_current_file_index;
+//! The id we're using for the current file.
+static uint8_t s_current_file_id;
+
+static int generation_to_index(int file_generation) {
+  int index = s_current_file_index - file_generation;
+  if (index < 0) {
+    index += DEBUG_DB_NUM_FILES;
+  }
+  return index;
+}
+
+static uint32_t get_file_address(int file_index) {
+  return FLASH_REGION_DEBUG_DB_BEGIN + (file_index * FILE_SIZE_BYTES);
+}
+
+static uint32_t get_current_file_address(void) {
+  return get_file_address(generation_to_index(0));
+}
+
+//! Get next FILE_ID_BIT_WIDTH bit value
+static uint8_t get_next_file_id(uint8_t file_id) {
+  return (file_id + 1) % (1 << (FILE_ID_BIT_WIDTH));
+}
+
+// Make sure this is out of range for uint8_t:FILE_ID_BIT_WIDTH.
+static const uint8_t INVALID_FILE_ID = 0xff;
+
+void debug_db_determine_current_index(uint8_t* file_id, int* current_file_index, uint8_t* current_file_id) {
+  for (int i = 0; i < DEBUG_DB_NUM_FILES; ++i) {
+    // If we find an unused slot, use that one. We fill in slots from left to right,
+    // so the first one we find when searching left to right is the one we should use.
+    if (file_id[i] == INVALID_FILE_ID) {
+      *current_file_index = i;
+      if (i == 0) {
+        *current_file_id = 0;
+      } else {
+        *current_file_id = get_next_file_id(file_id[i - 1]);
+      }
+      return;
+    }
+
+    if (i != 0) {
+      // If we find a reduction in an id, this is the end of the sequence and we've found
+      // the oldest file. For example, if the IDs are (5, 6, 3, 4), when we find three we'll
+      // see that the ids have stopped increasing. We should be using index 2 with an id of 7.
+      int32_t distance = serial_distance(file_id[i - 1], file_id[i], FILE_ID_BIT_WIDTH);
+      if (distance < 0 || distance > 2) {
+        *current_file_id = get_next_file_id(file_id[i - 1]);
+        *current_file_index = i;
+        return;
+      }
+    }
+  }
+
+  // Everything was increasing which means everything was in order from oldest to newest
+  // and we need to wrap around.
+  *current_file_index = 0;
+  *current_file_id = get_next_file_id(file_id[DEBUG_DB_NUM_FILES - 1]);
+}
+
+void debug_db_init(void) {
+  // Scan the flash to find out what the two file ids are
+  uint8_t file_id[DEBUG_DB_NUM_FILES] = { INVALID_FILE_ID, INVALID_FILE_ID };
+
+  for (int i = 0; i < DEBUG_DB_NUM_FILES; ++i) {
+    FileHeaderBasic file_header;
+    flash_read_bytes((uint8_t*) &file_header, get_file_address(i), sizeof(file_header));
+    if (file_header.magic == VALID_FILE_HEADER_MAGIC && file_header.version_id == CURRENT_VERSION_ID) {
+      file_id[i] = file_header.file_id;
+    }
+  }
+
+  debug_db_determine_current_index(file_id, &s_current_file_index, &s_current_file_id);
+
+  PBL_LOG(LOG_LEVEL_DEBUG, "Found files {%u, %u, %u, %u}, using file %u with new id %u",
+        file_id[0], file_id[1], file_id[2], file_id[3], s_current_file_index, s_current_file_id);
+
+  debug_db_reformat_header_section();
+}
+
+bool debug_db_is_generation_valid(int file_generation) {
+  PBL_ASSERTN(file_generation >= 0 && file_generation < DEBUG_DB_NUM_FILES);
+
+  FileHeaderBasic file_header;
+  flash_read_bytes((uint8_t*) &file_header, get_file_address(generation_to_index(file_generation)), sizeof(file_header));
+
+  if (file_header.magic != VALID_FILE_HEADER_MAGIC) {
+    return false;
+  }
+
+  if (file_header.version_id != CURRENT_VERSION_ID) {
+    return false;
+  }
+
+  if (file_header.file_id != (s_current_file_id - file_generation)) {
+    PBL_LOG(LOG_LEVEL_DEBUG, "Id: %"PRIu8" Expected: %u", file_header.file_id, (s_current_file_id - file_generation));
+    return false;
+  }
+
+  return true;
+}
+
+uint32_t debug_db_get_stats_base_address(int file_generation) {
+  PBL_ASSERTN(file_generation >= 0 && file_generation < DEBUG_DB_NUM_FILES);
+
+  return get_file_address(generation_to_index(file_generation)) + sizeof(FileHeader);
+}
+
+uint32_t debug_db_get_logs_base_address(int file_generation) {
+  PBL_ASSERTN(file_generation >= 0 && file_generation < DEBUG_DB_NUM_FILES);
+
+  return get_file_address(generation_to_index(file_generation)) + SECTION_HEADER_SIZE_BYTES;
+}
+
+void debug_db_reformat_header_section(void) {
+  flash_erase_subsector_blocking(get_current_file_address());
+
+  FirmwareMetadata md;
+  bool result = version_copy_running_fw_metadata(&md);
+  PBL_ASSERTN(result);
+
+  FileHeader file_header = {
+    .basic = {
+      .magic = VALID_FILE_HEADER_MAGIC,
+      .file_id = s_current_file_id,
+      .version_id = CURRENT_VERSION_ID
+    },
+    .details = {
+      .version_tag = "",
+      .is_recovery = md.is_recovery_firmware ? 1 : 0
+    }
+  };
+  strncpy(file_header.details.version_tag, md.version_tag, sizeof(md.version_tag));
+
+  flash_write_bytes((const uint8_t*) &file_header, get_current_file_address(), sizeof(file_header));
+}
+
+uint32_t debug_db_get_stat_section_size(void) {
+  return SECTION_HEADER_SIZE_BYTES - sizeof(FileHeader);
+}
+

src/fw/debug/legacy/debug_db.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+// These values need to multiples of subsectors (4k) to make them easy to erase.
+//! Includes the header + the metrics
+#define SECTION_HEADER_SIZE_BYTES 4096 // Contains both the file header and the stats
+#define SECTION_LOGS_SIZE_BYTES (4096 * 7)
+
+#define DEBUG_DB_NUM_FILES 4
+
+void debug_db_determine_current_index(uint8_t* file_id, int* current_file_index, uint8_t* current_file_id);
+
+void debug_db_init(void);
+
+bool debug_db_is_generation_valid(int file_generation);
+uint32_t debug_db_get_stats_base_address(int file_generation);
+uint32_t debug_db_get_logs_base_address(int file_generation);
+
+void debug_db_reformat_header_section(void);
+
+uint32_t debug_db_get_stat_section_size(void);
+

src/fw/debug/legacy/flash_logging.c

@@ -0,0 +1,247 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "debug_db.h"
+
+#include "board/board.h"
+#include "debug/flash_logging.h"
+#include "drivers/flash.h"
+#include "drivers/rtc.h"
+#include "drivers/watchdog.h"
+#include "flash_region/flash_region.h"
+#include "kernel/pbl_malloc.h"
+#include "system/logging.h"
+#include "system/passert.h"
+#include "kernel/util/sleep.h"
+#include "util/attributes.h"
+#include "util/math.h"
+#include "util/net.h"
+
+#include <inttypes.h>
+
+//! @file flash_logging.c
+//! Logs messages to SPI flash for later retreival.
+//!
+//! The different chunks allow us to implement a rolling log, where if we fill up all the chunks, we can erase the oldest
+//! chunk to find us some more space. Each chunk gets it's own header at the top of the chunk to indicate the order in
+//! which the chunks should be reassembled.
+
+//! Make sure chunks are still an even number of flash subsectors. Our log space is 7 subsectors, so our NUM_CHUNKS
+//! makes it so each chunk has it's own subsector.
+#define NUM_CHUNKS 7
+
+#define CHUNK_SIZE_BYTES (SECTION_LOGS_SIZE_BYTES / NUM_CHUNKS)
+
+#define CHUNK_ID_BIT_WIDTH 8
+
+//! None of the values in this struct are allowed to be equal to 0xff. 0xff is used as an invalid
+//! value (as the spi flash sets bytes to 0xff when they're erased)
+typedef struct PACKED {
+  bool invalid:1; //!< Whether or not this chunk is formatted
+  bool valid:1; //!< Set to 0 when the chunk is stale.
+
+  //! The ID of the current chunk. Each chunk in the gets an auto-incrementing ID. This
+  //! allows the logging infrastructure to find the head and tail of the circular buffer after a
+  //! reboot. The oldest chunk will also have the lowest ID.
+  uint8_t chunk_id:CHUNK_ID_BIT_WIDTH; // :6
+} LogChunkHeader;
+
+typedef struct PACKED {
+  //! The length of the log message after this header, not including this header. If this value
+  //! is 0xff that means no log message follows. If this value is 0x0 this means there are no
+  //! more logs remaining in this chunk.
+  uint8_t log_length;
+} LogHeader;
+
+//! Which chunk we're writing to. [0 - NUM_CHUNKS)
+static int s_current_chunk;
+//! The id we're using for the current chunk.
+static int s_current_chunk_id;
+
+//! The current offset in the chunk in bytes. [0 - CHUNK_SIZE_BYTES)
+static int s_current_offset;
+
+static bool s_enabled;
+
+static uint32_t get_current_address(int chunk, int offset) {
+  return debug_db_get_logs_base_address(0) + (chunk * CHUNK_SIZE_BYTES) + offset;
+}
+
+static uint32_t get_generation_address(int generation, int chunk, int offset) {
+  return debug_db_get_logs_base_address(generation) + (chunk * CHUNK_SIZE_BYTES) + offset;
+}
+
+//! Get next 8 bit value, avoiding 0xff (all bits set)
+static uint8_t get_next_chunk_id(uint8_t chunk_id) {
+  return (chunk_id + 1) % (1 << (CHUNK_ID_BIT_WIDTH));
+}
+
+static void format_current_chunk(void) {
+  uint32_t addr = get_current_address(s_current_chunk, 0);
+  PBL_ASSERT((addr & (SUBSECTOR_SIZE_BYTES - 1)) == 0,
+      "Sections must be subsector aligned! addr is 0x%" PRIx32, addr);
+  PBL_ASSERT((CHUNK_SIZE_BYTES & (SUBSECTOR_SIZE_BYTES - 1)) == 0,
+      "Sections divide into subsectors evenly, size is 0x%" PRIx16, CHUNK_SIZE_BYTES);
+
+  for (unsigned int i = 0; i < (CHUNK_SIZE_BYTES / SUBSECTOR_SIZE_BYTES); ++i) {
+    flash_erase_subsector_blocking(addr + (i * SUBSECTOR_SIZE_BYTES));
+  }
+
+  LogChunkHeader chunk_header = {
+    .invalid = false,
+    .valid = true,
+    .chunk_id = s_current_chunk_id
+  };
+  flash_write_bytes((const uint8_t*) &chunk_header, get_current_address(s_current_chunk, 0), sizeof(chunk_header));
+
+  s_current_offset = sizeof(LogChunkHeader);
+}
+
+static void make_space_for_log(int length) {
+  if (s_current_offset + sizeof(LogHeader) + length + sizeof(LogHeader) < CHUNK_SIZE_BYTES) {
+    // We got space, nothing to do here
+    return;
+  }
+
+  // Need to roll over to the next chunk
+
+  // Seal off the current chunk with a 0 length log message.
+  LogHeader log_header = { .log_length = 0 };
+  flash_write_bytes((const uint8_t*) &log_header, get_current_address(s_current_chunk, s_current_offset), sizeof(log_header));
+
+  // Set up the next chunk.
+  s_current_chunk = (s_current_chunk + 1) % NUM_CHUNKS;
+  s_current_chunk_id = get_next_chunk_id(s_current_chunk_id);
+  format_current_chunk();
+}
+
+uint32_t flash_logging_log_start(uint8_t msg_length) {
+  make_space_for_log(msg_length);
+
+  LogHeader log_header = { .log_length = msg_length };
+  flash_write_bytes((const uint8_t*) &log_header, get_current_address(s_current_chunk, s_current_offset), sizeof(log_header));
+  s_current_offset += sizeof(log_header);
+
+  uint32_t addr = get_current_address(s_current_chunk, s_current_offset);
+  s_current_offset += msg_length;
+  return addr;
+}
+
+bool flash_logging_write(const uint8_t *data_to_write, uint32_t flash_addr,
+    uint32_t data_length) {
+  flash_write_bytes(data_to_write, flash_addr, data_length);
+  return (true);
+}
+
+void flash_logging_init(void) {
+  debug_db_init();
+  s_current_chunk = 0;
+  s_current_chunk_id = 0;
+
+  // Formatting the file we're going to use by erasing the first chunk and writing a new header.
+  format_current_chunk();
+
+  // Mark all the other chunks as stale. This will mark the "valid" member of LogChunkHeader to 0.
+  for (int i = 1; i < NUM_CHUNKS; ++i) {
+    uint8_t zero = 0;
+    flash_write_bytes(&zero, get_current_address(i, 0), sizeof(zero));
+  }
+
+  s_enabled = true;
+}
+
+// Dumping commands
+///////////////////////////////////////////////////////////////////////////////
+
+static bool dump_chunk(int generation, int chunk_index, DumpLineCallback cb) {
+  int offset = sizeof(LogChunkHeader);
+  uint8_t* read_buffer = kernel_malloc(256);
+  bool error = false;
+
+  while (!error) {
+    LogHeader log_header;
+    flash_read_bytes((uint8_t*) &log_header, get_generation_address(generation, chunk_index, offset), sizeof(LogHeader));
+
+    if (log_header.log_length == 0 || log_header.log_length == 0xff) {
+      break;
+    }
+
+    offset += sizeof(LogHeader);
+
+    flash_read_bytes(read_buffer, get_generation_address(generation, chunk_index, offset), log_header.log_length);
+    offset += log_header.log_length;
+
+    int retries = 3;
+    while (true) {
+      --retries;
+      if (cb(read_buffer, log_header.log_length)) {
+        break;
+      } else if (retries == 0) {
+        error = true;
+        break;
+      }
+    }
+  }
+
+  kernel_free(read_buffer);
+  return !error;
+}
+
+bool flash_dump_log_file(int generation, DumpLineCallback cb, DumpCompletedCallback completed_cb) {
+  if (generation < 0 || generation >= DEBUG_DB_NUM_FILES) {
+    completed_cb(false);
+    return false;
+  }
+
+  if (!debug_db_is_generation_valid(generation)) {
+    completed_cb(false);
+    return (false);
+  }
+
+  int lowest_chunk_index = 0;
+  uint8_t lowest_chunk_id = 0;
+  int num_valid_chunks = 0;
+
+  for (int i = 0; i < NUM_CHUNKS; ++i) {
+    LogChunkHeader chunk_header;
+    flash_read_bytes((uint8_t*) &chunk_header, get_generation_address(generation, i, 0), sizeof(chunk_header));
+
+    if (chunk_header.invalid || !chunk_header.valid) {
+      // No more valid chunks
+      break;
+    }
+
+    if (i == 0 || serial_distance(lowest_chunk_id, chunk_header.chunk_id, CHUNK_ID_BIT_WIDTH) < 0) {
+      lowest_chunk_index = i;
+      lowest_chunk_id = chunk_header.chunk_id;
+    }
+
+    ++num_valid_chunks;
+  }
+
+  bool success = true;
+  for (int i = 0; success && i < num_valid_chunks; ++i) {
+    success = dump_chunk(generation, lowest_chunk_index, cb);
+    lowest_chunk_index = (lowest_chunk_index + 1) % NUM_CHUNKS;
+  }
+
+  completed_cb(num_valid_chunks != 0);
+  return true;
+}
+
+void flash_logging_set_enabled(bool enabled) {
+  s_enabled = enabled;
+}

src/fw/debug/power_tracking.c

@@ -0,0 +1,185 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "debug/power_tracking.h"
+
+#include "drivers/rtc.h"
+#include "services/common/regular_timer.h"
+#include "system/logging.h"
+#include "system/passert.h"
+
+#if !defined(SW_POWER_TRACKING)
+
+// a few dummy empty functions that should end up being compiled out
+void power_tracking_init(void) {
+}
+
+void power_tracking_start(PowerSystem system) {
+  // sanitize all uses of this function when you implement it
+  (void) system;
+}
+
+void power_tracking_stop(PowerSystem system) {
+  // sanitize all uses of this function when you implement it
+  (void) system;
+}
+
+#else // SW_POWER_TRACKING
+
+static bool s_initialized = false;
+
+typedef struct {
+  const char* const name;
+  RtcTicks start_ticks;
+  RtcTicks total_ticks;
+  bool dirty;
+} DiscreteSystemProfile;
+
+static DiscreteSystemProfile s_discrete_consumer_profiles[num_power_systems] = {
+  [PowerSystem2v5Reg] =           { "2v5Reg", 0, 0, false},
+  [PowerSystem5vReg] =            { "5vReg", 0, 0, false},
+  [PowerSystemMcuCoreSleep] =     { "McuCoreSleep", 0, 0, false},
+  [PowerSystemMcuCoreRun] =       { "McuCoreRun", 0, 0, false},
+  [PowerSystemMcuGpioA] =         { "McuGpioA", 0, 0, false},
+  [PowerSystemMcuGpioB] =         { "McuGpioB", 0, 0, false},
+  [PowerSystemMcuGpioC] =         { "McuGpioC", 0, 0, false},
+  [PowerSystemMcuGpioD] =         { "McuGpioD", 0, 0, false},
+  [PowerSystemMcuGpioH] =         { "McuGpioH", 0, 0, false},
+  [PowerSystemMcuCrc] =           { "McuCrc", 0, 0, false},
+  [PowerSystemMcuPwr] =           { "McuPwr", 0, 0, false},
+  [PowerSystemMcuDma1] =          { "McuDma1", 0, 0, false},
+  [PowerSystemMcuDma2] =          { "McuDma2", 0, 0, false},
+  [PowerSystemMcuTim1] =          { "McuTim1", 0, 0, false},
+  [PowerSystemMcuTim3] =          { "McuTim3", 0, 0, false},
+  [PowerSystemMcuTim4] =          { "McuTim4", 0, 0, false},
+  [PowerSystemMcuUsart1] =        { "McuUsart1", 0, 0, false},
+  [PowerSystemMcuUsart3] =        { "McuUsart3", 0, 0, false},
+  [PowerSystemMcuI2C1] =          { "McuI2C1", 0, 0, false},
+  [PowerSystemMcuI2C2] =          { "McuI2C2", 0, 0, false},
+  [PowerSystemMcuSpi1] =          { "McuSpi1", 0, 0, false},
+#ifdef PLATFORM_TINTIN
+  [PowerSystemMcuSpi2] =          { "McuSpi2", 0, 0, false},
+#else
+  [PowerSystemMcuSpi6] =          { "McuSpi6", 0, 0, false},
+#endif
+  [PowerSystemMcuAdc1] =          { "McuAdc1", 0, 0, false},
+  [PowerSystemMcuAdc2] =          { "McuAdc2", 0, 0, false},
+  [PowerSystemFlashRead] =        { "FlashRead", 0, 0, false},
+  [PowerSystemFlashWrite] =       { "FlashWrite", 0, 0, false},
+  [PowerSystemFlashErase] =       { "FlashErase", 0, 0, false},
+  [PowerSystemAccelLowPower] =    { "AccelLowPower", 0, 0, false},
+  [PowerSystemAccelNormal] =      { "AccelNormal", 0, 0, false},
+  [PowerSystemMfi] =              { "Mfi", 0, 0, false},
+  [PowerSystemMag] =              { "Mag", 0, 0, false},
+  [PowerSystemBtShutdown] =       { "BtShutdown", 0, 0, false},
+  [PowerSystemBtDeepSleep] =      { "BtDeepSleep", 0, 0, false},
+  [PowerSystemBtActive] =         { "BtActive", 0, 0, false},
+  [PowerSystemAmbient] =          { "Ambient", 0, 0, false},
+  [PowerSystemProfiling] =        { "Profiling", 0, 0, false},
+};
+
+static const uint16_t power_tracking_integration_period_s = 1;
+
+static void power_tracking_flush(void *);
+
+static RegularTimerInfo s_power_profile_timer = {
+  .list_node = { 0, 0 },
+  .cb = power_tracking_flush,
+};
+
+static void power_tracking_flush(void *null) {
+  power_tracking_start(PowerSystemProfiling);
+
+  RtcTicks log_record_time = rtc_get_ticks();
+  char buffer[32];
+
+  for (int i = 0; i<num_power_systems; ++i) {
+    DiscreteSystemProfile *current_profile = &s_discrete_consumer_profiles[i];
+
+    if (current_profile->dirty) {
+      RtcTicks total_ticks = current_profile->total_ticks;
+      if (current_profile->start_ticks != 0) {
+        // the event is still happening - log progress so far
+        RtcTicks current_ticks = rtc_get_ticks();
+        total_ticks += (current_ticks - current_profile->start_ticks);
+        current_profile->start_ticks = current_ticks;
+      } else {
+        // the event is done - clean up
+        current_profile->dirty = false;
+      }
+
+      current_profile->total_ticks = 0;
+
+      if (total_ticks != 0) {
+        // dump the current ticks
+        dbgserial_putstr_fmt(buffer, sizeof(buffer), ">>>PWR:%"PRIu64",%s,%"PRIu64"<", log_record_time, current_profile->name, total_ticks);
+      }
+    }
+  }
+  power_tracking_stop(PowerSystemProfiling);
+}
+
+void power_tracking_init(void) {
+  regular_timer_add_multisecond_callback(&s_power_profile_timer, power_tracking_integration_period_s);
+  char buffer[32];
+  dbgserial_putstr_fmt(buffer, sizeof(buffer), ">>>PWR:%"PRIu64",START,%"PRIu16, rtc_get_ticks(), power_tracking_integration_period_s);
+  s_initialized = true;
+}
+
+void power_tracking_start(PowerSystem system) {
+  if (!s_initialized) {
+    return;
+  }
+  PBL_ASSERTN(system < num_power_systems);
+
+  DiscreteSystemProfile *current_profile = &s_discrete_consumer_profiles[system];
+
+  if (current_profile->start_ticks != 0) {
+    PBL_LOG(LOG_LEVEL_WARNING, "repeat call to start ticks without stopping from %s", current_profile->name);
+    // Someone was careless: two cases:
+    // 1) someone forgot to call stop
+    // 2) someone re-enters a function that calls start before stop is called.
+    return;
+  }
+
+  current_profile->start_ticks = rtc_get_ticks();
+
+  current_profile->dirty = true;
+}
+
+void power_tracking_stop(PowerSystem system) {
+  if (!s_initialized) {
+    return;
+  }
+  PBL_ASSERTN(system < num_power_systems);
+
+  DiscreteSystemProfile *current_profile = &s_discrete_consumer_profiles[system];
+
+  if (current_profile->start_ticks == 0) {
+    PBL_LOG(LOG_LEVEL_WARNING, "Stop ticks before start called: probably losing profile accuracy in %s", current_profile->name);
+    // Someone was careless: two cases:
+    // 1) someone forgot to call start
+    // 2) someone re-entered a function that called stop already, so it is called twice.
+    return;
+  }
+
+  current_profile->total_ticks += (rtc_get_ticks() - current_profile->start_ticks);
+
+  current_profile->start_ticks = 0;
+}
+
+
+#endif // SW_POWER_TRACKING

src/fw/debug/power_tracking.h

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "drivers/rtc.h"
+#include "system/logging.h"
+
+#include <inttypes.h>
+
+/** Power Profiling
+ *  ===============
+ *  There are two main types of power consumers on the Pebble Smartwatch:
+ *  - Discrete systems   (one more more independent power states).
+ *  - Continuous systems (a continuum of power draw eg. the PWM-ed backlight).
+ *
+ *  The discrete systems will have their power profiled in a time-binned manner
+ *  where the on-time of each state is integrated over a pre-determined period.
+ *  General rule of thumb is that non-quiescent states should be tracked.
+ *      eg. don't track BT sniff mode, but do track Active mode because it is more
+ *      of an unusual condition...
+ *
+ *  The continuous systems will just dump their current state whenever it
+ *  is changed.
+ */
+
+// enum that most discrete consumers can fall into.
+
+// TODO: implement tracking on these in decreasing priority order:
+// Spi1, BtShutdown, BtDeepSleep
+// Tim1, Tim3, Tim4, I2C1, I2C2
+// AccelLowPower, AccelNormal, Mag,
+// probably will not implement(either very low power, or constantly on):
+// 5vReg, Pwr, Adc1, Adc2, Ambient, Usart3
+typedef enum {
+  PowerSystem2v5Reg = 0,
+  PowerSystem5vReg,
+  PowerSystemMcuCoreSleep,
+  PowerSystemMcuCoreRun,
+  PowerSystemMcuGpioA,
+  PowerSystemMcuGpioB,
+  PowerSystemMcuGpioC,
+  PowerSystemMcuGpioD,
+  PowerSystemMcuGpioH,
+  PowerSystemMcuCrc,       // Flash
+  PowerSystemMcuPwr,       // Everything
+  PowerSystemMcuDma1,      // Display
+  PowerSystemMcuDma2,      // BT
+  PowerSystemMcuTim1,      // Future use for the vibe PWM
+  PowerSystemMcuTim3,      // Used for the backlight PWM
+  PowerSystemMcuTim4,      // Used for the button debouncer
+  PowerSystemMcuUsart1,    // Used for BT
+  PowerSystemMcuUsart3,    // dbgserial
+  PowerSystemMcuI2C1,      // Main I2C
+  PowerSystemMcuI2C2,      // 2V5 I2C
+  PowerSystemMcuSpi1,      // FLASH
+#if PLATFORM_TINTIN || PLATFORM_SILK
+  PowerSystemMcuSpi2,      // LCD
+#else
+  PowerSystemMcuSpi6,      // LCD
+#endif
+  PowerSystemMcuAdc1,      // Voltage monitoring & ambient light sensing
+  PowerSystemMcuAdc2,      // Voltage monitoring & ambient light sensing
+  PowerSystemFlashRead,
+  PowerSystemFlashWrite,
+  PowerSystemFlashErase,
+  PowerSystemAccelLowPower,
+  PowerSystemAccelNormal,
+  PowerSystemMfi,
+  PowerSystemMag,
+  PowerSystemBtShutdown,
+  PowerSystemBtDeepSleep,
+  PowerSystemBtActive,
+  PowerSystemAmbient,       
+  PowerSystemProfiling,     // So that we can diminish the effects that dumping the profile logs has
+
+  num_power_systems,
+} PowerSystem;
+
+void power_tracking_init(void);
+void power_tracking_start(PowerSystem system);
+void power_tracking_stop(PowerSystem system);
+
+#if defined(SW_POWER_TRACKING)
+  #define PWR_TRACK(system, state_fmt, args...) \
+      { \
+        power_tracking_start(PowerSystemProfiling); \
+        char buffer[64]; \
+        dbgserial_putstr_fmt(buffer, sizeof(buffer), ">>>PWR:%"PRIu64",%s,"state_fmt"<", rtc_get_ticks(), system, ## args); \
+        power_tracking_stop(PowerSystemProfiling); \
+      }
+#else
+  #define PWR_TRACK(system, state_fmt, args...)
+#endif
+
+
+#define PWR_TRACK_BATT(chg_state, voltage)        PWR_TRACK("Battery", "%s,%u", chg_state, voltage)
+#define PWR_TRACK_ACCEL(state, frequency)         PWR_TRACK("Accel", "%s,%u", state, frequency)
+#define PWR_TRACK_MAG(state, adc_rate)            PWR_TRACK("Mag", "%s,%u", state, adc_rate)
+#define PWR_TRACK_VIBE(state, freq, duty)         PWR_TRACK("Vibe", "%s,%u,%u", state, freq, duty)
+#define PWR_TRACK_BACKLIGHT(state, freq, duty)    PWR_TRACK("Backlight", "%s,%"PRIu32",%u", state, freq, duty)

src/fw/debug/setup.c

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "setup.h"
+
+#include "kernel/util/stop.h"
+#include "system/logging.h"
+#include "system/passert.h"
+
+#define STM32F2_COMPATIBLE
+#define STM32F4_COMPATIBLE
+#define STM32F7_COMPATIBLE
+#include <mcu.h>
+
+void enable_mcu_debugging(void) {
+#ifndef RELEASE
+  DBGMCU_Config(DBGMCU_SLEEP | DBGMCU_STOP, ENABLE);
+  // Stop RTC, IWDG & TIM2 during debugging
+  // Note: TIM2 is used by the task watchdog
+  DBGMCU_APB1PeriphConfig(DBGMCU_RTC_STOP | DBGMCU_TIM2_STOP | DBGMCU_IWDG_STOP,
+                          ENABLE);
+#endif
+}
+
+void disable_mcu_debugging(void) {
+  DBGMCU->CR = 0;
+  DBGMCU->APB1FZ = 0;
+  DBGMCU->APB2FZ = 0;
+}
+
+void command_low_power_debug(char *cmd) {
+  bool low_power_debug_on = (strcmp(cmd, "on") == 0);
+
+#ifdef MICRO_FAMILY_STM32F4
+  sleep_mode_enable(!low_power_debug_on);
+#endif
+
+  if (low_power_debug_on) {
+    enable_mcu_debugging();
+  } else {
+    disable_mcu_debugging();
+  }
+}

src/fw/debug/setup.h

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+void enable_mcu_debugging(void);
+
+void disable_mcu_debugging(void);
+